@librechat/agents 3.2.57 → 3.2.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/cjs/agents/AgentContext.cjs +7 -1
  2. package/dist/cjs/agents/AgentContext.cjs.map +1 -1
  3. package/dist/cjs/graphs/Graph.cjs +2 -2
  4. package/dist/cjs/graphs/Graph.cjs.map +1 -1
  5. package/dist/cjs/langfuseToolOutputTracing.cjs +4 -0
  6. package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -1
  7. package/dist/cjs/langfuseTraceShaping.cjs +172 -0
  8. package/dist/cjs/langfuseTraceShaping.cjs.map +1 -0
  9. package/dist/cjs/main.cjs +7 -0
  10. package/dist/cjs/run.cjs +2 -2
  11. package/dist/cjs/run.cjs.map +1 -1
  12. package/dist/cjs/stream.cjs +2 -1
  13. package/dist/cjs/stream.cjs.map +1 -1
  14. package/dist/cjs/tools/BashExecutor.cjs +58 -9
  15. package/dist/cjs/tools/BashExecutor.cjs.map +1 -1
  16. package/dist/cjs/tools/BashProgrammaticToolCalling.cjs +4 -2
  17. package/dist/cjs/tools/BashProgrammaticToolCalling.cjs.map +1 -1
  18. package/dist/cjs/tools/CodeExecutor.cjs +57 -7
  19. package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
  20. package/dist/cjs/tools/ProgrammaticToolCalling.cjs +9 -3
  21. package/dist/cjs/tools/ProgrammaticToolCalling.cjs.map +1 -1
  22. package/dist/cjs/tools/ToolNode.cjs +19 -1
  23. package/dist/cjs/tools/ToolNode.cjs.map +1 -1
  24. package/dist/cjs/tools/eagerEventExecution.cjs +18 -1
  25. package/dist/cjs/tools/eagerEventExecution.cjs.map +1 -1
  26. package/dist/cjs/tools/search/keenable-search.cjs +68 -0
  27. package/dist/cjs/tools/search/keenable-search.cjs.map +1 -0
  28. package/dist/cjs/tools/search/rerankers.cjs +28 -11
  29. package/dist/cjs/tools/search/rerankers.cjs.map +1 -1
  30. package/dist/cjs/tools/search/search.cjs +30 -4
  31. package/dist/cjs/tools/search/search.cjs.map +1 -1
  32. package/dist/cjs/tools/search/tool.cjs +14 -6
  33. package/dist/cjs/tools/search/tool.cjs.map +1 -1
  34. package/dist/cjs/tools/subagent/SubagentExecutor.cjs +12 -1
  35. package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
  36. package/dist/cjs/utils/title.cjs +9 -9
  37. package/dist/cjs/utils/title.cjs.map +1 -1
  38. package/dist/esm/agents/AgentContext.mjs +7 -1
  39. package/dist/esm/agents/AgentContext.mjs.map +1 -1
  40. package/dist/esm/graphs/Graph.mjs +2 -2
  41. package/dist/esm/graphs/Graph.mjs.map +1 -1
  42. package/dist/esm/langfuseToolOutputTracing.mjs +4 -0
  43. package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -1
  44. package/dist/esm/langfuseTraceShaping.mjs +171 -0
  45. package/dist/esm/langfuseTraceShaping.mjs.map +1 -0
  46. package/dist/esm/main.mjs +3 -3
  47. package/dist/esm/run.mjs +2 -2
  48. package/dist/esm/run.mjs.map +1 -1
  49. package/dist/esm/stream.mjs +2 -1
  50. package/dist/esm/stream.mjs.map +1 -1
  51. package/dist/esm/tools/BashExecutor.mjs +56 -10
  52. package/dist/esm/tools/BashExecutor.mjs.map +1 -1
  53. package/dist/esm/tools/BashProgrammaticToolCalling.mjs +4 -2
  54. package/dist/esm/tools/BashProgrammaticToolCalling.mjs.map +1 -1
  55. package/dist/esm/tools/CodeExecutor.mjs +54 -8
  56. package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
  57. package/dist/esm/tools/ProgrammaticToolCalling.mjs +9 -3
  58. package/dist/esm/tools/ProgrammaticToolCalling.mjs.map +1 -1
  59. package/dist/esm/tools/ToolNode.mjs +20 -2
  60. package/dist/esm/tools/ToolNode.mjs.map +1 -1
  61. package/dist/esm/tools/eagerEventExecution.mjs +18 -2
  62. package/dist/esm/tools/eagerEventExecution.mjs.map +1 -1
  63. package/dist/esm/tools/search/keenable-search.mjs +66 -0
  64. package/dist/esm/tools/search/keenable-search.mjs.map +1 -0
  65. package/dist/esm/tools/search/rerankers.mjs +28 -11
  66. package/dist/esm/tools/search/rerankers.mjs.map +1 -1
  67. package/dist/esm/tools/search/search.mjs +30 -4
  68. package/dist/esm/tools/search/search.mjs.map +1 -1
  69. package/dist/esm/tools/search/tool.mjs +14 -6
  70. package/dist/esm/tools/search/tool.mjs.map +1 -1
  71. package/dist/esm/tools/subagent/SubagentExecutor.mjs +12 -1
  72. package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
  73. package/dist/esm/utils/title.mjs +9 -9
  74. package/dist/esm/utils/title.mjs.map +1 -1
  75. package/dist/types/langfuseTraceShaping.d.ts +20 -0
  76. package/dist/types/tools/BashExecutor.d.ts +13 -0
  77. package/dist/types/tools/CodeExecutor.d.ts +14 -0
  78. package/dist/types/tools/ToolNode.d.ts +3 -0
  79. package/dist/types/tools/eagerEventExecution.d.ts +8 -0
  80. package/dist/types/tools/search/keenable-search.d.ts +4 -0
  81. package/dist/types/tools/search/rerankers.d.ts +7 -2
  82. package/dist/types/tools/search/types.d.ts +38 -1
  83. package/dist/types/types/graph.d.ts +22 -0
  84. package/dist/types/types/tools.d.ts +58 -0
  85. package/package.json +1 -1
  86. package/src/agents/AgentContext.ts +9 -0
  87. package/src/agents/__tests__/AgentContext.test.ts +40 -0
  88. package/src/graphs/Graph.ts +25 -2
  89. package/src/graphs/__tests__/composition.smoke.test.ts +53 -0
  90. package/src/langfuseToolOutputTracing.ts +11 -0
  91. package/src/langfuseTraceShaping.ts +280 -0
  92. package/src/llm/anthropic/inherited-stream-events.spec.ts +6 -3
  93. package/src/run.ts +3 -3
  94. package/src/specs/langfuse-routing.integration.test.ts +1 -1
  95. package/src/specs/langfuse-trace-shaping.test.ts +194 -0
  96. package/src/stream.ts +17 -1
  97. package/src/tools/BashExecutor.ts +107 -14
  98. package/src/tools/BashProgrammaticToolCalling.ts +20 -1
  99. package/src/tools/CodeExecutor.ts +113 -9
  100. package/src/tools/ProgrammaticToolCalling.ts +27 -1
  101. package/src/tools/ToolNode.ts +36 -4
  102. package/src/tools/__tests__/BashExecutor.test.ts +39 -0
  103. package/src/tools/__tests__/CodeExecutor.stateful.test.ts +113 -0
  104. package/src/tools/__tests__/SubagentExecutor.test.ts +44 -0
  105. package/src/tools/__tests__/ToolNode.session.test.ts +86 -0
  106. package/src/tools/__tests__/eagerEventExecution.session.test.ts +92 -0
  107. package/src/tools/__tests__/hitl.test.ts +85 -0
  108. package/src/tools/eagerEventExecution.ts +32 -5
  109. package/src/tools/search/jina-reranker.test.ts +70 -1
  110. package/src/tools/search/keenable-search.ts +98 -0
  111. package/src/tools/search/keenable.test.ts +183 -0
  112. package/src/tools/search/rerankers.ts +41 -4
  113. package/src/tools/search/search.ts +58 -2
  114. package/src/tools/search/source-processing.test.ts +86 -0
  115. package/src/tools/search/tool.ts +29 -4
  116. package/src/tools/search/types.ts +42 -1
  117. package/src/tools/subagent/SubagentExecutor.ts +11 -0
  118. package/src/types/graph.ts +22 -0
  119. package/src/types/tools.ts +65 -0
  120. package/src/utils/title.ts +9 -9
@@ -22,6 +22,19 @@ function recordArgsEqual(left, right) {
22
22
  function normalizeError(error) {
23
23
  return error instanceof Error ? error : new Error(String(error));
24
24
  }
25
+ /**
26
+ * Stateful runtime session hint for the remote sandbox: only when
27
+ * `toolExecution.sandbox.statefulSessions` is on; explicit host hint else the
28
+ * conversation `thread_id`. Undefined disables the wire field. Shared by the
29
+ * direct ToolNode path and both event-driven planners so they stay in lockstep.
30
+ */
31
+ function resolveRuntimeSessionHint(toolExecution, threadId) {
32
+ const sandbox = toolExecution?.sandbox;
33
+ if (sandbox?.statefulSessions !== true) return;
34
+ const explicit = sandbox.runtimeSessionHint;
35
+ if (explicit != null && explicit !== "") return explicit;
36
+ return threadId != null && threadId !== "" ? threadId : void 0;
37
+ }
25
38
  function buildToolExecutionRequestPlan(args) {
26
39
  const invalidArgsBehavior = args.invalidArgsBehavior ?? "abort";
27
40
  const prepared = [];
@@ -36,6 +49,7 @@ function buildToolExecutionRequestPlan(args) {
36
49
  args: {},
37
50
  stepId: toolCall.stepId,
38
51
  codeSessionContext: toolCall.codeSessionContext,
52
+ runtimeSessionHint: toolCall.runtimeSessionHint,
39
53
  rejectedErrorMessage: "Invalid tool call arguments: expected a JSON object."
40
54
  });
41
55
  continue;
@@ -45,7 +59,8 @@ function buildToolExecutionRequestPlan(args) {
45
59
  name: toolCall.name,
46
60
  args: coercedArgs,
47
61
  stepId: toolCall.stepId,
48
- codeSessionContext: toolCall.codeSessionContext
62
+ codeSessionContext: toolCall.codeSessionContext,
63
+ runtimeSessionHint: toolCall.runtimeSessionHint
49
64
  });
50
65
  }
51
66
  const nextUsageCount = new Map(args.usageCount);
@@ -60,6 +75,7 @@ function buildToolExecutionRequestPlan(args) {
60
75
  turn
61
76
  };
62
77
  if (toolCall.codeSessionContext != null) request.codeSessionContext = toolCall.codeSessionContext;
78
+ if (toolCall.runtimeSessionHint != null && toolCall.runtimeSessionHint !== "") request.runtimeSessionHint = toolCall.runtimeSessionHint;
63
79
  return request;
64
80
  });
65
81
  const requests = allRequests.filter((_, index) => prepared[index].rejectedErrorMessage == null);
@@ -85,5 +101,6 @@ exports.buildToolExecutionRequestPlan = buildToolExecutionRequestPlan;
85
101
  exports.coerceRecordArgs = coerceRecordArgs;
86
102
  exports.normalizeError = normalizeError;
87
103
  exports.recordArgsEqual = recordArgsEqual;
104
+ exports.resolveRuntimeSessionHint = resolveRuntimeSessionHint;
88
105
 
89
106
  //# sourceMappingURL=eagerEventExecution.cjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"eagerEventExecution.cjs","names":[],"sources":["../../../src/tools/eagerEventExecution.ts"],"sourcesContent":["import type * as t from '@/types';\n\nexport function coerceRecordArgs(\n args: unknown\n): Record<string, unknown> | undefined {\n if (typeof args === 'string') {\n try {\n const parsed = JSON.parse(args) as unknown;\n return coerceRecordArgs(parsed);\n } catch {\n return undefined;\n }\n }\n\n if (args === null || typeof args !== 'object' || Array.isArray(args)) {\n return undefined;\n }\n\n return args as Record<string, unknown>;\n}\n\nexport function stableStringify(value: unknown): string {\n if (Array.isArray(value)) {\n return `[${value.map((item) => stableStringify(item)).join(',')}]`;\n }\n\n if (value !== null && typeof value === 'object') {\n const record = value as Record<string, unknown>;\n const keys = Object.keys(record).sort();\n return `{${keys\n .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)\n .join(',')}}`;\n }\n\n return JSON.stringify(value);\n}\n\nexport function recordArgsEqual(\n left: Record<string, unknown>,\n right: Record<string, unknown>\n): boolean {\n return stableStringify(left) === stableStringify(right);\n}\n\nexport function normalizeError(error: unknown): Error {\n return error instanceof Error ? error : new Error(String(error));\n}\n\nexport type ToolExecutionPlanCall = {\n id?: string;\n name: string;\n args: unknown;\n stepId?: string;\n codeSessionContext?: t.ToolCallRequest['codeSessionContext'];\n};\n\nexport type ToolExecutionRequestPlan = {\n allRequests: t.ToolCallRequest[];\n requests: t.ToolCallRequest[];\n rejectedResults: t.ToolExecuteResult[];\n};\n\nexport function buildToolExecutionRequestPlan(args: {\n toolCalls: ToolExecutionPlanCall[];\n usageCount: Map<string, number>;\n invalidArgsBehavior?: 'abort' | 'error-result';\n recordTurn?: (toolName: string, turn: number, callId: string) => void;\n}): ToolExecutionRequestPlan | undefined {\n const invalidArgsBehavior = args.invalidArgsBehavior ?? 'abort';\n const prepared: Array<{\n id: string;\n name: string;\n args: Record<string, unknown>;\n stepId?: string;\n codeSessionContext?: t.ToolCallRequest['codeSessionContext'];\n rejectedErrorMessage?: string;\n }> = [];\n\n for (const toolCall of args.toolCalls) {\n if (\n toolCall.id == null ||\n toolCall.id === '' ||\n toolCall.name === ''\n ) {\n return undefined;\n }\n const coercedArgs = coerceRecordArgs(toolCall.args);\n if (coercedArgs == null) {\n if (invalidArgsBehavior === 'abort') {\n return undefined;\n }\n prepared.push({\n id: toolCall.id,\n name: toolCall.name,\n args: {},\n stepId: toolCall.stepId,\n codeSessionContext: toolCall.codeSessionContext,\n rejectedErrorMessage:\n 'Invalid tool call arguments: expected a JSON object.',\n });\n continue;\n }\n prepared.push({\n id: toolCall.id,\n name: toolCall.name,\n args: coercedArgs,\n stepId: toolCall.stepId,\n codeSessionContext: toolCall.codeSessionContext,\n });\n }\n\n const nextUsageCount = new Map(args.usageCount);\n const allRequests = prepared.map((toolCall): t.ToolCallRequest => {\n const turn = nextUsageCount.get(toolCall.name) ?? 0;\n nextUsageCount.set(toolCall.name, turn + 1);\n const request: t.ToolCallRequest = {\n id: toolCall.id,\n name: toolCall.name,\n args: toolCall.args,\n stepId: toolCall.stepId,\n turn,\n };\n if (toolCall.codeSessionContext != null) {\n request.codeSessionContext = toolCall.codeSessionContext;\n }\n return request;\n });\n const requests = allRequests.filter(\n (_, index) => prepared[index].rejectedErrorMessage == null\n );\n const rejectedResults = prepared.flatMap((toolCall) => {\n if (toolCall.rejectedErrorMessage == null) {\n return [];\n }\n return [\n {\n toolCallId: toolCall.id,\n status: 'error' as const,\n content: '',\n errorMessage: toolCall.rejectedErrorMessage,\n },\n ];\n });\n\n for (const [toolName, count] of nextUsageCount) {\n args.usageCount.set(toolName, count);\n }\n for (const request of allRequests) {\n args.recordTurn?.(request.name, request.turn ?? 0, request.id);\n }\n\n return { allRequests, requests, rejectedResults };\n}\n"],"mappings":";AAEA,SAAgB,iBACd,MACqC;CACrC,IAAI,OAAO,SAAS,UAClB,IAAI;EAEF,OAAO,iBADQ,KAAK,MAAM,IACG,CAAC;CAChC,QAAQ;EACN;CACF;CAGF,IAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GACjE;CAGF,OAAO;AACT;AAEA,SAAgB,gBAAgB,OAAwB;CACtD,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,KAAK,SAAS,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAGlE,IAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;EAC/C,MAAM,SAAS;EAEf,OAAO,IADM,OAAO,KAAK,MAAM,CAAC,CAAC,KACnB,CAAC,CACZ,KAAK,QAAQ,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,gBAAgB,OAAO,IAAI,GAAG,CAAC,CACtE,KAAK,GAAG,EAAE;CACf;CAEA,OAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAgB,gBACd,MACA,OACS;CACT,OAAO,gBAAgB,IAAI,MAAM,gBAAgB,KAAK;AACxD;AAEA,SAAgB,eAAe,OAAuB;CACpD,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAgBA,SAAgB,8BAA8B,MAKL;CACvC,MAAM,sBAAsB,KAAK,uBAAuB;CACxD,MAAM,WAOD,CAAC;CAEN,KAAK,MAAM,YAAY,KAAK,WAAW;EACrC,IACE,SAAS,MAAM,QACf,SAAS,OAAO,MAChB,SAAS,SAAS,IAElB;EAEF,MAAM,cAAc,iBAAiB,SAAS,IAAI;EAClD,IAAI,eAAe,MAAM;GACvB,IAAI,wBAAwB,SAC1B;GAEF,SAAS,KAAK;IACZ,IAAI,SAAS;IACb,MAAM,SAAS;IACf,MAAM,CAAC;IACP,QAAQ,SAAS;IACjB,oBAAoB,SAAS;IAC7B,sBACE;GACJ,CAAC;GACD;EACF;EACA,SAAS,KAAK;GACZ,IAAI,SAAS;GACb,MAAM,SAAS;GACf,MAAM;GACN,QAAQ,SAAS;GACjB,oBAAoB,SAAS;EAC/B,CAAC;CACH;CAEA,MAAM,iBAAiB,IAAI,IAAI,KAAK,UAAU;CAC9C,MAAM,cAAc,SAAS,KAAK,aAAgC;EAChE,MAAM,OAAO,eAAe,IAAI,SAAS,IAAI,KAAK;EAClD,eAAe,IAAI,SAAS,MAAM,OAAO,CAAC;EAC1C,MAAM,UAA6B;GACjC,IAAI,SAAS;GACb,MAAM,SAAS;GACf,MAAM,SAAS;GACf,QAAQ,SAAS;GACjB;EACF;EACA,IAAI,SAAS,sBAAsB,MACjC,QAAQ,qBAAqB,SAAS;EAExC,OAAO;CACT,CAAC;CACD,MAAM,WAAW,YAAY,QAC1B,GAAG,UAAU,SAAS,MAAM,CAAC,wBAAwB,IACxD;CACA,MAAM,kBAAkB,SAAS,SAAS,aAAa;EACrD,IAAI,SAAS,wBAAwB,MACnC,OAAO,CAAC;EAEV,OAAO,CACL;GACE,YAAY,SAAS;GACrB,QAAQ;GACR,SAAS;GACT,cAAc,SAAS;EACzB,CACF;CACF,CAAC;CAED,KAAK,MAAM,CAAC,UAAU,UAAU,gBAC9B,KAAK,WAAW,IAAI,UAAU,KAAK;CAErC,KAAK,MAAM,WAAW,aACpB,KAAK,aAAa,QAAQ,MAAM,QAAQ,QAAQ,GAAG,QAAQ,EAAE;CAG/D,OAAO;EAAE;EAAa;EAAU;CAAgB;AAClD"}
1
+ {"version":3,"file":"eagerEventExecution.cjs","names":[],"sources":["../../../src/tools/eagerEventExecution.ts"],"sourcesContent":["import type * as t from '@/types';\n\nexport function coerceRecordArgs(\n args: unknown\n): Record<string, unknown> | undefined {\n if (typeof args === 'string') {\n try {\n const parsed = JSON.parse(args) as unknown;\n return coerceRecordArgs(parsed);\n } catch {\n return undefined;\n }\n }\n\n if (args === null || typeof args !== 'object' || Array.isArray(args)) {\n return undefined;\n }\n\n return args as Record<string, unknown>;\n}\n\nexport function stableStringify(value: unknown): string {\n if (Array.isArray(value)) {\n return `[${value.map((item) => stableStringify(item)).join(',')}]`;\n }\n\n if (value !== null && typeof value === 'object') {\n const record = value as Record<string, unknown>;\n const keys = Object.keys(record).sort();\n return `{${keys\n .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)\n .join(',')}}`;\n }\n\n return JSON.stringify(value);\n}\n\nexport function recordArgsEqual(\n left: Record<string, unknown>,\n right: Record<string, unknown>\n): boolean {\n return stableStringify(left) === stableStringify(right);\n}\n\nexport function normalizeError(error: unknown): Error {\n return error instanceof Error ? error : new Error(String(error));\n}\n\nexport type ToolExecutionPlanCall = {\n id?: string;\n name: string;\n args: unknown;\n stepId?: string;\n codeSessionContext?: t.ToolCallRequest['codeSessionContext'];\n runtimeSessionHint?: string;\n};\n\n/**\n * Stateful runtime session hint for the remote sandbox: only when\n * `toolExecution.sandbox.statefulSessions` is on; explicit host hint else the\n * conversation `thread_id`. Undefined disables the wire field. Shared by the\n * direct ToolNode path and both event-driven planners so they stay in lockstep.\n */\nexport function resolveRuntimeSessionHint(\n toolExecution: t.ToolExecutionConfig | undefined,\n threadId: string | undefined\n): string | undefined {\n const sandbox = toolExecution?.sandbox;\n if (sandbox?.statefulSessions !== true) {\n return undefined;\n }\n const explicit = sandbox.runtimeSessionHint;\n if (explicit != null && explicit !== '') {\n return explicit;\n }\n return threadId != null && threadId !== '' ? threadId : undefined;\n}\n\nexport type ToolExecutionRequestPlan = {\n allRequests: t.ToolCallRequest[];\n requests: t.ToolCallRequest[];\n rejectedResults: t.ToolExecuteResult[];\n};\n\nexport function buildToolExecutionRequestPlan(args: {\n toolCalls: ToolExecutionPlanCall[];\n usageCount: Map<string, number>;\n invalidArgsBehavior?: 'abort' | 'error-result';\n recordTurn?: (toolName: string, turn: number, callId: string) => void;\n}): ToolExecutionRequestPlan | undefined {\n const invalidArgsBehavior = args.invalidArgsBehavior ?? 'abort';\n const prepared: Array<{\n id: string;\n name: string;\n args: Record<string, unknown>;\n stepId?: string;\n codeSessionContext?: t.ToolCallRequest['codeSessionContext'];\n runtimeSessionHint?: string;\n rejectedErrorMessage?: string;\n }> = [];\n\n for (const toolCall of args.toolCalls) {\n if (toolCall.id == null || toolCall.id === '' || toolCall.name === '') {\n return undefined;\n }\n const coercedArgs = coerceRecordArgs(toolCall.args);\n if (coercedArgs == null) {\n if (invalidArgsBehavior === 'abort') {\n return undefined;\n }\n prepared.push({\n id: toolCall.id,\n name: toolCall.name,\n args: {},\n stepId: toolCall.stepId,\n codeSessionContext: toolCall.codeSessionContext,\n runtimeSessionHint: toolCall.runtimeSessionHint,\n rejectedErrorMessage:\n 'Invalid tool call arguments: expected a JSON object.',\n });\n continue;\n }\n prepared.push({\n id: toolCall.id,\n name: toolCall.name,\n args: coercedArgs,\n stepId: toolCall.stepId,\n codeSessionContext: toolCall.codeSessionContext,\n runtimeSessionHint: toolCall.runtimeSessionHint,\n });\n }\n\n const nextUsageCount = new Map(args.usageCount);\n const allRequests = prepared.map((toolCall): t.ToolCallRequest => {\n const turn = nextUsageCount.get(toolCall.name) ?? 0;\n nextUsageCount.set(toolCall.name, turn + 1);\n const request: t.ToolCallRequest = {\n id: toolCall.id,\n name: toolCall.name,\n args: toolCall.args,\n stepId: toolCall.stepId,\n turn,\n };\n if (toolCall.codeSessionContext != null) {\n request.codeSessionContext = toolCall.codeSessionContext;\n }\n if (\n toolCall.runtimeSessionHint != null &&\n toolCall.runtimeSessionHint !== ''\n ) {\n request.runtimeSessionHint = toolCall.runtimeSessionHint;\n }\n return request;\n });\n const requests = allRequests.filter(\n (_, index) => prepared[index].rejectedErrorMessage == null\n );\n const rejectedResults = prepared.flatMap((toolCall) => {\n if (toolCall.rejectedErrorMessage == null) {\n return [];\n }\n return [\n {\n toolCallId: toolCall.id,\n status: 'error' as const,\n content: '',\n errorMessage: toolCall.rejectedErrorMessage,\n },\n ];\n });\n\n for (const [toolName, count] of nextUsageCount) {\n args.usageCount.set(toolName, count);\n }\n for (const request of allRequests) {\n args.recordTurn?.(request.name, request.turn ?? 0, request.id);\n }\n\n return { allRequests, requests, rejectedResults };\n}\n"],"mappings":";AAEA,SAAgB,iBACd,MACqC;CACrC,IAAI,OAAO,SAAS,UAClB,IAAI;EAEF,OAAO,iBADQ,KAAK,MAAM,IACG,CAAC;CAChC,QAAQ;EACN;CACF;CAGF,IAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GACjE;CAGF,OAAO;AACT;AAEA,SAAgB,gBAAgB,OAAwB;CACtD,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,KAAK,SAAS,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAGlE,IAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;EAC/C,MAAM,SAAS;EAEf,OAAO,IADM,OAAO,KAAK,MAAM,CAAC,CAAC,KACnB,CAAC,CACZ,KAAK,QAAQ,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,gBAAgB,OAAO,IAAI,GAAG,CAAC,CACtE,KAAK,GAAG,EAAE;CACf;CAEA,OAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAgB,gBACd,MACA,OACS;CACT,OAAO,gBAAgB,IAAI,MAAM,gBAAgB,KAAK;AACxD;AAEA,SAAgB,eAAe,OAAuB;CACpD,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;;;;;;;AAiBA,SAAgB,0BACd,eACA,UACoB;CACpB,MAAM,UAAU,eAAe;CAC/B,IAAI,SAAS,qBAAqB,MAChC;CAEF,MAAM,WAAW,QAAQ;CACzB,IAAI,YAAY,QAAQ,aAAa,IACnC,OAAO;CAET,OAAO,YAAY,QAAQ,aAAa,KAAK,WAAW,KAAA;AAC1D;AAQA,SAAgB,8BAA8B,MAKL;CACvC,MAAM,sBAAsB,KAAK,uBAAuB;CACxD,MAAM,WAQD,CAAC;CAEN,KAAK,MAAM,YAAY,KAAK,WAAW;EACrC,IAAI,SAAS,MAAM,QAAQ,SAAS,OAAO,MAAM,SAAS,SAAS,IACjE;EAEF,MAAM,cAAc,iBAAiB,SAAS,IAAI;EAClD,IAAI,eAAe,MAAM;GACvB,IAAI,wBAAwB,SAC1B;GAEF,SAAS,KAAK;IACZ,IAAI,SAAS;IACb,MAAM,SAAS;IACf,MAAM,CAAC;IACP,QAAQ,SAAS;IACjB,oBAAoB,SAAS;IAC7B,oBAAoB,SAAS;IAC7B,sBACE;GACJ,CAAC;GACD;EACF;EACA,SAAS,KAAK;GACZ,IAAI,SAAS;GACb,MAAM,SAAS;GACf,MAAM;GACN,QAAQ,SAAS;GACjB,oBAAoB,SAAS;GAC7B,oBAAoB,SAAS;EAC/B,CAAC;CACH;CAEA,MAAM,iBAAiB,IAAI,IAAI,KAAK,UAAU;CAC9C,MAAM,cAAc,SAAS,KAAK,aAAgC;EAChE,MAAM,OAAO,eAAe,IAAI,SAAS,IAAI,KAAK;EAClD,eAAe,IAAI,SAAS,MAAM,OAAO,CAAC;EAC1C,MAAM,UAA6B;GACjC,IAAI,SAAS;GACb,MAAM,SAAS;GACf,MAAM,SAAS;GACf,QAAQ,SAAS;GACjB;EACF;EACA,IAAI,SAAS,sBAAsB,MACjC,QAAQ,qBAAqB,SAAS;EAExC,IACE,SAAS,sBAAsB,QAC/B,SAAS,uBAAuB,IAEhC,QAAQ,qBAAqB,SAAS;EAExC,OAAO;CACT,CAAC;CACD,MAAM,WAAW,YAAY,QAC1B,GAAG,UAAU,SAAS,MAAM,CAAC,wBAAwB,IACxD;CACA,MAAM,kBAAkB,SAAS,SAAS,aAAa;EACrD,IAAI,SAAS,wBAAwB,MACnC,OAAO,CAAC;EAEV,OAAO,CACL;GACE,YAAY,SAAS;GACrB,QAAQ;GACR,SAAS;GACT,cAAc,SAAS;EACzB,CACF;CACF,CAAC;CAED,KAAK,MAAM,CAAC,UAAU,UAAU,gBAC9B,KAAK,WAAW,IAAI,UAAU,KAAK;CAErC,KAAK,MAAM,WAAW,aACpB,KAAK,aAAa,QAAQ,MAAM,QAAQ,QAAQ,GAAG,QAAQ,EAAE;CAG/D,OAAO;EAAE;EAAa;EAAU;CAAgB;AAClD"}
@@ -0,0 +1,68 @@
1
+ const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
2
+ require("./schema.cjs");
3
+ let axios = require("axios");
4
+ axios = require_runtime.__toESM(axios, 1);
5
+ //#region src/tools/search/keenable-search.ts
6
+ const DEFAULT_KEENABLE_TIMEOUT = 15e3;
7
+ /** Authenticated and keyless endpoints. Keenable works without an API key by
8
+ * falling back to the public endpoint; a key only lifts rate limits. */
9
+ const KEENABLE_DEFAULT_API_URL = "https://api.keenable.ai/v1/search";
10
+ const KEENABLE_PUBLIC_API_URL = "https://api.keenable.ai/v1/search/public";
11
+ const KEENABLE_DATE_RANGES = {
12
+ ["h"]: "1h",
13
+ ["d"]: "1d",
14
+ ["w"]: "7d",
15
+ ["m"]: "1mo",
16
+ ["y"]: "1y"
17
+ };
18
+ const createKeenableAPI = (apiKey, apiUrl, options) => {
19
+ const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;
20
+ const hasKey = resolvedKey != null && resolvedKey !== "";
21
+ const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;
22
+ const resolvedUrl = apiUrl ?? process.env.KEENABLE_API_URL ?? (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);
23
+ /** Constant for the provider's lifetime. X-Keenable-Title is required for
24
+ * keyless requests and used for traffic attribution; the API key only lifts
25
+ * rate limits, so it is sent only when present. */
26
+ const headers = {
27
+ "Content-Type": "application/json",
28
+ "X-Keenable-Title": options?.attributionTitle ?? "LibreChat"
29
+ };
30
+ if (hasKey) headers["X-API-Key"] = resolvedKey;
31
+ const getSources = async ({ query, date, numResults = 8 }) => {
32
+ if (!query.trim()) return {
33
+ success: false,
34
+ error: "Query cannot be empty"
35
+ };
36
+ try {
37
+ /** Keenable's endpoint has no result-count parameter; the count is
38
+ * applied client-side after the response (see slice below). */
39
+ const payload = { query };
40
+ if (options?.site != null && options.site !== "") payload.site = options.site;
41
+ if (date != null) payload.published_after = KEENABLE_DATE_RANGES[date];
42
+ const response = await axios.default.post(resolvedUrl, payload, {
43
+ headers,
44
+ timeout
45
+ });
46
+ const maxResults = Math.min(Math.max(1, options?.maxResults ?? numResults), 20);
47
+ return {
48
+ success: true,
49
+ data: { organic: (Array.isArray(response.data.results) ? response.data.results.slice(0, maxResults) : []).map((result) => ({
50
+ title: result.title ?? "",
51
+ link: result.url ?? "",
52
+ snippet: result.description ?? result.snippet ?? "",
53
+ date: result.published_at
54
+ })) }
55
+ };
56
+ } catch (error) {
57
+ return {
58
+ success: false,
59
+ error: `Keenable API request failed: ${error instanceof Error ? error.message : String(error)}`
60
+ };
61
+ }
62
+ };
63
+ return { getSources };
64
+ };
65
+ //#endregion
66
+ exports.createKeenableAPI = createKeenableAPI;
67
+
68
+ //# sourceMappingURL=keenable-search.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"keenable-search.cjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { DATE_RANGE } from './schema';\n\nconst DEFAULT_KEENABLE_TIMEOUT = 15000;\n\n/** Authenticated and keyless endpoints. Keenable works without an API key by\n * falling back to the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_DEFAULT_API_URL = 'https://api.keenable.ai/v1/search';\nconst KEENABLE_PUBLIC_API_URL = 'https://api.keenable.ai/v1/search/public';\nconst KEENABLE_DATE_RANGES: Record<DATE_RANGE, string> = {\n [DATE_RANGE.PAST_HOUR]: '1h',\n [DATE_RANGE.PAST_24_HOURS]: '1d',\n [DATE_RANGE.PAST_WEEK]: '7d',\n [DATE_RANGE.PAST_MONTH]: '1mo',\n [DATE_RANGE.PAST_YEAR]: '1y',\n};\n\nexport const createKeenableAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.KeenableSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;\n const hasKey = resolvedKey != null && resolvedKey !== '';\n const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;\n const resolvedUrl =\n apiUrl ??\n process.env.KEENABLE_API_URL ??\n (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);\n\n /** Constant for the provider's lifetime. X-Keenable-Title is required for\n * keyless requests and used for traffic attribution; the API key only lifts\n * rate limits, so it is sent only when present. */\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n 'X-Keenable-Title': options?.attributionTitle ?? 'LibreChat',\n };\n if (hasKey) {\n headers['X-API-Key'] = resolvedKey;\n }\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n /** Keenable's endpoint has no result-count parameter; the count is\n * applied client-side after the response (see slice below). */\n const payload: t.KeenableSearchPayload = { query };\n if (options?.site != null && options.site !== '') {\n payload.site = options.site;\n }\n if (date != null) {\n payload.published_after = KEENABLE_DATE_RANGES[date];\n }\n\n const response = await axios.post<t.KeenableSearchResponse>(\n resolvedUrl,\n payload,\n { headers, timeout }\n );\n\n const maxResults = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n const rawResults = Array.isArray(response.data.results)\n ? response.data.results.slice(0, maxResults)\n : [];\n\n const organic: t.OrganicResult[] = rawResults.map((result) => ({\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.description ?? result.snippet ?? '',\n date: result.published_at,\n }));\n\n return { success: true, data: { organic } };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `Keenable API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;;;;AAIA,MAAM,2BAA2B;;;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;;;;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;;;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,aACA,SACA;IAAE;IAAS;GAAQ,CACrB;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
@@ -4,6 +4,10 @@ let axios = require("axios");
4
4
  axios = require_runtime.__toESM(axios, 1);
5
5
  //#region src/tools/search/rerankers.ts
6
6
  const DEFAULT_JINA_API_URL = "https://api.jina.ai/v1/rerank";
7
+ /** Every other network call in the search pipeline is bounded (scrapers,
8
+ * search providers); rerank requests must be too, or a hung rerank API
9
+ * stalls the whole tool. */
10
+ const DEFAULT_RERANKER_TIMEOUT = 1e4;
7
11
  const getDefaultJinaApiUrl = () => process.env.JINA_API_URL != null && process.env.JINA_API_URL !== "" ? process.env.JINA_API_URL : DEFAULT_JINA_API_URL;
8
12
  var BaseReranker = class {
9
13
  apiKey;
@@ -20,10 +24,12 @@ var BaseReranker = class {
20
24
  };
21
25
  var JinaReranker = class extends BaseReranker {
22
26
  apiUrl;
23
- constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), logger }) {
27
+ timeout;
28
+ constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
24
29
  super(logger);
25
30
  this.apiKey = apiKey;
26
31
  this.apiUrl = apiUrl;
32
+ this.timeout = timeout;
27
33
  }
28
34
  async rerank(query, documents, topK = 5) {
29
35
  this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);
@@ -39,10 +45,13 @@ var JinaReranker = class extends BaseReranker {
39
45
  documents,
40
46
  return_documents: true
41
47
  };
42
- const response = await axios.default.post(this.apiUrl, requestData, { headers: {
43
- "Content-Type": "application/json",
44
- Authorization: `Bearer ${this.apiKey}`
45
- } });
48
+ const response = await axios.default.post(this.apiUrl, requestData, {
49
+ headers: {
50
+ "Content-Type": "application/json",
51
+ Authorization: `Bearer ${this.apiKey}`
52
+ },
53
+ timeout: this.timeout
54
+ });
46
55
  this.logger.debug("Jina API Model:", response.data?.model);
47
56
  this.logger.debug("Jina API Usage:", response.data?.usage);
48
57
  if (response.data && response.data.results.length) return response.data.results.map((result) => {
@@ -70,9 +79,11 @@ var JinaReranker = class extends BaseReranker {
70
79
  }
71
80
  };
72
81
  var CohereReranker = class extends BaseReranker {
73
- constructor({ apiKey = process.env.COHERE_API_KEY, logger }) {
82
+ timeout;
83
+ constructor({ apiKey = process.env.COHERE_API_KEY, timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
74
84
  super(logger);
75
85
  this.apiKey = apiKey;
86
+ this.timeout = timeout;
76
87
  }
77
88
  async rerank(query, documents, topK = 5) {
78
89
  this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);
@@ -87,10 +98,13 @@ var CohereReranker = class extends BaseReranker {
87
98
  top_n: topK,
88
99
  documents
89
100
  };
90
- const response = await axios.default.post("https://api.cohere.com/v2/rerank", requestData, { headers: {
91
- "Content-Type": "application/json",
92
- Authorization: `Bearer ${this.apiKey}`
93
- } });
101
+ const response = await axios.default.post("https://api.cohere.com/v2/rerank", requestData, {
102
+ headers: {
103
+ "Content-Type": "application/json",
104
+ Authorization: `Bearer ${this.apiKey}`
105
+ },
106
+ timeout: this.timeout
107
+ });
94
108
  this.logger.debug("Cohere API ID:", response.data?.id);
95
109
  this.logger.debug("Cohere API Meta:", response.data?.meta);
96
110
  if (response.data && response.data.results.length) return response.data.results.map((result) => {
@@ -124,16 +138,18 @@ var InfinityReranker = class extends BaseReranker {
124
138
  * Creates the appropriate reranker based on type and configuration
125
139
  */
126
140
  const createReranker = (config) => {
127
- const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;
141
+ const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, rerankerTimeout, logger } = config;
128
142
  const defaultLogger = logger || require_utils.createDefaultLogger();
129
143
  switch (rerankerType.toLowerCase()) {
130
144
  case "jina": return new JinaReranker({
131
145
  apiKey: jinaApiKey,
132
146
  apiUrl: jinaApiUrl,
147
+ timeout: rerankerTimeout,
133
148
  logger: defaultLogger
134
149
  });
135
150
  case "cohere": return new CohereReranker({
136
151
  apiKey: cohereApiKey,
152
+ timeout: rerankerTimeout,
137
153
  logger: defaultLogger
138
154
  });
139
155
  case "infinity": return new InfinityReranker(defaultLogger);
@@ -145,6 +161,7 @@ const createReranker = (config) => {
145
161
  return new JinaReranker({
146
162
  apiKey: jinaApiKey,
147
163
  apiUrl: jinaApiUrl,
164
+ timeout: rerankerTimeout,
148
165
  logger: defaultLogger
149
166
  });
150
167
  }
@@ -1 +1 @@
1
- {"version":3,"file":"rerankers.cjs","names":["createDefaultLogger","formatErrorForLog"],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\nconst getDefaultJinaApiUrl = (): string =>\n process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n ? process.env.JINA_API_URL\n : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n protected apiKey: string | undefined;\n protected logger: t.Logger;\n\n constructor(logger?: t.Logger) {\n // Each specific reranker will set its API key\n this.logger = logger || createDefaultLogger();\n }\n\n abstract rerank(\n query: string,\n documents: string[],\n topK?: number\n ): Promise<t.Highlight[]>;\n\n protected getDefaultRanking(\n documents: string[],\n topK: number\n ): t.Highlight[] {\n return documents\n .slice(0, Math.min(topK, documents.length))\n .map((doc) => ({ text: doc, score: 0 }));\n }\n}\n\nexport class JinaReranker extends BaseReranker {\n private apiUrl: string;\n\n constructor({\n apiKey = process.env.JINA_API_KEY,\n apiUrl = getDefaultJinaApiUrl(),\n logger,\n }: {\n apiKey?: string;\n apiUrl?: string;\n logger?: t.Logger;\n }) {\n super(logger);\n this.apiKey = apiKey;\n this.apiUrl = apiUrl;\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);\n\n try {\n if (this.apiKey == null || this.apiKey === '') {\n this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n return this.getDefaultRanking(documents, topK);\n }\n\n const requestData = {\n model: 'jina-reranker-v2-base-multilingual',\n query: query,\n top_n: topK,\n documents: documents,\n return_documents: true,\n };\n\n const response = await axios.post<t.JinaRerankerResponse | undefined>(\n this.apiUrl,\n requestData,\n {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n }\n );\n\n this.logger.debug('Jina API Model:', response.data?.model);\n this.logger.debug('Jina API Usage:', response.data?.usage);\n\n if (response.data && response.data.results.length) {\n return response.data.results.map((result) => {\n const docIndex = result.index;\n const score = result.relevance_score;\n let text = '';\n\n // If return_documents is true, the document field will be present\n if (result.document != null) {\n const doc = result.document;\n if (typeof doc === 'object' && 'text' in doc) {\n text = doc.text;\n } else if (typeof doc === 'string') {\n text = doc;\n }\n } else {\n // Otherwise, use the index to get the document\n text = documents[docIndex];\n }\n\n return { text, score };\n });\n } else {\n this.logger.warn(\n 'Unexpected response format from Jina API. Using default ranking.'\n );\n return this.getDefaultRanking(documents, topK);\n }\n } catch (error) {\n this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n // Fallback to default ranking on error\n return this.getDefaultRanking(documents, topK);\n }\n }\n}\n\nexport class CohereReranker extends BaseReranker {\n constructor({\n apiKey = process.env.COHERE_API_KEY,\n logger,\n }: {\n apiKey?: string;\n logger?: t.Logger;\n }) {\n super(logger);\n this.apiKey = apiKey;\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n try {\n if (this.apiKey == null || this.apiKey === '') {\n this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n return this.getDefaultRanking(documents, topK);\n }\n\n const requestData = {\n model: 'rerank-v3.5',\n query: query,\n top_n: topK,\n documents: documents,\n };\n\n const response = await axios.post<t.CohereRerankerResponse | undefined>(\n 'https://api.cohere.com/v2/rerank',\n requestData,\n {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n }\n );\n\n this.logger.debug('Cohere API ID:', response.data?.id);\n this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n if (response.data && response.data.results.length) {\n return response.data.results.map((result) => {\n const docIndex = result.index;\n const score = result.relevance_score;\n const text = documents[docIndex];\n return { text, score };\n });\n } else {\n this.logger.warn(\n 'Unexpected response format from Cohere API. Using default ranking.'\n );\n return this.getDefaultRanking(documents, topK);\n }\n } catch (error) {\n this.logger.error(\n 'Error using Cohere reranker',\n formatErrorForLog(error)\n );\n // Fallback to default ranking on error\n return this.getDefaultRanking(documents, topK);\n }\n }\n}\n\nexport class InfinityReranker extends BaseReranker {\n constructor(logger?: t.Logger) {\n super(logger);\n // No API key needed for the placeholder implementation\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(\n `Reranking ${documents.length} chunks with Infinity (placeholder)`\n );\n // This would be replaced with actual Infinity reranker implementation\n return this.getDefaultRanking(documents, topK);\n }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n rerankerType: t.RerankerType;\n jinaApiKey?: string;\n jinaApiUrl?: string;\n cohereApiKey?: string;\n logger?: t.Logger;\n}): BaseReranker | undefined => {\n const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;\n\n // Create a default logger if none is provided\n const defaultLogger = logger || createDefaultLogger();\n\n switch (rerankerType.toLowerCase()) {\n case 'jina':\n return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n case 'cohere':\n return new CohereReranker({\n apiKey: cohereApiKey,\n logger: defaultLogger,\n });\n case 'infinity':\n return new InfinityReranker(defaultLogger);\n case 'none':\n defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n return undefined;\n default:\n defaultLogger.warn(\n `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n );\n return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;;;AAIA,MAAM,uBAAuB;AAE7B,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAUA,cAAAA,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAKC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;CAChB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,mCAAmC,KAAK,QAAQ;EAEhG,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,KAAK,QACL,aACA,EACE,SAAS;IACP,gBAAgB;IAChB,eAAe,UAAU,KAAK;GAChC,EACF,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6BC,cAAAA,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAIC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;CAChB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,oCACA,aACA,EACE,SAAS;IACP,gBAAgB;IAChB,eAAe,UAAU,KAAK;GAChC,EACF,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACAA,cAAAA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBAAkB,WAMC;CAC9B,MAAM,EAAE,cAAc,YAAY,YAAY,cAAc,WAAW;CAGvE,MAAM,gBAAgB,UAAUD,cAAAA,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GAAE,QAAQ;GAAY,QAAQ;GAAY,QAAQ;EAAc,CAAC;EAC3F,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,QAAQ;EACV,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IAAE,QAAQ;IAAY,QAAQ;IAAY,QAAQ;GAAc,CAAC;CAC3F;AACF"}
1
+ {"version":3,"file":"rerankers.cjs","names":["createDefaultLogger","formatErrorForLog"],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\n/** Every other network call in the search pipeline is bounded (scrapers,\n * search providers); rerank requests must be too, or a hung rerank API\n * stalls the whole tool. */\nconst DEFAULT_RERANKER_TIMEOUT = 10000;\n\nconst getDefaultJinaApiUrl = (): string =>\n process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n ? process.env.JINA_API_URL\n : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n protected apiKey: string | undefined;\n protected logger: t.Logger;\n\n constructor(logger?: t.Logger) {\n // Each specific reranker will set its API key\n this.logger = logger || createDefaultLogger();\n }\n\n abstract rerank(\n query: string,\n documents: string[],\n topK?: number\n ): Promise<t.Highlight[]>;\n\n protected getDefaultRanking(\n documents: string[],\n topK: number\n ): t.Highlight[] {\n return documents\n .slice(0, Math.min(topK, documents.length))\n .map((doc) => ({ text: doc, score: 0 }));\n }\n}\n\nexport class JinaReranker extends BaseReranker {\n private apiUrl: string;\n private timeout: number;\n\n constructor({\n apiKey = process.env.JINA_API_KEY,\n apiUrl = getDefaultJinaApiUrl(),\n timeout = DEFAULT_RERANKER_TIMEOUT,\n logger,\n }: {\n apiKey?: string;\n apiUrl?: string;\n timeout?: number;\n logger?: t.Logger;\n }) {\n super(logger);\n this.apiKey = apiKey;\n this.apiUrl = apiUrl;\n this.timeout = timeout;\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(\n `Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`\n );\n\n try {\n if (this.apiKey == null || this.apiKey === '') {\n this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n return this.getDefaultRanking(documents, topK);\n }\n\n const requestData = {\n model: 'jina-reranker-v2-base-multilingual',\n query: query,\n top_n: topK,\n documents: documents,\n return_documents: true,\n };\n\n const response = await axios.post<t.JinaRerankerResponse | undefined>(\n this.apiUrl,\n requestData,\n {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n timeout: this.timeout,\n }\n );\n\n this.logger.debug('Jina API Model:', response.data?.model);\n this.logger.debug('Jina API Usage:', response.data?.usage);\n\n if (response.data && response.data.results.length) {\n return response.data.results.map((result) => {\n const docIndex = result.index;\n const score = result.relevance_score;\n let text = '';\n\n // If return_documents is true, the document field will be present\n if (result.document != null) {\n const doc = result.document;\n if (typeof doc === 'object' && 'text' in doc) {\n text = doc.text;\n } else if (typeof doc === 'string') {\n text = doc;\n }\n } else {\n // Otherwise, use the index to get the document\n text = documents[docIndex];\n }\n\n return { text, score };\n });\n } else {\n this.logger.warn(\n 'Unexpected response format from Jina API. Using default ranking.'\n );\n return this.getDefaultRanking(documents, topK);\n }\n } catch (error) {\n this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n // Fallback to default ranking on error\n return this.getDefaultRanking(documents, topK);\n }\n }\n}\n\nexport class CohereReranker extends BaseReranker {\n private timeout: number;\n\n constructor({\n apiKey = process.env.COHERE_API_KEY,\n timeout = DEFAULT_RERANKER_TIMEOUT,\n logger,\n }: {\n apiKey?: string;\n timeout?: number;\n logger?: t.Logger;\n }) {\n super(logger);\n this.apiKey = apiKey;\n this.timeout = timeout;\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n try {\n if (this.apiKey == null || this.apiKey === '') {\n this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n return this.getDefaultRanking(documents, topK);\n }\n\n const requestData = {\n model: 'rerank-v3.5',\n query: query,\n top_n: topK,\n documents: documents,\n };\n\n const response = await axios.post<t.CohereRerankerResponse | undefined>(\n 'https://api.cohere.com/v2/rerank',\n requestData,\n {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n timeout: this.timeout,\n }\n );\n\n this.logger.debug('Cohere API ID:', response.data?.id);\n this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n if (response.data && response.data.results.length) {\n return response.data.results.map((result) => {\n const docIndex = result.index;\n const score = result.relevance_score;\n const text = documents[docIndex];\n return { text, score };\n });\n } else {\n this.logger.warn(\n 'Unexpected response format from Cohere API. Using default ranking.'\n );\n return this.getDefaultRanking(documents, topK);\n }\n } catch (error) {\n this.logger.error(\n 'Error using Cohere reranker',\n formatErrorForLog(error)\n );\n // Fallback to default ranking on error\n return this.getDefaultRanking(documents, topK);\n }\n }\n}\n\nexport class InfinityReranker extends BaseReranker {\n constructor(logger?: t.Logger) {\n super(logger);\n // No API key needed for the placeholder implementation\n }\n\n async rerank(\n query: string,\n documents: string[],\n topK: number = 5\n ): Promise<t.Highlight[]> {\n this.logger.debug(\n `Reranking ${documents.length} chunks with Infinity (placeholder)`\n );\n // This would be replaced with actual Infinity reranker implementation\n return this.getDefaultRanking(documents, topK);\n }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n rerankerType: t.RerankerType;\n jinaApiKey?: string;\n jinaApiUrl?: string;\n cohereApiKey?: string;\n rerankerTimeout?: number;\n logger?: t.Logger;\n}): BaseReranker | undefined => {\n const {\n rerankerType,\n jinaApiKey,\n jinaApiUrl,\n cohereApiKey,\n rerankerTimeout,\n logger,\n } = config;\n\n // Create a default logger if none is provided\n const defaultLogger = logger || createDefaultLogger();\n\n switch (rerankerType.toLowerCase()) {\n case 'jina':\n return new JinaReranker({\n apiKey: jinaApiKey,\n apiUrl: jinaApiUrl,\n timeout: rerankerTimeout,\n logger: defaultLogger,\n });\n case 'cohere':\n return new CohereReranker({\n apiKey: cohereApiKey,\n timeout: rerankerTimeout,\n logger: defaultLogger,\n });\n case 'infinity':\n return new InfinityReranker(defaultLogger);\n case 'none':\n defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n return undefined;\n default:\n defaultLogger.warn(\n `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n );\n return new JinaReranker({\n apiKey: jinaApiKey,\n apiUrl: jinaApiUrl,\n timeout: rerankerTimeout,\n logger: defaultLogger,\n });\n }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;;;AAIA,MAAM,uBAAuB;;;;AAK7B,MAAM,2BAA2B;AAEjC,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAUA,cAAAA,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CACA;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAAU,0BACV,UAMC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,mCAAmC,KAAK,QACxE;EAEA,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,KAAK,QACL,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6BC,cAAAA,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAAU,0BACV,UAKC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,oCACA,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACAA,cAAAA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBAAkB,WAOC;CAC9B,MAAM,EACJ,cACA,YACA,YACA,cACA,iBACA,WACE;CAGJ,MAAM,gBAAgB,UAAUD,cAAAA,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GACtB,QAAQ;GACR,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IACtB,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;GACV,CAAC;CACH;AACF"}
@@ -1,5 +1,6 @@
1
1
  const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
2
2
  const require_utils = require("./utils.cjs");
3
+ const require_keenable_search = require("./keenable-search.cjs");
3
4
  const require_tavily_search = require("./tavily-search.cjs");
4
5
  let axios = require("axios");
5
6
  axios = require_runtime.__toESM(axios, 1);
@@ -29,6 +30,27 @@ const chunker = {
29
30
  }
30
31
  };
31
32
  const DEFAULT_MAX_CONTENT_LENGTH = 5e4;
33
+ const DEFAULT_CHUNK_SIZE = 150;
34
+ const DEFAULT_CHUNK_OVERLAP = 50;
35
+ /** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /
36
+ * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The
37
+ * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`
38
+ * throws when overlap >= size. */
39
+ function resolveChunkOptions(chunkSize, chunkOverlap) {
40
+ const resolve = (configValue, envVar, fallback) => {
41
+ if (configValue != null && configValue > 0) return configValue;
42
+ const envValue = Number(process.env[envVar]);
43
+ if (Number.isFinite(envValue) && envValue > 0) return envValue;
44
+ return fallback;
45
+ };
46
+ const size = resolve(chunkSize, "SEARCH_CHUNK_SIZE", DEFAULT_CHUNK_SIZE);
47
+ let overlap = resolve(chunkOverlap, "SEARCH_CHUNK_OVERLAP", DEFAULT_CHUNK_OVERLAP);
48
+ if (overlap >= size) overlap = Math.floor(size / 3);
49
+ return {
50
+ chunkSize: size,
51
+ chunkOverlap: overlap
52
+ };
53
+ }
32
54
  /** Resolves the per-source scraped content cap from config, the
33
55
  * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */
34
56
  function resolveMaxContentLength(maxContentLength) {
@@ -49,7 +71,7 @@ function createSourceUpdateCallback(sourceMap) {
49
71
  });
50
72
  };
51
73
  }
52
- const getHighlights = async ({ query, content, reranker, topResults = 5, maxContentLength = DEFAULT_MAX_CONTENT_LENGTH, logger }) => {
74
+ const getHighlights = async ({ query, content, reranker, topResults = 5, maxContentLength = DEFAULT_MAX_CONTENT_LENGTH, chunkOptions, logger }) => {
53
75
  const logger_ = logger || require_utils.createDefaultLogger();
54
76
  if (!content) {
55
77
  logger_.warn("No content provided for highlights");
@@ -60,7 +82,7 @@ const getHighlights = async ({ query, content, reranker, topResults = 5, maxCont
60
82
  return;
61
83
  }
62
84
  try {
63
- const documents = await chunker.splitText(truncateContent(content, maxContentLength));
85
+ const documents = await chunker.splitText(truncateContent(content, maxContentLength), chunkOptions);
64
86
  if (Array.isArray(documents)) return await reranker.rerank(query, documents, topResults);
65
87
  else {
66
88
  logger_.error("Expected documents to be an array, got:", typeof documents);
@@ -252,16 +274,18 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
252
274
  return { getSources };
253
275
  };
254
276
  const createSearchAPI = (config) => {
255
- const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions } = config;
277
+ const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions } = config;
256
278
  if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey);
257
279
  else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
258
280
  else if (searchProvider.toLowerCase() === "tavily") return require_tavily_search.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
259
- else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
281
+ else if (searchProvider.toLowerCase() === "keenable") return require_keenable_search.createKeenableAPI(keenableApiKey, keenableApiUrl, keenableSearchOptions);
282
+ else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', or 'keenable'`);
260
283
  };
261
284
  const createSourceProcessor = (config = {}, scraperInstance) => {
262
285
  if (!scraperInstance) throw new Error("Scraper instance is required");
263
286
  const { topResults = 5, reranker, logger } = config;
264
287
  const maxContentLength = resolveMaxContentLength(config.maxContentLength);
288
+ const chunkOptions = resolveChunkOptions(config.chunkSize, config.chunkOverlap);
265
289
  const logger_ = logger || require_utils.createDefaultLogger();
266
290
  const scraper = scraperInstance;
267
291
  const processResponse = (url, response) => {
@@ -290,8 +314,10 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
290
314
  const highlights = await getHighlights({
291
315
  query,
292
316
  reranker,
317
+ topResults,
293
318
  content: result.content,
294
319
  maxContentLength,
320
+ chunkOptions,
295
321
  logger: logger_
296
322
  });
297
323
  if (onGetHighlights) onGetHighlights(result.url);
@@ -1 +1 @@
1
- {"version":3,"file":"search.cjs","names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","getAttribution"],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n if (maxContentLength != null && maxContentLength > 0) {\n return maxContentLength;\n }\n const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n if (Number.isFinite(envValue) && envValue > 0) {\n return envValue;\n }\n return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n maxContentLength?: number;\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(\n truncateContent(content, maxContentLength)\n );\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n const payload: t.SerperSearchPayload = {\n q: query,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n tavilyApiKey,\n tavilySearchUrl,\n tavilySearchOptions,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else if (searchProvider.toLowerCase() === 'tavily') {\n return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const processResponse = (\n url: string,\n response: t.AnyScraperResponse\n ): t.ScrapeResult => {\n const rawMetadata = scraper.extractMetadata(response);\n const metadata =\n Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n const attribution = getAttribution(url, metadata, logger_);\n\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: truncateContent(chunker.cleanText(content), maxContentLength),\n };\n }\n\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n return { url, attribution, error: true, content: '' };\n };\n\n const addHighlights = async (\n result: t.ScrapeResult,\n query: string,\n onGetHighlights: t.SearchToolConfig['onGetHighlights']\n ): Promise<t.ScrapeResult> => {\n if (result.error != null) {\n return result;\n }\n try {\n const highlights = await getHighlights({\n query,\n reranker,\n content: result.content,\n maxContentLength,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return { ...result, highlights };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return result;\n }\n };\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n try {\n let responses: Array<[string, t.AnyScraperResponse]>;\n\n if (scraper.scrapeUrls) {\n responses = await scraper.scrapeUrls(links);\n } else {\n responses = await Promise.all(\n links.map((link) =>\n scraper\n .scrapeUrl(link, {})\n .catch((error): [string, t.AnyScraperResponse] => {\n logger_.error(`Error scraping ${link}:`, error);\n return [link, { success: false, error: String(error) }];\n })\n )\n );\n }\n\n const withHighlights = await Promise.all(\n responses.map(([url, response]) =>\n addHighlights(\n processResponse(url, response),\n query,\n onGetHighlights\n )\n )\n );\n return withHighlights;\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n }\n\n if (\n result.data.topStories != null &&\n result.data.topStories.length > numElements\n ) {\n /** Merged news results can far exceed the requested source count;\n * every entry is formatted into the LLM output, so cap them up\n * front — before any early return below and before scraping\n * entries the cap would discard */\n result.data.topStories = result.data.topStories.slice(0, numElements);\n }\n\n if (!result.data.organic) {\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"mappings":";;;;;;;AAOA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQA,yBAAAA,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAUC,cAAAA,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;;;AAInC,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,aAQwC;CACxC,MAAM,UAAU,UAAUA,cAAAA,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,CAC3C;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAA,QAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAA,QAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,wBACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAOC,sBAAAA,gBAAgB,cAAc,iBAAiB,mBAAmB;MAEzE,MAAM,IAAI,MACR,4BAA4B,eAAe,2CAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,UAAU,UAAUD,cAAAA,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAcE,cAAAA,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA,SAAS,OAAO;IAChB;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}
1
+ {"version":3,"file":"search.cjs","names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","createKeenableAPI","getAttribution"],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n chunkSize?: number,\n chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n const resolve = (\n configValue: number | undefined,\n envVar: string,\n fallback: number\n ): number => {\n if (configValue != null && configValue > 0) {\n return configValue;\n }\n const envValue = Number(process.env[envVar]);\n if (Number.isFinite(envValue) && envValue > 0) {\n return envValue;\n }\n return fallback;\n };\n\n const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n let overlap = resolve(\n chunkOverlap,\n 'SEARCH_CHUNK_OVERLAP',\n DEFAULT_CHUNK_OVERLAP\n );\n if (overlap >= size) {\n overlap = Math.floor(size / 3);\n }\n return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n if (maxContentLength != null && maxContentLength > 0) {\n return maxContentLength;\n }\n const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n if (Number.isFinite(envValue) && envValue > 0) {\n return envValue;\n }\n return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n chunkOptions,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n maxContentLength?: number;\n chunkOptions?: { chunkSize: number; chunkOverlap: number };\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(\n truncateContent(content, maxContentLength),\n chunkOptions\n );\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n const payload: t.SerperSearchPayload = {\n q: query,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n tavilyApiKey,\n tavilySearchUrl,\n tavilySearchOptions,\n keenableApiKey,\n keenableApiUrl,\n keenableSearchOptions,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else if (searchProvider.toLowerCase() === 'tavily') {\n return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n } else if (searchProvider.toLowerCase() === 'keenable') {\n return createKeenableAPI(\n keenableApiKey,\n keenableApiUrl,\n keenableSearchOptions\n );\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', or 'keenable'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n const chunkOptions = resolveChunkOptions(\n config.chunkSize,\n config.chunkOverlap\n );\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const processResponse = (\n url: string,\n response: t.AnyScraperResponse\n ): t.ScrapeResult => {\n const rawMetadata = scraper.extractMetadata(response);\n const metadata =\n Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n const attribution = getAttribution(url, metadata, logger_);\n\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: truncateContent(chunker.cleanText(content), maxContentLength),\n };\n }\n\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n return { url, attribution, error: true, content: '' };\n };\n\n const addHighlights = async (\n result: t.ScrapeResult,\n query: string,\n onGetHighlights: t.SearchToolConfig['onGetHighlights']\n ): Promise<t.ScrapeResult> => {\n if (result.error != null) {\n return result;\n }\n try {\n const highlights = await getHighlights({\n query,\n reranker,\n topResults,\n content: result.content,\n maxContentLength,\n chunkOptions,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return { ...result, highlights };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return result;\n }\n };\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n try {\n let responses: Array<[string, t.AnyScraperResponse]>;\n\n if (scraper.scrapeUrls) {\n responses = await scraper.scrapeUrls(links);\n } else {\n responses = await Promise.all(\n links.map((link) =>\n scraper\n .scrapeUrl(link, {})\n .catch((error): [string, t.AnyScraperResponse] => {\n logger_.error(`Error scraping ${link}:`, error);\n return [link, { success: false, error: String(error) }];\n })\n )\n );\n }\n\n const withHighlights = await Promise.all(\n responses.map(([url, response]) =>\n addHighlights(\n processResponse(url, response),\n query,\n onGetHighlights\n )\n )\n );\n return withHighlights;\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n }\n\n if (\n result.data.topStories != null &&\n result.data.topStories.length > numElements\n ) {\n /** Merged news results can far exceed the requested source count;\n * every entry is formatted into the LLM output, so cap them up\n * front — before any early return below and before scraping\n * entries the cap would discard */\n result.data.topStories = result.data.topStories.slice(0, numElements);\n }\n\n if (!result.data.organic) {\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"mappings":";;;;;;;;AAQA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQA,yBAAAA,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAUC,cAAAA,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAUA,cAAAA,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAA,QAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAA,QAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,0BACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAOC,sBAAAA,gBAAgB,cAAc,iBAAiB,mBAAmB;MACpE,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAOC,wBAAAA,kBACL,gBACA,gBACA,qBACF;MAEA,MAAM,IAAI,MACR,4BAA4B,eAAe,uDAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAUF,cAAAA,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAcG,cAAAA,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}