@staticn0va/wigolo 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1003) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +146 -227
  3. package/SKILL.md +382 -0
  4. package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
  5. package/assets/blocks/claude-code/wigolo-command.md +40 -0
  6. package/assets/blocks/cursor/wigolo.mdc +46 -0
  7. package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
  8. package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
  9. package/assets/skills/wigolo/SKILL.md +50 -0
  10. package/assets/skills/wigolo/rules/cache-first.md +30 -0
  11. package/assets/skills/wigolo/rules/synthesis.md +43 -0
  12. package/assets/skills/wigolo-agent/SKILL.md +73 -0
  13. package/assets/skills/wigolo-crawl/SKILL.md +60 -0
  14. package/assets/skills/wigolo-extract/SKILL.md +59 -0
  15. package/assets/skills/wigolo-fetch/SKILL.md +65 -0
  16. package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
  17. package/assets/skills/wigolo-research/SKILL.md +77 -0
  18. package/assets/skills/wigolo-search/SKILL.md +78 -0
  19. package/dist/agent/executor.d.ts +33 -0
  20. package/dist/agent/executor.d.ts.map +1 -0
  21. package/dist/agent/executor.js +233 -0
  22. package/dist/agent/executor.js.map +1 -0
  23. package/dist/agent/pipeline.d.ts +5 -0
  24. package/dist/agent/pipeline.d.ts.map +1 -0
  25. package/dist/agent/pipeline.js +238 -0
  26. package/dist/agent/pipeline.js.map +1 -0
  27. package/dist/agent/planner.d.ts +13 -0
  28. package/dist/agent/planner.d.ts.map +1 -0
  29. package/dist/agent/planner.js +271 -0
  30. package/dist/agent/planner.js.map +1 -0
  31. package/dist/agent/relevance.d.ts +15 -0
  32. package/dist/agent/relevance.d.ts.map +1 -0
  33. package/dist/agent/relevance.js +60 -0
  34. package/dist/agent/relevance.js.map +1 -0
  35. package/dist/cache/backfill-embeddings.d.ts +23 -0
  36. package/dist/cache/backfill-embeddings.d.ts.map +1 -0
  37. package/dist/cache/backfill-embeddings.js +105 -0
  38. package/dist/cache/backfill-embeddings.js.map +1 -0
  39. package/dist/cache/change-detector.d.ts +7 -0
  40. package/dist/cache/change-detector.d.ts.map +1 -0
  41. package/dist/cache/change-detector.js +43 -0
  42. package/dist/cache/change-detector.js.map +1 -0
  43. package/dist/cache/db.d.ts +1 -0
  44. package/dist/cache/db.d.ts.map +1 -1
  45. package/dist/cache/db.js +94 -22
  46. package/dist/cache/db.js.map +1 -1
  47. package/dist/cache/diff-summary.d.ts +2 -0
  48. package/dist/cache/diff-summary.d.ts.map +1 -0
  49. package/dist/cache/diff-summary.js +82 -0
  50. package/dist/cache/diff-summary.js.map +1 -0
  51. package/dist/cache/migrations/runner.d.ts +29 -0
  52. package/dist/cache/migrations/runner.d.ts.map +1 -0
  53. package/dist/cache/migrations/runner.js +147 -0
  54. package/dist/cache/migrations/runner.js.map +1 -0
  55. package/dist/cache/sqlite-vec-store.d.ts +42 -0
  56. package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
  57. package/dist/cache/sqlite-vec-store.js +176 -0
  58. package/dist/cache/sqlite-vec-store.js.map +1 -0
  59. package/dist/cache/store.d.ts +47 -1
  60. package/dist/cache/store.d.ts.map +1 -1
  61. package/dist/cache/store.js +364 -168
  62. package/dist/cache/store.js.map +1 -1
  63. package/dist/cli/agents/antigravity.d.ts +20 -0
  64. package/dist/cli/agents/antigravity.d.ts.map +1 -0
  65. package/dist/cli/agents/antigravity.js +49 -0
  66. package/dist/cli/agents/antigravity.js.map +1 -0
  67. package/dist/cli/agents/claude-code.d.ts +25 -0
  68. package/dist/cli/agents/claude-code.d.ts.map +1 -0
  69. package/dist/cli/agents/claude-code.js +111 -0
  70. package/dist/cli/agents/claude-code.js.map +1 -0
  71. package/dist/cli/agents/cursor.d.ts +21 -0
  72. package/dist/cli/agents/cursor.d.ts.map +1 -0
  73. package/dist/cli/agents/cursor.js +58 -0
  74. package/dist/cli/agents/cursor.js.map +1 -0
  75. package/dist/cli/agents/gemini-cli.d.ts +21 -0
  76. package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
  77. package/dist/cli/agents/gemini-cli.js +55 -0
  78. package/dist/cli/agents/gemini-cli.js.map +1 -0
  79. package/dist/cli/agents/registry.d.ts +21 -0
  80. package/dist/cli/agents/registry.d.ts.map +1 -0
  81. package/dist/cli/agents/registry.js +27 -0
  82. package/dist/cli/agents/registry.js.map +1 -0
  83. package/dist/cli/agents/utils.d.ts +26 -0
  84. package/dist/cli/agents/utils.d.ts.map +1 -0
  85. package/dist/cli/agents/utils.js +136 -0
  86. package/dist/cli/agents/utils.js.map +1 -0
  87. package/dist/cli/agents/vscode.d.ts +21 -0
  88. package/dist/cli/agents/vscode.d.ts.map +1 -0
  89. package/dist/cli/agents/vscode.js +62 -0
  90. package/dist/cli/agents/vscode.js.map +1 -0
  91. package/dist/cli/auth.d.ts +2 -0
  92. package/dist/cli/auth.d.ts.map +1 -0
  93. package/dist/cli/auth.js +94 -0
  94. package/dist/cli/auth.js.map +1 -0
  95. package/dist/cli/backfill.d.ts +2 -0
  96. package/dist/cli/backfill.d.ts.map +1 -0
  97. package/dist/cli/backfill.js +58 -0
  98. package/dist/cli/backfill.js.map +1 -0
  99. package/dist/cli/daemon.d.ts +6 -1
  100. package/dist/cli/daemon.d.ts.map +1 -1
  101. package/dist/cli/daemon.js +61 -3
  102. package/dist/cli/daemon.js.map +1 -1
  103. package/dist/cli/doctor.d.ts +8 -0
  104. package/dist/cli/doctor.d.ts.map +1 -0
  105. package/dist/cli/doctor.js +344 -0
  106. package/dist/cli/doctor.js.map +1 -0
  107. package/dist/cli/health.d.ts +1 -1
  108. package/dist/cli/health.d.ts.map +1 -1
  109. package/dist/cli/health.js +42 -3
  110. package/dist/cli/health.js.map +1 -1
  111. package/dist/cli/help.d.ts +6 -0
  112. package/dist/cli/help.d.ts.map +1 -0
  113. package/dist/cli/help.js +63 -0
  114. package/dist/cli/help.js.map +1 -0
  115. package/dist/cli/index.d.ts +1 -1
  116. package/dist/cli/index.d.ts.map +1 -1
  117. package/dist/cli/index.js +35 -7
  118. package/dist/cli/index.js.map +1 -1
  119. package/dist/cli/init.d.ts +2 -0
  120. package/dist/cli/init.d.ts.map +1 -0
  121. package/dist/cli/init.js +201 -0
  122. package/dist/cli/init.js.map +1 -0
  123. package/dist/cli/plugin.d.ts +5 -0
  124. package/dist/cli/plugin.d.ts.map +1 -0
  125. package/dist/cli/plugin.js +185 -0
  126. package/dist/cli/plugin.js.map +1 -0
  127. package/dist/cli/setup-mcp.d.ts +2 -0
  128. package/dist/cli/setup-mcp.d.ts.map +1 -0
  129. package/dist/cli/setup-mcp.js +114 -0
  130. package/dist/cli/setup-mcp.js.map +1 -0
  131. package/dist/cli/shell.d.ts +2 -0
  132. package/dist/cli/shell.d.ts.map +1 -0
  133. package/dist/cli/shell.js +86 -0
  134. package/dist/cli/shell.js.map +1 -0
  135. package/dist/cli/shutdown.d.ts +2 -0
  136. package/dist/cli/shutdown.d.ts.map +1 -0
  137. package/dist/cli/shutdown.js +26 -0
  138. package/dist/cli/shutdown.js.map +1 -0
  139. package/dist/cli/status.d.ts +2 -0
  140. package/dist/cli/status.d.ts.map +1 -0
  141. package/dist/cli/status.js +31 -0
  142. package/dist/cli/status.js.map +1 -0
  143. package/dist/cli/telemetry.d.ts +10 -0
  144. package/dist/cli/telemetry.d.ts.map +1 -0
  145. package/dist/cli/telemetry.js +56 -0
  146. package/dist/cli/telemetry.js.map +1 -0
  147. package/dist/cli/tui/agents-types.d.ts +28 -0
  148. package/dist/cli/tui/agents-types.d.ts.map +1 -0
  149. package/dist/cli/tui/agents-types.js +1 -0
  150. package/dist/cli/tui/agents-types.js.map +1 -0
  151. package/dist/cli/tui/agents.d.ts +11 -0
  152. package/dist/cli/tui/agents.d.ts.map +1 -0
  153. package/dist/cli/tui/agents.js +93 -0
  154. package/dist/cli/tui/agents.js.map +1 -0
  155. package/dist/cli/tui/banner.d.ts +3 -0
  156. package/dist/cli/tui/banner.d.ts.map +1 -0
  157. package/dist/cli/tui/banner.js +30 -0
  158. package/dist/cli/tui/banner.js.map +1 -0
  159. package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
  160. package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
  161. package/dist/cli/tui/components/AgentSelect.js +116 -0
  162. package/dist/cli/tui/components/AgentSelect.js.map +1 -0
  163. package/dist/cli/tui/components/Banner.d.ts +6 -0
  164. package/dist/cli/tui/components/Banner.d.ts.map +1 -0
  165. package/dist/cli/tui/components/Banner.js +25 -0
  166. package/dist/cli/tui/components/Banner.js.map +1 -0
  167. package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
  168. package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
  169. package/dist/cli/tui/components/BrowserSelect.js +19 -0
  170. package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
  171. package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
  172. package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
  173. package/dist/cli/tui/components/InstallProgress.js +67 -0
  174. package/dist/cli/tui/components/InstallProgress.js.map +1 -0
  175. package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
  176. package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
  177. package/dist/cli/tui/components/SkillInstall.js +94 -0
  178. package/dist/cli/tui/components/SkillInstall.js.map +1 -0
  179. package/dist/cli/tui/components/Summary.d.ts +22 -0
  180. package/dist/cli/tui/components/Summary.d.ts.map +1 -0
  181. package/dist/cli/tui/components/Summary.js +135 -0
  182. package/dist/cli/tui/components/Summary.js.map +1 -0
  183. package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
  184. package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
  185. package/dist/cli/tui/components/SystemCheck.js +71 -0
  186. package/dist/cli/tui/components/SystemCheck.js.map +1 -0
  187. package/dist/cli/tui/components/Verification.d.ts +8 -0
  188. package/dist/cli/tui/components/Verification.d.ts.map +1 -0
  189. package/dist/cli/tui/components/Verification.js +63 -0
  190. package/dist/cli/tui/components/Verification.js.map +1 -0
  191. package/dist/cli/tui/config-writer-cli.d.ts +12 -0
  192. package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
  193. package/dist/cli/tui/config-writer-cli.js +39 -0
  194. package/dist/cli/tui/config-writer-cli.js.map +1 -0
  195. package/dist/cli/tui/config-writer-json.d.ts +16 -0
  196. package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
  197. package/dist/cli/tui/config-writer-json.js +86 -0
  198. package/dist/cli/tui/config-writer-json.js.map +1 -0
  199. package/dist/cli/tui/config-writer-toml.d.ts +16 -0
  200. package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
  201. package/dist/cli/tui/config-writer-toml.js +83 -0
  202. package/dist/cli/tui/config-writer-toml.js.map +1 -0
  203. package/dist/cli/tui/config-writer.d.ts +25 -0
  204. package/dist/cli/tui/config-writer.d.ts.map +1 -0
  205. package/dist/cli/tui/config-writer.js +101 -0
  206. package/dist/cli/tui/config-writer.js.map +1 -0
  207. package/dist/cli/tui/detect-helpers.d.ts +6 -0
  208. package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
  209. package/dist/cli/tui/detect-helpers.js +45 -0
  210. package/dist/cli/tui/detect-helpers.js.map +1 -0
  211. package/dist/cli/tui/extras-prompt.d.ts +7 -0
  212. package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
  213. package/dist/cli/tui/extras-prompt.js +42 -0
  214. package/dist/cli/tui/extras-prompt.js.map +1 -0
  215. package/dist/cli/tui/flags-types.d.ts +19 -0
  216. package/dist/cli/tui/flags-types.d.ts.map +1 -0
  217. package/dist/cli/tui/flags-types.js +23 -0
  218. package/dist/cli/tui/flags-types.js.map +1 -0
  219. package/dist/cli/tui/flags.d.ts +5 -0
  220. package/dist/cli/tui/flags.d.ts.map +1 -0
  221. package/dist/cli/tui/flags.js +132 -0
  222. package/dist/cli/tui/flags.js.map +1 -0
  223. package/dist/cli/tui/format.d.ts +14 -0
  224. package/dist/cli/tui/format.d.ts.map +1 -0
  225. package/dist/cli/tui/format.js +37 -0
  226. package/dist/cli/tui/format.js.map +1 -0
  227. package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
  228. package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
  229. package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
  230. package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
  231. package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
  232. package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
  233. package/dist/cli/tui/hooks/useInstall.js +90 -0
  234. package/dist/cli/tui/hooks/useInstall.js.map +1 -0
  235. package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
  236. package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
  237. package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
  238. package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
  239. package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
  240. package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
  241. package/dist/cli/tui/hooks/useVerify.js +71 -0
  242. package/dist/cli/tui/hooks/useVerify.js.map +1 -0
  243. package/dist/cli/tui/ink-init.d.ts +2 -0
  244. package/dist/cli/tui/ink-init.d.ts.map +1 -0
  245. package/dist/cli/tui/ink-init.js +198 -0
  246. package/dist/cli/tui/ink-init.js.map +1 -0
  247. package/dist/cli/tui/reporter-auto.d.ts +7 -0
  248. package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
  249. package/dist/cli/tui/reporter-auto.js +15 -0
  250. package/dist/cli/tui/reporter-auto.js.map +1 -0
  251. package/dist/cli/tui/reporter.d.ts +26 -0
  252. package/dist/cli/tui/reporter.d.ts.map +1 -0
  253. package/dist/cli/tui/reporter.js +32 -0
  254. package/dist/cli/tui/reporter.js.map +1 -0
  255. package/dist/cli/tui/run-command.d.ts +14 -0
  256. package/dist/cli/tui/run-command.d.ts.map +1 -0
  257. package/dist/cli/tui/run-command.js +72 -0
  258. package/dist/cli/tui/run-command.js.map +1 -0
  259. package/dist/cli/tui/select-agents.d.ts +6 -0
  260. package/dist/cli/tui/select-agents.d.ts.map +1 -0
  261. package/dist/cli/tui/select-agents.js +32 -0
  262. package/dist/cli/tui/select-agents.js.map +1 -0
  263. package/dist/cli/tui/status-agents.d.ts +11 -0
  264. package/dist/cli/tui/status-agents.d.ts.map +1 -0
  265. package/dist/cli/tui/status-agents.js +53 -0
  266. package/dist/cli/tui/status-agents.js.map +1 -0
  267. package/dist/cli/tui/status-cache.d.ts +6 -0
  268. package/dist/cli/tui/status-cache.d.ts.map +1 -0
  269. package/dist/cli/tui/status-cache.js +39 -0
  270. package/dist/cli/tui/status-cache.js.map +1 -0
  271. package/dist/cli/tui/status-format.d.ts +14 -0
  272. package/dist/cli/tui/status-format.d.ts.map +1 -0
  273. package/dist/cli/tui/status-format.js +41 -0
  274. package/dist/cli/tui/status-format.js.map +1 -0
  275. package/dist/cli/tui/status-python.d.ts +6 -0
  276. package/dist/cli/tui/status-python.d.ts.map +1 -0
  277. package/dist/cli/tui/status-python.js +30 -0
  278. package/dist/cli/tui/status-python.js.map +1 -0
  279. package/dist/cli/tui/system-check.d.ts +24 -0
  280. package/dist/cli/tui/system-check.d.ts.map +1 -0
  281. package/dist/cli/tui/system-check.js +103 -0
  282. package/dist/cli/tui/system-check.js.map +1 -0
  283. package/dist/cli/tui/tui-reporter.d.ts +19 -0
  284. package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
  285. package/dist/cli/tui/tui-reporter.js +95 -0
  286. package/dist/cli/tui/tui-reporter.js.map +1 -0
  287. package/dist/cli/tui/utils/config-writer.d.ts +3 -0
  288. package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
  289. package/dist/cli/tui/utils/config-writer.js +22 -0
  290. package/dist/cli/tui/utils/config-writer.js.map +1 -0
  291. package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
  292. package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
  293. package/dist/cli/tui/utils/suppress-logs.js +11 -0
  294. package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
  295. package/dist/cli/tui/verify-suggestions.d.ts +5 -0
  296. package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
  297. package/dist/cli/tui/verify-suggestions.js +20 -0
  298. package/dist/cli/tui/verify-suggestions.js.map +1 -0
  299. package/dist/cli/tui/verify.d.ts +14 -0
  300. package/dist/cli/tui/verify.d.ts.map +1 -0
  301. package/dist/cli/tui/verify.js +101 -0
  302. package/dist/cli/tui/verify.js.map +1 -0
  303. package/dist/cli/tui/version.d.ts +2 -0
  304. package/dist/cli/tui/version.d.ts.map +1 -0
  305. package/dist/cli/tui/version.js +14 -0
  306. package/dist/cli/tui/version.js.map +1 -0
  307. package/dist/cli/uninstall.d.ts +2 -0
  308. package/dist/cli/uninstall.d.ts.map +1 -0
  309. package/dist/cli/uninstall.js +57 -0
  310. package/dist/cli/uninstall.js.map +1 -0
  311. package/dist/cli/warmup.d.ts +10 -2
  312. package/dist/cli/warmup.d.ts.map +1 -1
  313. package/dist/cli/warmup.js +226 -93
  314. package/dist/cli/warmup.js.map +1 -1
  315. package/dist/config.d.ts +28 -2
  316. package/dist/config.d.ts.map +1 -1
  317. package/dist/config.js +106 -56
  318. package/dist/config.js.map +1 -1
  319. package/dist/crawl/crawler.d.ts +6 -0
  320. package/dist/crawl/crawler.d.ts.map +1 -1
  321. package/dist/crawl/crawler.js +210 -209
  322. package/dist/crawl/crawler.js.map +1 -1
  323. package/dist/crawl/dedup.d.ts +1 -0
  324. package/dist/crawl/dedup.d.ts.map +1 -1
  325. package/dist/crawl/dedup.js +124 -81
  326. package/dist/crawl/dedup.js.map +1 -1
  327. package/dist/crawl/etag-incremental.d.ts +43 -0
  328. package/dist/crawl/etag-incremental.d.ts.map +1 -0
  329. package/dist/crawl/etag-incremental.js +94 -0
  330. package/dist/crawl/etag-incremental.js.map +1 -0
  331. package/dist/crawl/index-to-vec.d.ts +10 -0
  332. package/dist/crawl/index-to-vec.d.ts.map +1 -0
  333. package/dist/crawl/index-to-vec.js +44 -0
  334. package/dist/crawl/index-to-vec.js.map +1 -0
  335. package/dist/crawl/mapper.js +136 -164
  336. package/dist/crawl/mapper.js.map +1 -1
  337. package/dist/crawl/rate-limiter.js +63 -66
  338. package/dist/crawl/rate-limiter.js.map +1 -1
  339. package/dist/crawl/robots.js +58 -57
  340. package/dist/crawl/robots.js.map +1 -1
  341. package/dist/crawl/sitemap-first.d.ts +12 -0
  342. package/dist/crawl/sitemap-first.d.ts.map +1 -0
  343. package/dist/crawl/sitemap-first.js +47 -0
  344. package/dist/crawl/sitemap-first.js.map +1 -0
  345. package/dist/crawl/sitemap.js +33 -32
  346. package/dist/crawl/sitemap.js.map +1 -1
  347. package/dist/crawl/url-utils.d.ts +1 -0
  348. package/dist/crawl/url-utils.d.ts.map +1 -1
  349. package/dist/crawl/url-utils.js +49 -37
  350. package/dist/crawl/url-utils.js.map +1 -1
  351. package/dist/daemon/health-check.d.ts +16 -0
  352. package/dist/daemon/health-check.d.ts.map +1 -0
  353. package/dist/daemon/health-check.js +33 -0
  354. package/dist/daemon/health-check.js.map +1 -0
  355. package/dist/daemon/http-server.d.ts +26 -0
  356. package/dist/daemon/http-server.d.ts.map +1 -0
  357. package/dist/daemon/http-server.js +275 -0
  358. package/dist/daemon/http-server.js.map +1 -0
  359. package/dist/daemon/proxy.d.ts +10 -0
  360. package/dist/daemon/proxy.d.ts.map +1 -0
  361. package/dist/daemon/proxy.js +93 -0
  362. package/dist/daemon/proxy.js.map +1 -0
  363. package/dist/embedding/embed.d.ts +59 -0
  364. package/dist/embedding/embed.d.ts.map +1 -0
  365. package/dist/embedding/embed.js +233 -0
  366. package/dist/embedding/embed.js.map +1 -0
  367. package/dist/embedding/fastembed-provider.d.ts +19 -0
  368. package/dist/embedding/fastembed-provider.d.ts.map +1 -0
  369. package/dist/embedding/fastembed-provider.js +51 -0
  370. package/dist/embedding/fastembed-provider.js.map +1 -0
  371. package/dist/embedding/key-terms.d.ts +12 -0
  372. package/dist/embedding/key-terms.d.ts.map +1 -0
  373. package/dist/embedding/key-terms.js +234 -0
  374. package/dist/embedding/key-terms.js.map +1 -0
  375. package/dist/extraction/boilerplate.d.ts +15 -0
  376. package/dist/extraction/boilerplate.d.ts.map +1 -0
  377. package/dist/extraction/boilerplate.js +52 -0
  378. package/dist/extraction/boilerplate.js.map +1 -0
  379. package/dist/extraction/defuddle.d.ts.map +1 -1
  380. package/dist/extraction/defuddle.js +27 -23
  381. package/dist/extraction/defuddle.js.map +1 -1
  382. package/dist/extraction/extract.d.ts.map +1 -1
  383. package/dist/extraction/extract.js +76 -76
  384. package/dist/extraction/extract.js.map +1 -1
  385. package/dist/extraction/jsonld.js +50 -54
  386. package/dist/extraction/jsonld.js.map +1 -1
  387. package/dist/extraction/lang-hints.d.ts +2 -0
  388. package/dist/extraction/lang-hints.d.ts.map +1 -0
  389. package/dist/extraction/lang-hints.js +30 -0
  390. package/dist/extraction/lang-hints.js.map +1 -0
  391. package/dist/extraction/llm-fallback.d.ts +17 -0
  392. package/dist/extraction/llm-fallback.d.ts.map +1 -0
  393. package/dist/extraction/llm-fallback.js +130 -0
  394. package/dist/extraction/llm-fallback.js.map +1 -0
  395. package/dist/extraction/markdown-sanitize.d.ts +2 -0
  396. package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
  397. package/dist/extraction/markdown-sanitize.js +151 -0
  398. package/dist/extraction/markdown-sanitize.js.map +1 -0
  399. package/dist/extraction/markdown.d.ts +11 -0
  400. package/dist/extraction/markdown.d.ts.map +1 -1
  401. package/dist/extraction/markdown.js +195 -91
  402. package/dist/extraction/markdown.js.map +1 -1
  403. package/dist/extraction/pipeline.d.ts +8 -0
  404. package/dist/extraction/pipeline.d.ts.map +1 -1
  405. package/dist/extraction/pipeline.js +57 -91
  406. package/dist/extraction/pipeline.js.map +1 -1
  407. package/dist/extraction/readability.d.ts +1 -1
  408. package/dist/extraction/readability.d.ts.map +1 -1
  409. package/dist/extraction/readability.js +28 -29
  410. package/dist/extraction/readability.js.map +1 -1
  411. package/dist/extraction/schema.d.ts +12 -0
  412. package/dist/extraction/schema.d.ts.map +1 -1
  413. package/dist/extraction/schema.js +135 -72
  414. package/dist/extraction/schema.js.map +1 -1
  415. package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
  416. package/dist/extraction/site-extractors/docs-generic.js +81 -91
  417. package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
  418. package/dist/extraction/site-extractors/github.d.ts.map +1 -1
  419. package/dist/extraction/site-extractors/github.js +87 -95
  420. package/dist/extraction/site-extractors/github.js.map +1 -1
  421. package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
  422. package/dist/extraction/site-extractors/mdn.js +46 -54
  423. package/dist/extraction/site-extractors/mdn.js.map +1 -1
  424. package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
  425. package/dist/extraction/site-extractors/stackoverflow.js +71 -80
  426. package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
  427. package/dist/extraction/structured-data.d.ts +4 -0
  428. package/dist/extraction/structured-data.d.ts.map +1 -0
  429. package/dist/extraction/structured-data.js +173 -0
  430. package/dist/extraction/structured-data.js.map +1 -0
  431. package/dist/extraction/structured.d.ts +4 -0
  432. package/dist/extraction/structured.d.ts.map +1 -0
  433. package/dist/extraction/structured.js +163 -0
  434. package/dist/extraction/structured.js.map +1 -0
  435. package/dist/extraction/v1/classifier.d.ts +3 -0
  436. package/dist/extraction/v1/classifier.d.ts.map +1 -0
  437. package/dist/extraction/v1/classifier.js +110 -0
  438. package/dist/extraction/v1/classifier.js.map +1 -0
  439. package/dist/extraction/v1/extract-provider.d.ts +16 -0
  440. package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
  441. package/dist/extraction/v1/extract-provider.js +43 -0
  442. package/dist/extraction/v1/extract-provider.js.map +1 -0
  443. package/dist/extraction/v1/local-llm.d.ts +8 -0
  444. package/dist/extraction/v1/local-llm.d.ts.map +1 -0
  445. package/dist/extraction/v1/local-llm.js +34 -0
  446. package/dist/extraction/v1/local-llm.js.map +1 -0
  447. package/dist/extraction/v1/news.d.ts +3 -0
  448. package/dist/extraction/v1/news.d.ts.map +1 -0
  449. package/dist/extraction/v1/news.js +61 -0
  450. package/dist/extraction/v1/news.js.map +1 -0
  451. package/dist/extraction/v1/product.d.ts +3 -0
  452. package/dist/extraction/v1/product.d.ts.map +1 -0
  453. package/dist/extraction/v1/product.js +166 -0
  454. package/dist/extraction/v1/product.js.map +1 -0
  455. package/dist/extraction/v1/recipe.d.ts +3 -0
  456. package/dist/extraction/v1/recipe.d.ts.map +1 -0
  457. package/dist/extraction/v1/recipe.js +136 -0
  458. package/dist/extraction/v1/recipe.js.map +1 -0
  459. package/dist/extraction/v1/routed.d.ts +17 -0
  460. package/dist/extraction/v1/routed.d.ts.map +1 -0
  461. package/dist/extraction/v1/routed.js +68 -0
  462. package/dist/extraction/v1/routed.js.map +1 -0
  463. package/dist/extraction/v1/schemas/Article.d.ts +11 -0
  464. package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
  465. package/dist/extraction/v1/schemas/Article.js +23 -0
  466. package/dist/extraction/v1/schemas/Article.js.map +1 -0
  467. package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
  468. package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
  469. package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
  470. package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
  471. package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
  472. package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
  473. package/dist/extraction/v1/schemas/EventListing.js +122 -0
  474. package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
  475. package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
  476. package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
  477. package/dist/extraction/v1/schemas/Paper.js +156 -0
  478. package/dist/extraction/v1/schemas/Paper.js.map +1 -0
  479. package/dist/extraction/v1/schemas/Product.d.ts +17 -0
  480. package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
  481. package/dist/extraction/v1/schemas/Product.js +149 -0
  482. package/dist/extraction/v1/schemas/Product.js.map +1 -0
  483. package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
  484. package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
  485. package/dist/extraction/v1/schemas/Recipe.js +160 -0
  486. package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
  487. package/dist/extraction/v1/schemas/index.d.ts +13 -0
  488. package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
  489. package/dist/extraction/v1/schemas/index.js +44 -0
  490. package/dist/extraction/v1/schemas/index.js.map +1 -0
  491. package/dist/extraction/v1/site-extractors.d.ts +5 -0
  492. package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
  493. package/dist/extraction/v1/site-extractors.js +31 -0
  494. package/dist/extraction/v1/site-extractors.js.map +1 -0
  495. package/dist/fetch/action-executor.d.ts +28 -0
  496. package/dist/fetch/action-executor.d.ts.map +1 -0
  497. package/dist/fetch/action-executor.js +88 -0
  498. package/dist/fetch/action-executor.js.map +1 -0
  499. package/dist/fetch/auth.d.ts +2 -1
  500. package/dist/fetch/auth.d.ts.map +1 -1
  501. package/dist/fetch/auth.js +56 -26
  502. package/dist/fetch/auth.js.map +1 -1
  503. package/dist/fetch/browser-pool.d.ts +30 -11
  504. package/dist/fetch/browser-pool.d.ts.map +1 -1
  505. package/dist/fetch/browser-pool.js +303 -127
  506. package/dist/fetch/browser-pool.js.map +1 -1
  507. package/dist/fetch/browser-selector.d.ts +17 -0
  508. package/dist/fetch/browser-selector.d.ts.map +1 -0
  509. package/dist/fetch/browser-selector.js +72 -0
  510. package/dist/fetch/browser-selector.js.map +1 -0
  511. package/dist/fetch/browser-types.d.ts +3 -0
  512. package/dist/fetch/browser-types.d.ts.map +1 -0
  513. package/dist/fetch/browser-types.js +45 -0
  514. package/dist/fetch/browser-types.js.map +1 -0
  515. package/dist/fetch/cdp-client.d.ts +9 -0
  516. package/dist/fetch/cdp-client.d.ts.map +1 -0
  517. package/dist/fetch/cdp-client.js +89 -0
  518. package/dist/fetch/cdp-client.js.map +1 -0
  519. package/dist/fetch/content-check.js +39 -46
  520. package/dist/fetch/content-check.js.map +1 -1
  521. package/dist/fetch/error-describe.d.ts +7 -0
  522. package/dist/fetch/error-describe.d.ts.map +1 -0
  523. package/dist/fetch/error-describe.js +37 -0
  524. package/dist/fetch/error-describe.js.map +1 -0
  525. package/dist/fetch/http-client.d.ts +4 -0
  526. package/dist/fetch/http-client.d.ts.map +1 -1
  527. package/dist/fetch/http-client.js +147 -128
  528. package/dist/fetch/http-client.js.map +1 -1
  529. package/dist/fetch/lightpanda.d.ts +28 -0
  530. package/dist/fetch/lightpanda.d.ts.map +1 -0
  531. package/dist/fetch/lightpanda.js +174 -0
  532. package/dist/fetch/lightpanda.js.map +1 -0
  533. package/dist/fetch/playwright-tier.d.ts +19 -0
  534. package/dist/fetch/playwright-tier.d.ts.map +1 -0
  535. package/dist/fetch/playwright-tier.js +76 -0
  536. package/dist/fetch/playwright-tier.js.map +1 -0
  537. package/dist/fetch/router.d.ts +49 -3
  538. package/dist/fetch/router.d.ts.map +1 -1
  539. package/dist/fetch/router.js +187 -81
  540. package/dist/fetch/router.js.map +1 -1
  541. package/dist/index.js +102 -17
  542. package/dist/index.js.map +1 -1
  543. package/dist/instructions.d.ts +31 -0
  544. package/dist/instructions.d.ts.map +1 -0
  545. package/dist/instructions.js +245 -0
  546. package/dist/instructions.js.map +1 -0
  547. package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
  548. package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
  549. package/dist/integrations/cloud/llm/anthropic.js +41 -0
  550. package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
  551. package/dist/integrations/cloud/llm/cache.d.ts +5 -0
  552. package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
  553. package/dist/integrations/cloud/llm/cache.js +49 -0
  554. package/dist/integrations/cloud/llm/cache.js.map +1 -0
  555. package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
  556. package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
  557. package/dist/integrations/cloud/llm/gemini.js +37 -0
  558. package/dist/integrations/cloud/llm/gemini.js.map +1 -0
  559. package/dist/integrations/cloud/llm/groq.d.ts +3 -0
  560. package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
  561. package/dist/integrations/cloud/llm/groq.js +74 -0
  562. package/dist/integrations/cloud/llm/groq.js.map +1 -0
  563. package/dist/integrations/cloud/llm/hash.d.ts +3 -0
  564. package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
  565. package/dist/integrations/cloud/llm/hash.js +26 -0
  566. package/dist/integrations/cloud/llm/hash.js.map +1 -0
  567. package/dist/integrations/cloud/llm/model-select.d.ts +5 -0
  568. package/dist/integrations/cloud/llm/model-select.d.ts.map +1 -0
  569. package/dist/integrations/cloud/llm/model-select.js +32 -0
  570. package/dist/integrations/cloud/llm/model-select.js.map +1 -0
  571. package/dist/integrations/cloud/llm/openai.d.ts +3 -0
  572. package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
  573. package/dist/integrations/cloud/llm/openai.js +43 -0
  574. package/dist/integrations/cloud/llm/openai.js.map +1 -0
  575. package/dist/integrations/cloud/llm/run.d.ts +27 -0
  576. package/dist/integrations/cloud/llm/run.d.ts.map +1 -0
  577. package/dist/integrations/cloud/llm/run.js +99 -0
  578. package/dist/integrations/cloud/llm/run.js.map +1 -0
  579. package/dist/integrations/cloud/llm/select.d.ts +5 -0
  580. package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
  581. package/dist/integrations/cloud/llm/select.js +30 -0
  582. package/dist/integrations/cloud/llm/select.js.map +1 -0
  583. package/dist/integrations/cloud/llm/text-adapters.d.ts +19 -0
  584. package/dist/integrations/cloud/llm/text-adapters.d.ts.map +1 -0
  585. package/dist/integrations/cloud/llm/text-adapters.js +103 -0
  586. package/dist/integrations/cloud/llm/text-adapters.js.map +1 -0
  587. package/dist/integrations/cloud/llm/types.d.ts +24 -0
  588. package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
  589. package/dist/integrations/cloud/llm/types.js +1 -0
  590. package/dist/integrations/cloud/llm/types.js.map +1 -0
  591. package/dist/integrations/cloud/llm/validate.d.ts +6 -0
  592. package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
  593. package/dist/integrations/cloud/llm/validate.js +63 -0
  594. package/dist/integrations/cloud/llm/validate.js.map +1 -0
  595. package/dist/logger.d.ts +4 -1
  596. package/dist/logger.d.ts.map +1 -1
  597. package/dist/logger.js +71 -30
  598. package/dist/logger.js.map +1 -1
  599. package/dist/pdf-parse.d.js +1 -0
  600. package/dist/pdf-parse.d.js.map +1 -0
  601. package/dist/plugins/loader.d.ts +20 -0
  602. package/dist/plugins/loader.d.ts.map +1 -0
  603. package/dist/plugins/loader.js +157 -0
  604. package/dist/plugins/loader.js.map +1 -0
  605. package/dist/plugins/registry.d.ts +26 -0
  606. package/dist/plugins/registry.d.ts.map +1 -0
  607. package/dist/plugins/registry.js +71 -0
  608. package/dist/plugins/registry.js.map +1 -0
  609. package/dist/plugins/validate.d.ts +9 -0
  610. package/dist/plugins/validate.d.ts.map +1 -0
  611. package/dist/plugins/validate.js +79 -0
  612. package/dist/plugins/validate.js.map +1 -0
  613. package/dist/providers/embed-provider.d.ts +11 -0
  614. package/dist/providers/embed-provider.d.ts.map +1 -0
  615. package/dist/providers/embed-provider.js +24 -0
  616. package/dist/providers/embed-provider.js.map +1 -0
  617. package/dist/providers/extract-provider.d.ts +23 -0
  618. package/dist/providers/extract-provider.d.ts.map +1 -0
  619. package/dist/providers/extract-provider.js +25 -0
  620. package/dist/providers/extract-provider.js.map +1 -0
  621. package/dist/providers/rerank-provider.d.ts +17 -0
  622. package/dist/providers/rerank-provider.d.ts.map +1 -0
  623. package/dist/providers/rerank-provider.js +41 -0
  624. package/dist/providers/rerank-provider.js.map +1 -0
  625. package/dist/providers/search-provider.d.ts +25 -0
  626. package/dist/providers/search-provider.d.ts.map +1 -0
  627. package/dist/providers/search-provider.js +44 -0
  628. package/dist/providers/search-provider.js.map +1 -0
  629. package/dist/providers/vector-store.d.ts +27 -0
  630. package/dist/providers/vector-store.d.ts.map +1 -0
  631. package/dist/providers/vector-store.js +27 -0
  632. package/dist/providers/vector-store.js.map +1 -0
  633. package/dist/python-env.d.ts +9 -0
  634. package/dist/python-env.d.ts.map +1 -0
  635. package/dist/python-env.js +13 -0
  636. package/dist/python-env.js.map +1 -0
  637. package/dist/repl/commands/agent.d.ts +5 -0
  638. package/dist/repl/commands/agent.d.ts.map +1 -0
  639. package/dist/repl/commands/agent.js +62 -0
  640. package/dist/repl/commands/agent.js.map +1 -0
  641. package/dist/repl/commands/cache.d.ts +4 -0
  642. package/dist/repl/commands/cache.d.ts.map +1 -0
  643. package/dist/repl/commands/cache.js +43 -0
  644. package/dist/repl/commands/cache.js.map +1 -0
  645. package/dist/repl/commands/crawl.d.ts +7 -0
  646. package/dist/repl/commands/crawl.d.ts.map +1 -0
  647. package/dist/repl/commands/crawl.js +44 -0
  648. package/dist/repl/commands/crawl.js.map +1 -0
  649. package/dist/repl/commands/extract.d.ts +5 -0
  650. package/dist/repl/commands/extract.d.ts.map +1 -0
  651. package/dist/repl/commands/extract.js +47 -0
  652. package/dist/repl/commands/extract.js.map +1 -0
  653. package/dist/repl/commands/fetch.d.ts +5 -0
  654. package/dist/repl/commands/fetch.d.ts.map +1 -0
  655. package/dist/repl/commands/fetch.js +67 -0
  656. package/dist/repl/commands/fetch.js.map +1 -0
  657. package/dist/repl/commands/find-similar.d.ts +5 -0
  658. package/dist/repl/commands/find-similar.d.ts.map +1 -0
  659. package/dist/repl/commands/find-similar.js +74 -0
  660. package/dist/repl/commands/find-similar.js.map +1 -0
  661. package/dist/repl/commands/research.d.ts +5 -0
  662. package/dist/repl/commands/research.d.ts.map +1 -0
  663. package/dist/repl/commands/research.js +65 -0
  664. package/dist/repl/commands/research.js.map +1 -0
  665. package/dist/repl/commands/search.d.ts +5 -0
  666. package/dist/repl/commands/search.d.ts.map +1 -0
  667. package/dist/repl/commands/search.js +74 -0
  668. package/dist/repl/commands/search.js.map +1 -0
  669. package/dist/repl/commands/types.d.ts +9 -0
  670. package/dist/repl/commands/types.d.ts.map +1 -0
  671. package/dist/repl/commands/types.js +1 -0
  672. package/dist/repl/commands/types.js.map +1 -0
  673. package/dist/repl/formatters.d.ts +13 -0
  674. package/dist/repl/formatters.d.ts.map +1 -0
  675. package/dist/repl/formatters.js +283 -0
  676. package/dist/repl/formatters.js.map +1 -0
  677. package/dist/repl/parser.d.ts +9 -0
  678. package/dist/repl/parser.d.ts.map +1 -0
  679. package/dist/repl/parser.js +86 -0
  680. package/dist/repl/parser.js.map +1 -0
  681. package/dist/repl/shell.d.ts +8 -0
  682. package/dist/repl/shell.d.ts.map +1 -0
  683. package/dist/repl/shell.js +184 -0
  684. package/dist/repl/shell.js.map +1 -0
  685. package/dist/research/branch-exploration.d.ts +14 -0
  686. package/dist/research/branch-exploration.d.ts.map +1 -0
  687. package/dist/research/branch-exploration.js +100 -0
  688. package/dist/research/branch-exploration.js.map +1 -0
  689. package/dist/research/brief.d.ts +6 -0
  690. package/dist/research/brief.d.ts.map +1 -0
  691. package/dist/research/brief.js +246 -0
  692. package/dist/research/brief.js.map +1 -0
  693. package/dist/research/citation-graph.d.ts +9 -0
  694. package/dist/research/citation-graph.d.ts.map +1 -0
  695. package/dist/research/citation-graph.js +114 -0
  696. package/dist/research/citation-graph.js.map +1 -0
  697. package/dist/research/decompose.d.ts +14 -0
  698. package/dist/research/decompose.d.ts.map +1 -0
  699. package/dist/research/decompose.js +439 -0
  700. package/dist/research/decompose.js.map +1 -0
  701. package/dist/research/pipeline.d.ts +5 -0
  702. package/dist/research/pipeline.d.ts.map +1 -0
  703. package/dist/research/pipeline.js +269 -0
  704. package/dist/research/pipeline.js.map +1 -0
  705. package/dist/research/synthesis-local.d.ts +19 -0
  706. package/dist/research/synthesis-local.d.ts.map +1 -0
  707. package/dist/research/synthesis-local.js +62 -0
  708. package/dist/research/synthesis-local.js.map +1 -0
  709. package/dist/research/synthesize.d.ts +10 -0
  710. package/dist/research/synthesize.d.ts.map +1 -0
  711. package/dist/research/synthesize.js +137 -0
  712. package/dist/research/synthesize.js.map +1 -0
  713. package/dist/search/answer-synthesis.d.ts +33 -0
  714. package/dist/search/answer-synthesis.d.ts.map +1 -0
  715. package/dist/search/answer-synthesis.js +244 -0
  716. package/dist/search/answer-synthesis.js.map +1 -0
  717. package/dist/search/context-formatter.d.ts +3 -0
  718. package/dist/search/context-formatter.d.ts.map +1 -0
  719. package/dist/search/context-formatter.js +56 -0
  720. package/dist/search/context-formatter.js.map +1 -0
  721. package/dist/search/dedup.d.ts +1 -0
  722. package/dist/search/dedup.d.ts.map +1 -1
  723. package/dist/search/dedup.js +40 -32
  724. package/dist/search/dedup.js.map +1 -1
  725. package/dist/search/engines/arxiv.d.ts +7 -0
  726. package/dist/search/engines/arxiv.d.ts.map +1 -0
  727. package/dist/search/engines/arxiv.js +70 -0
  728. package/dist/search/engines/arxiv.js.map +1 -0
  729. package/dist/search/engines/bing-news.d.ts +7 -0
  730. package/dist/search/engines/bing-news.d.ts.map +1 -0
  731. package/dist/search/engines/bing-news.js +97 -0
  732. package/dist/search/engines/bing-news.js.map +1 -0
  733. package/dist/search/engines/bing.d.ts +1 -0
  734. package/dist/search/engines/bing.d.ts.map +1 -1
  735. package/dist/search/engines/bing.js +100 -44
  736. package/dist/search/engines/bing.js.map +1 -1
  737. package/dist/search/engines/devdocs.d.ts +6 -0
  738. package/dist/search/engines/devdocs.d.ts.map +1 -0
  739. package/dist/search/engines/devdocs.js +56 -0
  740. package/dist/search/engines/devdocs.js.map +1 -0
  741. package/dist/search/engines/duckduckgo.d.ts.map +1 -1
  742. package/dist/search/engines/duckduckgo.js +56 -44
  743. package/dist/search/engines/duckduckgo.js.map +1 -1
  744. package/dist/search/engines/github-code.d.ts +7 -0
  745. package/dist/search/engines/github-code.d.ts.map +1 -0
  746. package/dist/search/engines/github-code.js +55 -0
  747. package/dist/search/engines/github-code.js.map +1 -0
  748. package/dist/search/engines/hn-algolia.d.ts +7 -0
  749. package/dist/search/engines/hn-algolia.d.ts.map +1 -0
  750. package/dist/search/engines/hn-algolia.js +76 -0
  751. package/dist/search/engines/hn-algolia.js.map +1 -0
  752. package/dist/search/engines/lobsters.d.ts +7 -0
  753. package/dist/search/engines/lobsters.d.ts.map +1 -0
  754. package/dist/search/engines/lobsters.js +83 -0
  755. package/dist/search/engines/lobsters.js.map +1 -0
  756. package/dist/search/engines/mdn.d.ts +7 -0
  757. package/dist/search/engines/mdn.d.ts.map +1 -0
  758. package/dist/search/engines/mdn.js +48 -0
  759. package/dist/search/engines/mdn.js.map +1 -0
  760. package/dist/search/engines/semantic-scholar.d.ts +7 -0
  761. package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
  762. package/dist/search/engines/semantic-scholar.js +69 -0
  763. package/dist/search/engines/semantic-scholar.js.map +1 -0
  764. package/dist/search/engines/stackoverflow.d.ts +7 -0
  765. package/dist/search/engines/stackoverflow.d.ts.map +1 -0
  766. package/dist/search/engines/stackoverflow.js +73 -0
  767. package/dist/search/engines/stackoverflow.js.map +1 -0
  768. package/dist/search/engines/startpage.d.ts.map +1 -1
  769. package/dist/search/engines/startpage.js +65 -46
  770. package/dist/search/engines/startpage.js.map +1 -1
  771. package/dist/search/evidence.d.ts +25 -0
  772. package/dist/search/evidence.d.ts.map +1 -0
  773. package/dist/search/evidence.js +220 -0
  774. package/dist/search/evidence.js.map +1 -0
  775. package/dist/search/filters.d.ts.map +1 -1
  776. package/dist/search/filters.js +58 -54
  777. package/dist/search/filters.js.map +1 -1
  778. package/dist/search/find-similar/crawl-rank.d.ts +9 -0
  779. package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
  780. package/dist/search/find-similar/crawl-rank.js +272 -0
  781. package/dist/search/find-similar/crawl-rank.js.map +1 -0
  782. package/dist/search/find-similar/mode.d.ts +4 -0
  783. package/dist/search/find-similar/mode.d.ts.map +1 -0
  784. package/dist/search/find-similar/mode.js +12 -0
  785. package/dist/search/find-similar/mode.js.map +1 -0
  786. package/dist/search/find-similar.d.ts +5 -0
  787. package/dist/search/find-similar.d.ts.map +1 -0
  788. package/dist/search/find-similar.js +509 -0
  789. package/dist/search/find-similar.js.map +1 -0
  790. package/dist/search/highlights.d.ts +19 -0
  791. package/dist/search/highlights.d.ts.map +1 -0
  792. package/dist/search/highlights.js +167 -0
  793. package/dist/search/highlights.js.map +1 -0
  794. package/dist/search/language-filter.d.ts +29 -0
  795. package/dist/search/language-filter.d.ts.map +1 -0
  796. package/dist/search/language-filter.js +126 -0
  797. package/dist/search/language-filter.js.map +1 -0
  798. package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
  799. package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
  800. package/dist/search/legacy/searxng-orchestrator.js +501 -0
  801. package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
  802. package/dist/search/legacy/searxng-provider.d.ts +7 -0
  803. package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
  804. package/dist/search/legacy/searxng-provider.js +11 -0
  805. package/dist/search/legacy/searxng-provider.js.map +1 -0
  806. package/dist/search/multi-query.d.ts +25 -0
  807. package/dist/search/multi-query.d.ts.map +1 -0
  808. package/dist/search/multi-query.js +228 -0
  809. package/dist/search/multi-query.js.map +1 -0
  810. package/dist/search/query.js +32 -34
  811. package/dist/search/query.js.map +1 -1
  812. package/dist/search/rerank.d.ts +3 -1
  813. package/dist/search/rerank.d.ts.map +1 -1
  814. package/dist/search/rerank.js +44 -35
  815. package/dist/search/rerank.js.map +1 -1
  816. package/dist/search/reranker/authority-boost.d.ts +3 -0
  817. package/dist/search/reranker/authority-boost.d.ts.map +1 -0
  818. package/dist/search/reranker/authority-boost.js +179 -0
  819. package/dist/search/reranker/authority-boost.js.map +1 -0
  820. package/dist/search/reranker/consensus-boost.d.ts +3 -0
  821. package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
  822. package/dist/search/reranker/consensus-boost.js +27 -0
  823. package/dist/search/reranker/consensus-boost.js.map +1 -0
  824. package/dist/search/reranker/recency-boost.d.ts +3 -0
  825. package/dist/search/reranker/recency-boost.d.ts.map +1 -0
  826. package/dist/search/reranker/recency-boost.js +13 -0
  827. package/dist/search/reranker/recency-boost.js.map +1 -0
  828. package/dist/search/reranker/recency.d.ts +3 -0
  829. package/dist/search/reranker/recency.d.ts.map +1 -0
  830. package/dist/search/reranker/recency.js +23 -0
  831. package/dist/search/reranker/recency.js.map +1 -0
  832. package/dist/search/reranker/transformers-rerank-provider.d.ts +13 -0
  833. package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
  834. package/dist/search/reranker/transformers-rerank-provider.js +94 -0
  835. package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
  836. package/dist/search/rrf.d.ts +17 -0
  837. package/dist/search/rrf.d.ts.map +1 -0
  838. package/dist/search/rrf.js +39 -0
  839. package/dist/search/rrf.js.map +1 -0
  840. package/dist/search/sampling.d.ts +25 -0
  841. package/dist/search/sampling.d.ts.map +1 -0
  842. package/dist/search/sampling.js +52 -0
  843. package/dist/search/sampling.js.map +1 -0
  844. package/dist/search/searxng.d.ts.map +1 -1
  845. package/dist/search/searxng.js +69 -79
  846. package/dist/search/searxng.js.map +1 -1
  847. package/dist/search/tokens.d.ts +3 -0
  848. package/dist/search/tokens.d.ts.map +1 -0
  849. package/dist/search/tokens.js +39 -0
  850. package/dist/search/tokens.js.map +1 -0
  851. package/dist/search/truncate.d.ts +6 -0
  852. package/dist/search/truncate.d.ts.map +1 -0
  853. package/dist/search/truncate.js +26 -0
  854. package/dist/search/truncate.js.map +1 -0
  855. package/dist/search/url-unwrap.d.ts +3 -0
  856. package/dist/search/url-unwrap.d.ts.map +1 -0
  857. package/dist/search/url-unwrap.js +43 -0
  858. package/dist/search/url-unwrap.js.map +1 -0
  859. package/dist/search/v1/context-rank.d.ts +13 -0
  860. package/dist/search/v1/context-rank.d.ts.map +1 -0
  861. package/dist/search/v1/context-rank.js +74 -0
  862. package/dist/search/v1/context-rank.js.map +1 -0
  863. package/dist/search/v1/engine-base.d.ts +27 -0
  864. package/dist/search/v1/engine-base.d.ts.map +1 -0
  865. package/dist/search/v1/engine-base.js +110 -0
  866. package/dist/search/v1/engine-base.js.map +1 -0
  867. package/dist/search/v1/intent-router.d.ts +22 -0
  868. package/dist/search/v1/intent-router.d.ts.map +1 -0
  869. package/dist/search/v1/intent-router.js +138 -0
  870. package/dist/search/v1/intent-router.js.map +1 -0
  871. package/dist/search/v1/orchestrator.d.ts +24 -0
  872. package/dist/search/v1/orchestrator.d.ts.map +1 -0
  873. package/dist/search/v1/orchestrator.js +163 -0
  874. package/dist/search/v1/orchestrator.js.map +1 -0
  875. package/dist/search/v1/recency-boost.d.ts +9 -0
  876. package/dist/search/v1/recency-boost.d.ts.map +1 -0
  877. package/dist/search/v1/recency-boost.js +37 -0
  878. package/dist/search/v1/recency-boost.js.map +1 -0
  879. package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
  880. package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
  881. package/dist/search/v1/recent-cache-dedup.js +85 -0
  882. package/dist/search/v1/recent-cache-dedup.js.map +1 -0
  883. package/dist/search/v1/rss/feed-config.d.ts +21 -0
  884. package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
  885. package/dist/search/v1/rss/feed-config.js +90 -0
  886. package/dist/search/v1/rss/feed-config.js.map +1 -0
  887. package/dist/search/v1/rss/feed-parser.d.ts +14 -0
  888. package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
  889. package/dist/search/v1/rss/feed-parser.js +104 -0
  890. package/dist/search/v1/rss/feed-parser.js.map +1 -0
  891. package/dist/search/v1/rss/feed-poller.d.ts +22 -0
  892. package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
  893. package/dist/search/v1/rss/feed-poller.js +102 -0
  894. package/dist/search/v1/rss/feed-poller.js.map +1 -0
  895. package/dist/search/v1/rss/feed-store.d.ts +30 -0
  896. package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
  897. package/dist/search/v1/rss/feed-store.js +134 -0
  898. package/dist/search/v1/rss/feed-store.js.map +1 -0
  899. package/dist/search/v1/rss/rss-engine.d.ts +6 -0
  900. package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
  901. package/dist/search/v1/rss/rss-engine.js +28 -0
  902. package/dist/search/v1/rss/rss-engine.js.map +1 -0
  903. package/dist/search/v1/v1-provider.d.ts +7 -0
  904. package/dist/search/v1/v1-provider.d.ts.map +1 -0
  905. package/dist/search/v1/v1-provider.js +68 -0
  906. package/dist/search/v1/v1-provider.js.map +1 -0
  907. package/dist/search/v1/verticals/code.d.ts +4 -0
  908. package/dist/search/v1/verticals/code.d.ts.map +1 -0
  909. package/dist/search/v1/verticals/code.js +20 -0
  910. package/dist/search/v1/verticals/code.js.map +1 -0
  911. package/dist/search/v1/verticals/docs.d.ts +4 -0
  912. package/dist/search/v1/verticals/docs.d.ts.map +1 -0
  913. package/dist/search/v1/verticals/docs.js +20 -0
  914. package/dist/search/v1/verticals/docs.js.map +1 -0
  915. package/dist/search/v1/verticals/general.d.ts +4 -0
  916. package/dist/search/v1/verticals/general.d.ts.map +1 -0
  917. package/dist/search/v1/verticals/general.js +22 -0
  918. package/dist/search/v1/verticals/general.js.map +1 -0
  919. package/dist/search/v1/verticals/news.d.ts +10 -0
  920. package/dist/search/v1/verticals/news.d.ts.map +1 -0
  921. package/dist/search/v1/verticals/news.js +52 -0
  922. package/dist/search/v1/verticals/news.js.map +1 -0
  923. package/dist/search/v1/verticals/papers.d.ts +4 -0
  924. package/dist/search/v1/verticals/papers.d.ts.map +1 -0
  925. package/dist/search/v1/verticals/papers.js +23 -0
  926. package/dist/search/v1/verticals/papers.js.map +1 -0
  927. package/dist/search/validator.js +31 -31
  928. package/dist/search/validator.js.map +1 -1
  929. package/dist/searxng/bootstrap.d.ts +30 -0
  930. package/dist/searxng/bootstrap.d.ts.map +1 -1
  931. package/dist/searxng/bootstrap.js +223 -85
  932. package/dist/searxng/bootstrap.js.map +1 -1
  933. package/dist/searxng/docker.d.ts.map +1 -1
  934. package/dist/searxng/docker.js +69 -60
  935. package/dist/searxng/docker.js.map +1 -1
  936. package/dist/searxng/process.d.ts +13 -1
  937. package/dist/searxng/process.d.ts.map +1 -1
  938. package/dist/searxng/process.js +231 -164
  939. package/dist/searxng/process.js.map +1 -1
  940. package/dist/server/backend-status.d.ts +13 -0
  941. package/dist/server/backend-status.d.ts.map +1 -0
  942. package/dist/server/backend-status.js +40 -0
  943. package/dist/server/backend-status.js.map +1 -0
  944. package/dist/server/tool-schemas.d.ts +549 -0
  945. package/dist/server/tool-schemas.d.ts.map +1 -0
  946. package/dist/server/tool-schemas.js +464 -0
  947. package/dist/server/tool-schemas.js.map +1 -0
  948. package/dist/server/warmup-on-start.d.ts +9 -0
  949. package/dist/server/warmup-on-start.d.ts.map +1 -0
  950. package/dist/server/warmup-on-start.js +55 -0
  951. package/dist/server/warmup-on-start.js.map +1 -0
  952. package/dist/server.d.ts +17 -0
  953. package/dist/server.d.ts.map +1 -1
  954. package/dist/server.js +454 -297
  955. package/dist/server.js.map +1 -1
  956. package/dist/tools/agent.d.ts +5 -0
  957. package/dist/tools/agent.d.ts.map +1 -0
  958. package/dist/tools/agent.js +128 -0
  959. package/dist/tools/agent.js.map +1 -0
  960. package/dist/tools/cache.d.ts +2 -1
  961. package/dist/tools/cache.d.ts.map +1 -1
  962. package/dist/tools/cache.js +177 -44
  963. package/dist/tools/cache.js.map +1 -1
  964. package/dist/tools/crawl.d.ts.map +1 -1
  965. package/dist/tools/crawl.js +171 -88
  966. package/dist/tools/crawl.js.map +1 -1
  967. package/dist/tools/extract.d.ts +2 -2
  968. package/dist/tools/extract.d.ts.map +1 -1
  969. package/dist/tools/extract.js +175 -59
  970. package/dist/tools/extract.js.map +1 -1
  971. package/dist/tools/fetch.d.ts +2 -2
  972. package/dist/tools/fetch.d.ts.map +1 -1
  973. package/dist/tools/fetch.js +174 -68
  974. package/dist/tools/fetch.js.map +1 -1
  975. package/dist/tools/find-similar.d.ts +5 -0
  976. package/dist/tools/find-similar.d.ts.map +1 -0
  977. package/dist/tools/find-similar.js +127 -0
  978. package/dist/tools/find-similar.js.map +1 -0
  979. package/dist/tools/research.d.ts +5 -0
  980. package/dist/tools/research.d.ts.map +1 -0
  981. package/dist/tools/research.js +107 -0
  982. package/dist/tools/research.js.map +1 -0
  983. package/dist/tools/search.d.ts +10 -2
  984. package/dist/tools/search.d.ts.map +1 -1
  985. package/dist/tools/search.js +13 -158
  986. package/dist/tools/search.js.map +1 -1
  987. package/dist/types.d.ts +350 -7
  988. package/dist/types.d.ts.map +1 -1
  989. package/dist/types.js +6 -1
  990. package/dist/types.js.map +1 -1
  991. package/dist/util/mode.d.ts +4 -0
  992. package/dist/util/mode.d.ts.map +1 -0
  993. package/dist/util/mode.js +34 -0
  994. package/dist/util/mode.js.map +1 -0
  995. package/package.json +78 -8
  996. package/dist/extraction/trafilatura.d.ts +0 -6
  997. package/dist/extraction/trafilatura.d.ts.map +0 -1
  998. package/dist/extraction/trafilatura.js +0 -105
  999. package/dist/extraction/trafilatura.js.map +0 -1
  1000. package/dist/search/flashrank.d.ts +0 -12
  1001. package/dist/search/flashrank.d.ts.map +0 -1
  1002. package/dist/search/flashrank.js +0 -63
  1003. package/dist/search/flashrank.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAiB5C,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACpD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExD,MAAM,mBAAmB,GAAG;IAC1B,uHAAuH;IACvH,iHAAiH;IACjH,uGAAuG;CACxG,CAAC;AAEF,SAAS,oBAAoB,CAAC,MAAqC;IACjE,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC;IAC9C,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,IAAI,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzD,iEAAiE;QACjE,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAA4B,EAAE;IACzE,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC;IAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAEzC,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACvF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,CAAC;YAEhB,IAAI,GAAG,YAAY,cAAc,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpD,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,YAAY,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAExF,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBACxC,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,OAAO;gBACP,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,cAAe,SAAQ,KAAK;IACa;IAA7C,YAAY,OAAe,EAAkB,SAAkB;QAC7D,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,cAAS,GAAT,SAAS,CAAS;QAE7D,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,KAAK,UAAU,kBAAkB,CAC/B,WAAmB,EACnB,OAAyB,EACzB,SAAiB,EACjB,YAAoB,EACpB,MAAuC;IAEvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,6BAA6B,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QAEtE,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/D,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACjC,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ;gBAClB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;YACtE,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,qBAAqB,CAAC,GAAG,CAAE,GAA6B,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC/G,MAAM,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC;YACzC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,cAAc,CAAC,uCAAuC,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;YAED,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,cAAc,CAAC,wBAAwB,YAAY,UAAU,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC/F,CAAC;YAED,6BAA6B;YAC7B,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,SAAS;QACX,CAAC;QAED,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,IAAY,CAAC;QACjB,IAAI,SAA6B,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC9C,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,UAAU;YACpB,IAAI;YACJ,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,OAAO;YACP,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC"}
1
+ {"version":3,"sources":["../../src/fetch/http-client.ts"],"sourcesContent":["import { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nexport interface HttpFetchOptions {\n headers?: Record<string, string>;\n timeoutMs?: number;\n conditionalHeaders?: {\n ifNoneMatch?: string;\n ifModifiedSince?: string;\n };\n}\n\nexport interface HttpFetchResult {\n url: string;\n finalUrl: string;\n html: string;\n contentType: string;\n statusCode: number;\n headers: Record<string, string>;\n rawBuffer?: Buffer;\n}\n\nconst RETRYABLE_STATUSES = new Set([429, 502, 503]);\nconst RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);\nconst REDIRECT_STATUSES = new Set([301, 302, 307, 308]);\n\nconst DEFAULT_USER_AGENTS = [\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n];\n\nfunction getRotatingUserAgent(config: { userAgent?: string | null }): string {\n if (config.userAgent) return config.userAgent;\n return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];\n}\n\nfunction isRetryableError(err: unknown): boolean {\n if (err instanceof Error) {\n const code = (err as NodeJS.ErrnoException).code;\n if (code && RETRYABLE_ERROR_CODES.has(code)) return true;\n // AbortSignal timeout throws DOMException with name TimeoutError\n if (err.name === 'TimeoutError') return true;\n }\n return false;\n}\n\nfunction backoffMs(attempt: number): number {\n return 500 * Math.pow(2, attempt) + Math.random() * 500;\n}\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport async function httpFetch(url: string, options: HttpFetchOptions = {}): Promise<HttpFetchResult> {\n const config = getConfig();\n const logger = createLogger('fetch');\n const maxRetries = config.fetchMaxRetries;\n const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;\n const maxRedirects = config.maxRedirects;\n\n let lastError: unknown;\n\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n if (attempt > 0) {\n const delay = backoffMs(attempt - 1);\n logger.debug('retrying after backoff', { attempt, delayMs: delay, url });\n await sleep(delay);\n }\n\n try {\n const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);\n return result;\n } catch (err) {\n lastError = err;\n\n if (err instanceof HttpFetchError && !err.retryable) {\n throw err;\n }\n\n const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);\n\n if (!retryable || attempt >= maxRetries) {\n throw err;\n }\n\n logger.warn('fetch failed, will retry', {\n attempt,\n url,\n error: err instanceof Error ? err.message : String(err),\n });\n }\n }\n\n throw lastError;\n}\n\nclass HttpFetchError extends Error {\n constructor(message: string, public readonly retryable: boolean) {\n super(message);\n this.name = 'HttpFetchError';\n }\n}\n\nasync function fetchWithRedirects(\n originalUrl: string,\n options: HttpFetchOptions,\n timeoutMs: number,\n maxRedirects: number,\n logger: ReturnType<typeof createLogger>,\n): Promise<HttpFetchResult> {\n const visited = new Set<string>();\n let currentUrl = originalUrl;\n let redirectCount = 0;\n\n while (true) {\n if (visited.has(currentUrl)) {\n throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);\n }\n visited.add(currentUrl);\n\n logger.debug('fetching', { url: currentUrl, attempt: redirectCount });\n\n const signal = AbortSignal.timeout(timeoutMs);\n\n let response: Response;\n try {\n const ua = getRotatingUserAgent(getConfig());\n const mergedHeaders: Record<string, string> = { 'User-Agent': ua, ...options.headers };\n // Conditional GET: inject If-None-Match / If-Modified-Since so the\n // server can return 304 + no body when the resource hasn't changed.\n // Callers (eg. etag-incremental crawl) wire these from the persisted\n // crawl_etags row for the URL.\n if (options.conditionalHeaders?.ifNoneMatch) {\n mergedHeaders['If-None-Match'] = options.conditionalHeaders.ifNoneMatch;\n }\n if (options.conditionalHeaders?.ifModifiedSince) {\n mergedHeaders['If-Modified-Since'] = options.conditionalHeaders.ifModifiedSince;\n }\n response = await fetch(currentUrl, {\n headers: mergedHeaders,\n redirect: 'manual',\n signal,\n });\n } catch (err) {\n const isTimeout = err instanceof Error && err.name === 'TimeoutError';\n const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has((err as NodeJS.ErrnoException).code ?? '');\n const retryable = isTimeout || isConnErr;\n throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });\n }\n\n if (response.status === 304) {\n const headers: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n headers[key] = value;\n });\n // Drain so the connection can be released; ignore the (empty) body.\n try { await response.arrayBuffer(); } catch { /* */ }\n return {\n url: originalUrl,\n finalUrl: currentUrl,\n html: '',\n contentType: response.headers.get('content-type') ?? '',\n statusCode: 304,\n headers,\n };\n }\n\n if (REDIRECT_STATUSES.has(response.status)) {\n const location = response.headers.get('location');\n if (!location) {\n throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);\n }\n\n redirectCount++;\n if (redirectCount > maxRedirects) {\n throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);\n }\n\n // Resolve relative redirects\n currentUrl = new URL(location, currentUrl).toString();\n continue;\n }\n\n if (RETRYABLE_STATUSES.has(response.status)) {\n throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);\n }\n\n const contentType = response.headers.get('content-type') ?? '';\n const headers: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n headers[key] = value;\n });\n\n const isPdf = contentType.includes('application/pdf');\n let html: string;\n let rawBuffer: Buffer | undefined;\n\n if (isPdf) {\n const arrayBuf = await response.arrayBuffer();\n rawBuffer = Buffer.from(arrayBuf);\n html = '';\n } else {\n html = await response.text();\n }\n\n return {\n url: originalUrl,\n finalUrl: currentUrl,\n html,\n contentType,\n statusCode: response.status,\n headers,\n rawBuffer,\n };\n }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAqB7B,MAAM,qBAAqB,oBAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC;AAClD,MAAM,wBAAwB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,CAAC;AACjF,MAAM,oBAAoB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAEtD,MAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AACF;AAEA,SAAS,qBAAqB,QAA+C;AAC3E,MAAI,OAAO,UAAW,QAAO,OAAO;AACpC,SAAO,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,oBAAoB,MAAM,CAAC;AACnF;AAEA,SAAS,iBAAiB,KAAuB;AAC/C,MAAI,eAAe,OAAO;AACxB,UAAM,OAAQ,IAA8B;AAC5C,QAAI,QAAQ,sBAAsB,IAAI,IAAI,EAAG,QAAO;AAEpD,QAAI,IAAI,SAAS,eAAgB,QAAO;AAAA,EAC1C;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAAyB;AAC1C,SAAO,MAAM,KAAK,IAAI,GAAG,OAAO,IAAI,KAAK,OAAO,IAAI;AACtD;AAEA,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AAEA,eAAsB,UAAU,KAAa,UAA4B,CAAC,GAA6B;AACrG,QAAM,SAAS,UAAU;AACzB,QAAM,SAAS,aAAa,OAAO;AACnC,QAAM,aAAa,OAAO;AAC1B,QAAM,YAAY,QAAQ,aAAa,OAAO;AAC9C,QAAM,eAAe,OAAO;AAE5B,MAAI;AAEJ,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,QAAI,UAAU,GAAG;AACf,YAAM,QAAQ,UAAU,UAAU,CAAC;AACnC,aAAO,MAAM,0BAA0B,EAAE,SAAS,SAAS,OAAO,IAAI,CAAC;AACvE,YAAM,MAAM,KAAK;AAAA,IACnB;AAEA,QAAI;AACF,YAAM,SAAS,MAAM,mBAAmB,KAAK,SAAS,WAAW,cAAc,MAAM;AACrF,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,kBAAY;AAEZ,UAAI,eAAe,kBAAkB,CAAC,IAAI,WAAW;AACnD,cAAM;AAAA,MACR;AAEA,YAAM,YAAY,eAAe,iBAAiB,IAAI,YAAY,iBAAiB,GAAG;AAEtF,UAAI,CAAC,aAAa,WAAW,YAAY;AACvC,cAAM;AAAA,MACR;AAEA,aAAO,KAAK,4BAA4B;AAAA,QACtC;AAAA,QACA;AAAA,QACA,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM;AACR;AAEA,MAAM,uBAAuB,MAAM;AAAA,EACjC,YAAY,SAAiC,WAAoB;AAC/D,UAAM,OAAO;AAD8B;AAE3C,SAAK,OAAO;AAAA,EACd;AAAA,EAH6C;AAI/C;AAEA,eAAe,mBACb,aACA,SACA,WACA,cACA,QAC0B;AAC1B,QAAM,UAAU,oBAAI,IAAY;AAChC,MAAI,aAAa;AACjB,MAAI,gBAAgB;AAEpB,SAAO,MAAM;AACX,QAAI,QAAQ,IAAI,UAAU,GAAG;AAC3B,YAAM,IAAI,eAAe,6BAA6B,UAAU,IAAI,KAAK;AAAA,IAC3E;AACA,YAAQ,IAAI,UAAU;AAEtB,WAAO,MAAM,YAAY,EAAE,KAAK,YAAY,SAAS,cAAc,CAAC;AAEpE,UAAM,SAAS,YAAY,QAAQ,SAAS;AAE5C,QAAI;AACJ,QAAI;AACF,YAAM,KAAK,qBAAqB,UAAU,CAAC;AAC3C,YAAM,gBAAwC,EAAE,cAAc,IAAI,GAAG,QAAQ,QAAQ;AAKrF,UAAI,QAAQ,oBAAoB,aAAa;AAC3C,sBAAc,eAAe,IAAI,QAAQ,mBAAmB;AAAA,MAC9D;AACA,UAAI,QAAQ,oBAAoB,iBAAiB;AAC/C,sBAAc,mBAAmB,IAAI,QAAQ,mBAAmB;AAAA,MAClE;AACA,iBAAW,MAAM,MAAM,YAAY;AAAA,QACjC,SAAS;AAAA,QACT,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,YAAM,YAAY,eAAe,SAAS,IAAI,SAAS;AACvD,YAAM,YAAY,eAAe,SAAS,sBAAsB,IAAK,IAA8B,QAAQ,EAAE;AAC7G,YAAM,YAAY,aAAa;AAC/B,YAAM,OAAO,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC;AAAA,IACxF;AAEA,QAAI,SAAS,WAAW,KAAK;AAC3B,YAAMA,WAAkC,CAAC;AACzC,eAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,QAAAA,SAAQ,GAAG,IAAI;AAAA,MACjB,CAAC;AAED,UAAI;AAAE,cAAM,SAAS,YAAY;AAAA,MAAG,QAAQ;AAAA,MAAQ;AACpD,aAAO;AAAA,QACL,KAAK;AAAA,QACL,UAAU;AAAA,QACV,MAAM;AAAA,QACN,aAAa,SAAS,QAAQ,IAAI,cAAc,KAAK;AAAA,QACrD,YAAY;AAAA,QACZ,SAAAA;AAAA,MACF;AAAA,IACF;AAEA,QAAI,kBAAkB,IAAI,SAAS,MAAM,GAAG;AAC1C,YAAM,WAAW,SAAS,QAAQ,IAAI,UAAU;AAChD,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,eAAe,uCAAuC,UAAU,IAAI,KAAK;AAAA,MACrF;AAEA;AACA,UAAI,gBAAgB,cAAc;AAChC,cAAM,IAAI,eAAe,wBAAwB,YAAY,UAAU,WAAW,IAAI,KAAK;AAAA,MAC7F;AAGA,mBAAa,IAAI,IAAI,UAAU,UAAU,EAAE,SAAS;AACpD;AAAA,IACF;AAEA,QAAI,mBAAmB,IAAI,SAAS,MAAM,GAAG;AAC3C,YAAM,IAAI,eAAe,QAAQ,SAAS,MAAM,SAAS,UAAU,IAAI,IAAI;AAAA,IAC7E;AAEA,UAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,UAAM,UAAkC,CAAC;AACzC,aAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,cAAQ,GAAG,IAAI;AAAA,IACjB,CAAC;AAED,UAAM,QAAQ,YAAY,SAAS,iBAAiB;AACpD,QAAI;AACJ,QAAI;AAEJ,QAAI,OAAO;AACT,YAAM,WAAW,MAAM,SAAS,YAAY;AAC5C,kBAAY,OAAO,KAAK,QAAQ;AAChC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,MAAM,SAAS,KAAK;AAAA,IAC7B;AAEA,WAAO;AAAA,MACL,KAAK;AAAA,MACL,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA,YAAY,SAAS;AAAA,MACrB;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;","names":["headers"]}
@@ -0,0 +1,28 @@
1
+ import { type Browser, type BrowserContext } from 'playwright';
2
+ export interface DomainRoutingStats {
3
+ domain: string;
4
+ successCount: number;
5
+ failureCount: number;
6
+ preferChromium: boolean;
7
+ lastSuccess?: string;
8
+ lastFailure?: string;
9
+ }
10
+ export interface ConnectionResult {
11
+ connected: boolean;
12
+ error?: string;
13
+ }
14
+ export declare function shouldUseLightpanda(domain: string): boolean;
15
+ export declare function recordSuccess(domain: string): void;
16
+ export declare function recordFailure(domain: string): void;
17
+ export declare function getDomainStats(domain: string): DomainRoutingStats | null;
18
+ export declare class LightpandaAdapter {
19
+ private browser;
20
+ private url;
21
+ constructor(url?: string);
22
+ connect(): Promise<ConnectionResult>;
23
+ disconnect(): Promise<void>;
24
+ isHealthy(): Promise<boolean>;
25
+ getContext(): Promise<BrowserContext | null>;
26
+ getBrowser(): Browser | null;
27
+ }
28
+ //# sourceMappingURL=lightpanda.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lightpanda.d.ts","sourceRoot":"","sources":["../../src/fetch/lightpanda.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAQzE,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAoC3D;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAgBlD;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAuBlD;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CA4BxE;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,GAAG,CAAS;gBAER,GAAG,CAAC,EAAE,MAAM;IAKlB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAwBpC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAa3B,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAc7B,UAAU,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAgBlD,UAAU,IAAI,OAAO,GAAG,IAAI;CAG7B"}
@@ -0,0 +1,174 @@
1
+ import { chromium } from "playwright";
2
+ import { getConfig } from "../config.js";
3
+ import { createLogger } from "../logger.js";
4
+ import { isCDPReachable } from "./cdp-client.js";
5
+ import { getDatabase } from "../cache/db.js";
6
+ const log = createLogger("fetch");
7
+ function shouldUseLightpanda(domain) {
8
+ try {
9
+ const config = getConfig();
10
+ if (!config.lightpandaEnabled || !config.lightpandaUrl) {
11
+ return false;
12
+ }
13
+ const db = getDatabase();
14
+ const row = db.prepare(
15
+ "SELECT failure_count, prefer_chromium FROM lightpanda_routing WHERE domain = ?"
16
+ ).get(domain);
17
+ if (!row) return true;
18
+ if (row.prefer_chromium === 1) {
19
+ log.debug("domain prefers chromium over lightpanda", { domain });
20
+ return false;
21
+ }
22
+ if (row.failure_count >= config.lightpandaFailureThreshold) {
23
+ log.debug("domain failure threshold reached for lightpanda", {
24
+ domain,
25
+ failures: row.failure_count,
26
+ threshold: config.lightpandaFailureThreshold
27
+ });
28
+ return false;
29
+ }
30
+ return true;
31
+ } catch (err) {
32
+ log.warn("shouldUseLightpanda check failed, defaulting to chromium", {
33
+ domain,
34
+ error: String(err)
35
+ });
36
+ return false;
37
+ }
38
+ }
39
+ function recordSuccess(domain) {
40
+ try {
41
+ const db = getDatabase();
42
+ db.prepare(`
43
+ INSERT INTO lightpanda_routing (domain, success_count, last_success, last_updated)
44
+ VALUES (?, 1, datetime('now'), datetime('now'))
45
+ ON CONFLICT(domain) DO UPDATE SET
46
+ success_count = success_count + 1,
47
+ last_success = datetime('now'),
48
+ last_updated = datetime('now')
49
+ `).run(domain);
50
+ log.debug("recorded lightpanda success", { domain });
51
+ } catch (err) {
52
+ log.warn("failed to record lightpanda success", { domain, error: String(err) });
53
+ }
54
+ }
55
+ function recordFailure(domain) {
56
+ try {
57
+ const config = getConfig();
58
+ const db = getDatabase();
59
+ db.prepare(`
60
+ INSERT INTO lightpanda_routing (domain, failure_count, last_failure, last_updated)
61
+ VALUES (?, 1, datetime('now'), datetime('now'))
62
+ ON CONFLICT(domain) DO UPDATE SET
63
+ failure_count = failure_count + 1,
64
+ last_failure = datetime('now'),
65
+ last_updated = datetime('now'),
66
+ prefer_chromium = CASE
67
+ WHEN failure_count + 1 >= ?
68
+ THEN 1
69
+ ELSE prefer_chromium
70
+ END
71
+ `).run(domain, config.lightpandaFailureThreshold);
72
+ log.debug("recorded lightpanda failure", { domain });
73
+ } catch (err) {
74
+ log.warn("failed to record lightpanda failure", { domain, error: String(err) });
75
+ }
76
+ }
77
+ function getDomainStats(domain) {
78
+ try {
79
+ const db = getDatabase();
80
+ const row = db.prepare(
81
+ "SELECT * FROM lightpanda_routing WHERE domain = ?"
82
+ ).get(domain);
83
+ if (!row) return null;
84
+ return {
85
+ domain: row.domain,
86
+ successCount: row.success_count,
87
+ failureCount: row.failure_count,
88
+ preferChromium: row.prefer_chromium === 1,
89
+ lastSuccess: row.last_success ?? void 0,
90
+ lastFailure: row.last_failure ?? void 0
91
+ };
92
+ } catch (err) {
93
+ log.warn("getDomainStats failed", { domain, error: String(err) });
94
+ return null;
95
+ }
96
+ }
97
+ class LightpandaAdapter {
98
+ browser = null;
99
+ url;
100
+ constructor(url) {
101
+ const config = getConfig();
102
+ this.url = url ?? config.lightpandaUrl ?? "http://localhost:9222";
103
+ }
104
+ async connect() {
105
+ try {
106
+ const reachable = await isCDPReachable(this.url);
107
+ if (!reachable) {
108
+ log.debug("lightpanda CDP not reachable", { url: this.url });
109
+ return { connected: false, error: "CDP endpoint not reachable" };
110
+ }
111
+ this.browser = await chromium.connectOverCDP(this.url);
112
+ log.info("connected to lightpanda via CDP", { url: this.url });
113
+ return { connected: true };
114
+ } catch (err) {
115
+ log.warn("lightpanda connection failed", {
116
+ url: this.url,
117
+ error: err instanceof Error ? err.message : String(err)
118
+ });
119
+ this.browser = null;
120
+ return {
121
+ connected: false,
122
+ error: err instanceof Error ? err.message : String(err)
123
+ };
124
+ }
125
+ }
126
+ async disconnect() {
127
+ try {
128
+ if (this.browser) {
129
+ await this.browser.close();
130
+ this.browser = null;
131
+ log.debug("disconnected from lightpanda");
132
+ }
133
+ } catch (err) {
134
+ log.warn("lightpanda disconnect error", { error: String(err) });
135
+ this.browser = null;
136
+ }
137
+ }
138
+ async isHealthy() {
139
+ try {
140
+ if (!this.browser) return false;
141
+ if ("isConnected" in this.browser && typeof this.browser.isConnected === "function") {
142
+ if (!this.browser.isConnected()) return false;
143
+ }
144
+ return await isCDPReachable(this.url);
145
+ } catch {
146
+ return false;
147
+ }
148
+ }
149
+ async getContext() {
150
+ try {
151
+ if (!this.browser) {
152
+ const result = await this.connect();
153
+ if (!result.connected || !this.browser) return null;
154
+ }
155
+ const contexts = this.browser.contexts();
156
+ if (contexts.length > 0) return contexts[0];
157
+ return await this.browser.newContext();
158
+ } catch (err) {
159
+ log.warn("failed to get lightpanda context", { error: String(err) });
160
+ return null;
161
+ }
162
+ }
163
+ getBrowser() {
164
+ return this.browser;
165
+ }
166
+ }
167
+ export {
168
+ LightpandaAdapter,
169
+ getDomainStats,
170
+ recordFailure,
171
+ recordSuccess,
172
+ shouldUseLightpanda
173
+ };
174
+ //# sourceMappingURL=lightpanda.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/fetch/lightpanda.ts"],"sourcesContent":["import { chromium, type Browser, type BrowserContext } from 'playwright';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\nimport { isCDPReachable } from './cdp-client.js';\nimport { getDatabase } from '../cache/db.js';\n\nconst log = createLogger('fetch');\n\nexport interface DomainRoutingStats {\n domain: string;\n successCount: number;\n failureCount: number;\n preferChromium: boolean;\n lastSuccess?: string;\n lastFailure?: string;\n}\n\nexport interface ConnectionResult {\n connected: boolean;\n error?: string;\n}\n\nexport function shouldUseLightpanda(domain: string): boolean {\n try {\n const config = getConfig();\n if (!config.lightpandaEnabled || !config.lightpandaUrl) {\n return false;\n }\n\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT failure_count, prefer_chromium FROM lightpanda_routing WHERE domain = ?',\n ).get(domain) as { failure_count: number; prefer_chromium: number } | undefined;\n\n if (!row) return true;\n\n if (row.prefer_chromium === 1) {\n log.debug('domain prefers chromium over lightpanda', { domain });\n return false;\n }\n\n if (row.failure_count >= config.lightpandaFailureThreshold) {\n log.debug('domain failure threshold reached for lightpanda', {\n domain,\n failures: row.failure_count,\n threshold: config.lightpandaFailureThreshold,\n });\n return false;\n }\n\n return true;\n } catch (err) {\n log.warn('shouldUseLightpanda check failed, defaulting to chromium', {\n domain,\n error: String(err),\n });\n return false;\n }\n}\n\nexport function recordSuccess(domain: string): void {\n try {\n const db = getDatabase();\n db.prepare(`\n INSERT INTO lightpanda_routing (domain, success_count, last_success, last_updated)\n VALUES (?, 1, datetime('now'), datetime('now'))\n ON CONFLICT(domain) DO UPDATE SET\n success_count = success_count + 1,\n last_success = datetime('now'),\n last_updated = datetime('now')\n `).run(domain);\n\n log.debug('recorded lightpanda success', { domain });\n } catch (err) {\n log.warn('failed to record lightpanda success', { domain, error: String(err) });\n }\n}\n\nexport function recordFailure(domain: string): void {\n try {\n const config = getConfig();\n const db = getDatabase();\n\n db.prepare(`\n INSERT INTO lightpanda_routing (domain, failure_count, last_failure, last_updated)\n VALUES (?, 1, datetime('now'), datetime('now'))\n ON CONFLICT(domain) DO UPDATE SET\n failure_count = failure_count + 1,\n last_failure = datetime('now'),\n last_updated = datetime('now'),\n prefer_chromium = CASE\n WHEN failure_count + 1 >= ?\n THEN 1\n ELSE prefer_chromium\n END\n `).run(domain, config.lightpandaFailureThreshold);\n\n log.debug('recorded lightpanda failure', { domain });\n } catch (err) {\n log.warn('failed to record lightpanda failure', { domain, error: String(err) });\n }\n}\n\nexport function getDomainStats(domain: string): DomainRoutingStats | null {\n try {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT * FROM lightpanda_routing WHERE domain = ?',\n ).get(domain) as {\n domain: string;\n success_count: number;\n failure_count: number;\n prefer_chromium: number;\n last_success: string | null;\n last_failure: string | null;\n } | undefined;\n\n if (!row) return null;\n\n return {\n domain: row.domain,\n successCount: row.success_count,\n failureCount: row.failure_count,\n preferChromium: row.prefer_chromium === 1,\n lastSuccess: row.last_success ?? undefined,\n lastFailure: row.last_failure ?? undefined,\n };\n } catch (err) {\n log.warn('getDomainStats failed', { domain, error: String(err) });\n return null;\n }\n}\n\nexport class LightpandaAdapter {\n private browser: Browser | null = null;\n private url: string;\n\n constructor(url?: string) {\n const config = getConfig();\n this.url = url ?? config.lightpandaUrl ?? 'http://localhost:9222';\n }\n\n async connect(): Promise<ConnectionResult> {\n try {\n const reachable = await isCDPReachable(this.url);\n if (!reachable) {\n log.debug('lightpanda CDP not reachable', { url: this.url });\n return { connected: false, error: 'CDP endpoint not reachable' };\n }\n\n this.browser = await chromium.connectOverCDP(this.url);\n log.info('connected to lightpanda via CDP', { url: this.url });\n return { connected: true };\n } catch (err) {\n log.warn('lightpanda connection failed', {\n url: this.url,\n error: err instanceof Error ? err.message : String(err),\n });\n this.browser = null;\n return {\n connected: false,\n error: err instanceof Error ? err.message : String(err),\n };\n }\n }\n\n async disconnect(): Promise<void> {\n try {\n if (this.browser) {\n await this.browser.close();\n this.browser = null;\n log.debug('disconnected from lightpanda');\n }\n } catch (err) {\n log.warn('lightpanda disconnect error', { error: String(err) });\n this.browser = null;\n }\n }\n\n async isHealthy(): Promise<boolean> {\n try {\n if (!this.browser) return false;\n\n if ('isConnected' in this.browser && typeof this.browser.isConnected === 'function') {\n if (!this.browser.isConnected()) return false;\n }\n\n return await isCDPReachable(this.url);\n } catch {\n return false;\n }\n }\n\n async getContext(): Promise<BrowserContext | null> {\n try {\n if (!this.browser) {\n const result = await this.connect();\n if (!result.connected || !this.browser) return null;\n }\n\n const contexts = this.browser.contexts();\n if (contexts.length > 0) return contexts[0];\n return await this.browser.newContext();\n } catch (err) {\n log.warn('failed to get lightpanda context', { error: String(err) });\n return null;\n }\n }\n\n getBrowser(): Browser | null {\n return this.browser;\n }\n}\n"],"mappings":"AAAA,SAAS,gBAAmD;AAC5D,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB;AAC/B,SAAS,mBAAmB;AAE5B,MAAM,MAAM,aAAa,OAAO;AAgBzB,SAAS,oBAAoB,QAAyB;AAC3D,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,QAAI,CAAC,OAAO,qBAAqB,CAAC,OAAO,eAAe;AACtD,aAAO;AAAA,IACT;AAEA,UAAM,KAAK,YAAY;AACvB,UAAM,MAAM,GAAG;AAAA,MACb;AAAA,IACF,EAAE,IAAI,MAAM;AAEZ,QAAI,CAAC,IAAK,QAAO;AAEjB,QAAI,IAAI,oBAAoB,GAAG;AAC7B,UAAI,MAAM,2CAA2C,EAAE,OAAO,CAAC;AAC/D,aAAO;AAAA,IACT;AAEA,QAAI,IAAI,iBAAiB,OAAO,4BAA4B;AAC1D,UAAI,MAAM,mDAAmD;AAAA,QAC3D;AAAA,QACA,UAAU,IAAI;AAAA,QACd,WAAW,OAAO;AAAA,MACpB,CAAC;AACD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,KAAK,4DAA4D;AAAA,MACnE;AAAA,MACA,OAAO,OAAO,GAAG;AAAA,IACnB,CAAC;AACD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,cAAc,QAAsB;AAClD,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,OAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOV,EAAE,IAAI,MAAM;AAEb,QAAI,MAAM,+BAA+B,EAAE,OAAO,CAAC;AAAA,EACrD,SAAS,KAAK;AACZ,QAAI,KAAK,uCAAuC,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EAChF;AACF;AAEO,SAAS,cAAc,QAAsB;AAClD,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,UAAM,KAAK,YAAY;AAEvB,OAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAYV,EAAE,IAAI,QAAQ,OAAO,0BAA0B;AAEhD,QAAI,MAAM,+BAA+B,EAAE,OAAO,CAAC;AAAA,EACrD,SAAS,KAAK;AACZ,QAAI,KAAK,uCAAuC,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EAChF;AACF;AAEO,SAAS,eAAe,QAA2C;AACxE,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,UAAM,MAAM,GAAG;AAAA,MACb;AAAA,IACF,EAAE,IAAI,MAAM;AASZ,QAAI,CAAC,IAAK,QAAO;AAEjB,WAAO;AAAA,MACL,QAAQ,IAAI;AAAA,MACZ,cAAc,IAAI;AAAA,MAClB,cAAc,IAAI;AAAA,MAClB,gBAAgB,IAAI,oBAAoB;AAAA,MACxC,aAAa,IAAI,gBAAgB;AAAA,MACjC,aAAa,IAAI,gBAAgB;AAAA,IACnC;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,KAAK,yBAAyB,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,WAAO;AAAA,EACT;AACF;AAEO,MAAM,kBAAkB;AAAA,EACrB,UAA0B;AAAA,EAC1B;AAAA,EAER,YAAY,KAAc;AACxB,UAAM,SAAS,UAAU;AACzB,SAAK,MAAM,OAAO,OAAO,iBAAiB;AAAA,EAC5C;AAAA,EAEA,MAAM,UAAqC;AACzC,QAAI;AACF,YAAM,YAAY,MAAM,eAAe,KAAK,GAAG;AAC/C,UAAI,CAAC,WAAW;AACd,YAAI,MAAM,gCAAgC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC3D,eAAO,EAAE,WAAW,OAAO,OAAO,6BAA6B;AAAA,MACjE;AAEA,WAAK,UAAU,MAAM,SAAS,eAAe,KAAK,GAAG;AACrD,UAAI,KAAK,mCAAmC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC7D,aAAO,EAAE,WAAW,KAAK;AAAA,IAC3B,SAAS,KAAK;AACZ,UAAI,KAAK,gCAAgC;AAAA,QACvC,KAAK,KAAK;AAAA,QACV,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AACD,WAAK,UAAU;AACf,aAAO;AAAA,QACL,WAAW;AAAA,QACX,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,QAAI;AACF,UAAI,KAAK,SAAS;AAChB,cAAM,KAAK,QAAQ,MAAM;AACzB,aAAK,UAAU;AACf,YAAI,MAAM,8BAA8B;AAAA,MAC1C;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,KAAK,+BAA+B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC9D,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAM,YAA8B;AAClC,QAAI;AACF,UAAI,CAAC,KAAK,QAAS,QAAO;AAE1B,UAAI,iBAAiB,KAAK,WAAW,OAAO,KAAK,QAAQ,gBAAgB,YAAY;AACnF,YAAI,CAAC,KAAK,QAAQ,YAAY,EAAG,QAAO;AAAA,MAC1C;AAEA,aAAO,MAAM,eAAe,KAAK,GAAG;AAAA,IACtC,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAA6C;AACjD,QAAI;AACF,UAAI,CAAC,KAAK,SAAS;AACjB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAClC,YAAI,CAAC,OAAO,aAAa,CAAC,KAAK,QAAS,QAAO;AAAA,MACjD;AAEA,YAAM,WAAW,KAAK,QAAQ,SAAS;AACvC,UAAI,SAAS,SAAS,EAAG,QAAO,SAAS,CAAC;AAC1C,aAAO,MAAM,KAAK,QAAQ,WAAW;AAAA,IACvC,SAAS,KAAK;AACZ,UAAI,KAAK,oCAAoC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACnE,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,aAA6B;AAC3B,WAAO,KAAK;AAAA,EACd;AACF;","names":[]}
@@ -0,0 +1,19 @@
1
+ import { type Browser, type BrowserContext } from 'playwright';
2
+ export interface InstallStatus {
3
+ installed: boolean;
4
+ hint?: string;
5
+ }
6
+ export declare function detectPlaywrightInstall(): Promise<InstallStatus>;
7
+ export declare function shouldEscalate(body: string): boolean;
8
+ export declare function getDaemonBrowser(): Promise<{
9
+ browser: Browser;
10
+ context: BrowserContext;
11
+ }>;
12
+ export declare function closeDaemonBrowser(): Promise<void>;
13
+ export declare function fetchWithPlaywright(url: string, opts?: {
14
+ timeoutMs?: number;
15
+ }): Promise<{
16
+ html: string;
17
+ text: string;
18
+ }>;
19
+ //# sourceMappingURL=playwright-tier.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"playwright-tier.d.ts","sourceRoot":"","sources":["../../src/fetch/playwright-tier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAMzE,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,aAAa,CAAC,CAQtE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAKpD;AAMD,wBAAsB,gBAAgB,IAAI,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAAC,CAsB/F;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAWjI"}
@@ -0,0 +1,76 @@
1
+ import { chromium } from "playwright";
2
+ import { existsSync } from "node:fs";
3
+ import { createLogger } from "../logger.js";
4
+ const log = createLogger("playwright-tier");
5
+ async function detectPlaywrightInstall() {
6
+ try {
7
+ const exec = chromium.executablePath();
8
+ if (exec && existsSync(exec)) return { installed: true };
9
+ return { installed: false, hint: "npx playwright install chromium" };
10
+ } catch {
11
+ return { installed: false, hint: "npx playwright install chromium" };
12
+ }
13
+ }
14
+ function shouldEscalate(body) {
15
+ if (!body) return true;
16
+ if (body.length < 500) return true;
17
+ if (/enable javascript/i.test(body)) return true;
18
+ return false;
19
+ }
20
+ let _browser = null;
21
+ let _ctx = null;
22
+ let _launching = null;
23
+ async function getDaemonBrowser() {
24
+ if (_browser && _ctx) return { browser: _browser, context: _ctx };
25
+ if (_launching) return _launching;
26
+ _launching = (async () => {
27
+ try {
28
+ const status = await detectPlaywrightInstall();
29
+ if (!status.installed) {
30
+ const err = new Error("playwright_not_installed");
31
+ err.hint = status.hint;
32
+ throw err;
33
+ }
34
+ const browser = await chromium.launch({ headless: true });
35
+ const context = await browser.newContext();
36
+ _browser = browser;
37
+ _ctx = context;
38
+ log.info("Playwright daemon browser launched");
39
+ return { browser, context };
40
+ } finally {
41
+ _launching = null;
42
+ }
43
+ })();
44
+ return _launching;
45
+ }
46
+ async function closeDaemonBrowser() {
47
+ _launching = null;
48
+ if (_ctx) {
49
+ await _ctx.close();
50
+ _ctx = null;
51
+ }
52
+ if (_browser) {
53
+ await _browser.close();
54
+ _browser = null;
55
+ }
56
+ }
57
+ async function fetchWithPlaywright(url, opts = {}) {
58
+ const { context } = await getDaemonBrowser();
59
+ const page = await context.newPage();
60
+ try {
61
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: opts.timeoutMs ?? 3e4 });
62
+ const html = await page.content();
63
+ const text = await page.evaluate(() => document.body?.innerText ?? "");
64
+ return { html, text };
65
+ } finally {
66
+ await page.close();
67
+ }
68
+ }
69
+ export {
70
+ closeDaemonBrowser,
71
+ detectPlaywrightInstall,
72
+ fetchWithPlaywright,
73
+ getDaemonBrowser,
74
+ shouldEscalate
75
+ };
76
+ //# sourceMappingURL=playwright-tier.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/fetch/playwright-tier.ts"],"sourcesContent":["import { chromium, type Browser, type BrowserContext } from 'playwright';\nimport { existsSync } from 'node:fs';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('playwright-tier');\n\nexport interface InstallStatus {\n installed: boolean;\n hint?: string;\n}\n\nexport async function detectPlaywrightInstall(): Promise<InstallStatus> {\n try {\n const exec = chromium.executablePath();\n if (exec && existsSync(exec)) return { installed: true };\n return { installed: false, hint: 'npx playwright install chromium' };\n } catch {\n return { installed: false, hint: 'npx playwright install chromium' };\n }\n}\n\nexport function shouldEscalate(body: string): boolean {\n if (!body) return true;\n if (body.length < 500) return true;\n if (/enable javascript/i.test(body)) return true;\n return false;\n}\n\nlet _browser: Browser | null = null;\nlet _ctx: BrowserContext | null = null;\nlet _launching: Promise<{ browser: Browser; context: BrowserContext }> | null = null;\n\nexport async function getDaemonBrowser(): Promise<{ browser: Browser; context: BrowserContext }> {\n if (_browser && _ctx) return { browser: _browser, context: _ctx };\n if (_launching) return _launching;\n _launching = (async () => {\n try {\n const status = await detectPlaywrightInstall();\n if (!status.installed) {\n const err = new Error('playwright_not_installed') as Error & { hint?: string };\n err.hint = status.hint;\n throw err;\n }\n const browser = await chromium.launch({ headless: true });\n const context = await browser.newContext();\n _browser = browser;\n _ctx = context;\n log.info('Playwright daemon browser launched');\n return { browser, context };\n } finally {\n _launching = null;\n }\n })();\n return _launching;\n}\n\nexport async function closeDaemonBrowser(): Promise<void> {\n _launching = null;\n if (_ctx) { await _ctx.close(); _ctx = null; }\n if (_browser) { await _browser.close(); _browser = null; }\n}\n\nexport async function fetchWithPlaywright(url: string, opts: { timeoutMs?: number } = {}): Promise<{ html: string; text: string }> {\n const { context } = await getDaemonBrowser();\n const page = await context.newPage();\n try {\n await page.goto(url, { waitUntil: 'domcontentloaded', timeout: opts.timeoutMs ?? 30000 });\n const html = await page.content();\n const text = await page.evaluate(() => document.body?.innerText ?? '');\n return { html, text };\n } finally {\n await page.close();\n }\n}\n"],"mappings":"AAAA,SAAS,gBAAmD;AAC5D,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,iBAAiB;AAO1C,eAAsB,0BAAkD;AACtE,MAAI;AACF,UAAM,OAAO,SAAS,eAAe;AACrC,QAAI,QAAQ,WAAW,IAAI,EAAG,QAAO,EAAE,WAAW,KAAK;AACvD,WAAO,EAAE,WAAW,OAAO,MAAM,kCAAkC;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,WAAW,OAAO,MAAM,kCAAkC;AAAA,EACrE;AACF;AAEO,SAAS,eAAe,MAAuB;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,MAAI,KAAK,SAAS,IAAK,QAAO;AAC9B,MAAI,qBAAqB,KAAK,IAAI,EAAG,QAAO;AAC5C,SAAO;AACT;AAEA,IAAI,WAA2B;AAC/B,IAAI,OAA8B;AAClC,IAAI,aAA4E;AAEhF,eAAsB,mBAA2E;AAC/F,MAAI,YAAY,KAAM,QAAO,EAAE,SAAS,UAAU,SAAS,KAAK;AAChE,MAAI,WAAY,QAAO;AACvB,gBAAc,YAAY;AACxB,QAAI;AACF,YAAM,SAAS,MAAM,wBAAwB;AAC7C,UAAI,CAAC,OAAO,WAAW;AACrB,cAAM,MAAM,IAAI,MAAM,0BAA0B;AAChD,YAAI,OAAO,OAAO;AAClB,cAAM;AAAA,MACR;AACA,YAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,YAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,iBAAW;AACX,aAAO;AACP,UAAI,KAAK,oCAAoC;AAC7C,aAAO,EAAE,SAAS,QAAQ;AAAA,IAC5B,UAAE;AACA,mBAAa;AAAA,IACf;AAAA,EACF,GAAG;AACH,SAAO;AACT;AAEA,eAAsB,qBAAoC;AACxD,eAAa;AACb,MAAI,MAAM;AAAE,UAAM,KAAK,MAAM;AAAG,WAAO;AAAA,EAAM;AAC7C,MAAI,UAAU;AAAE,UAAM,SAAS,MAAM;AAAG,eAAW;AAAA,EAAM;AAC3D;AAEA,eAAsB,oBAAoB,KAAa,OAA+B,CAAC,GAA4C;AACjI,QAAM,EAAE,QAAQ,IAAI,MAAM,iBAAiB;AAC3C,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,MAAI;AACF,UAAM,KAAK,KAAK,KAAK,EAAE,WAAW,oBAAoB,SAAS,KAAK,aAAa,IAAM,CAAC;AACxF,UAAM,OAAO,MAAM,KAAK,QAAQ;AAChC,UAAM,OAAO,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,aAAa,EAAE;AACrE,WAAO,EAAE,MAAM,KAAK;AAAA,EACtB,UAAE;AACA,UAAM,KAAK,MAAM;AAAA,EACnB;AACF;","names":[]}
@@ -1,14 +1,31 @@
1
- import type { RawFetchResult } from '../types.js';
1
+ import type { RawFetchResult, BrowserAction, Mode, StageError } from '../types.js';
2
2
  export interface RouterFetchOptions {
3
3
  renderJs?: 'auto' | 'always' | 'never';
4
4
  useAuth?: boolean;
5
5
  headers?: Record<string, string>;
6
6
  screenshot?: boolean;
7
+ actions?: BrowserAction[];
8
+ force_refresh?: boolean;
9
+ mode?: Mode;
10
+ /**
11
+ * Conditional-GET headers. When set, the HTTP path sends them with the
12
+ * request and a 304 response is returned as RawFetchResult with
13
+ * statusCode=304 + html=''. Routes that always escalate to Playwright
14
+ * (renderJs=always, useAuth, actions) ignore these headers.
15
+ */
16
+ conditionalHeaders?: {
17
+ ifNoneMatch?: string;
18
+ ifModifiedSince?: string;
19
+ };
7
20
  }
8
21
  export interface HttpClient {
9
22
  fetch(url: string, options?: {
10
23
  headers?: Record<string, string>;
11
24
  timeoutMs?: number;
25
+ conditionalHeaders?: {
26
+ ifNoneMatch?: string;
27
+ ifModifiedSince?: string;
28
+ };
12
29
  }): Promise<{
13
30
  url: string;
14
31
  finalUrl: string;
@@ -25,17 +42,46 @@ export interface BrowserPoolInterface {
25
42
  storageStatePath?: string;
26
43
  userDataDir?: string;
27
44
  screenshot?: boolean;
45
+ actions?: BrowserAction[];
46
+ cdpUrl?: string;
28
47
  }): Promise<RawFetchResult>;
29
48
  }
49
+ export type HttpFetcher = (url: string, options?: {
50
+ headers?: Record<string, string>;
51
+ timeoutMs?: number;
52
+ }) => Promise<{
53
+ url: string;
54
+ html: string;
55
+ text: string;
56
+ }>;
57
+ export type PlaywrightFetcher = (url: string, options?: {
58
+ timeoutMs?: number;
59
+ }) => Promise<{
60
+ html: string;
61
+ text: string;
62
+ }>;
63
+ export interface SmartRouterOptions {
64
+ httpClient?: HttpClient;
65
+ browserPool?: BrowserPoolInterface;
66
+ httpFetcher?: HttpFetcher;
67
+ playwrightFetcher?: PlaywrightFetcher;
68
+ }
30
69
  interface DomainStats {
31
70
  failureCount: number;
32
71
  preferPlaywright: boolean;
33
72
  }
34
73
  export declare class SmartRouter {
35
- private readonly httpClient;
36
- private readonly browserPool;
37
74
  private readonly domainMap;
75
+ private readonly httpClient?;
76
+ private readonly browserPool?;
77
+ private readonly httpFetcher;
78
+ private readonly playwrightFetcher;
38
79
  constructor(httpClient: HttpClient, browserPool: BrowserPoolInterface);
80
+ constructor(options: SmartRouterOptions);
81
+ private makeDefaultHttpFetcher;
82
+ fetch(url: string, options: RouterFetchOptions & {
83
+ mode: 'stealth';
84
+ }): Promise<RawFetchResult | StageError>;
39
85
  fetch(url: string, options?: RouterFetchOptions): Promise<RawFetchResult>;
40
86
  getDomainStats(domain: string): DomainStats | undefined;
41
87
  private ensureStats;
@@ -1 +1 @@
1
- {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACjE,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GACpH,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;gBAGzC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,oBAAoB;IAG9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC;IA6DnF,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}
1
+ {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEnF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE;YACnB,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,eAAe,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;KACH,GACA,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChK,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KAC/D,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE1D,MAAM,MAAM,iBAAiB,GAAG,CAC9B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KAC7B,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE7C,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,WAAW,CAAC,EAAE,oBAAoB,CAAC;IACnC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAuB;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;gBAE1C,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,oBAAoB;gBACzD,OAAO,EAAE,kBAAkB;IAiCvC,OAAO,CAAC,sBAAsB;IAUxB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,GAAG,UAAU,CAAC;IAC3G,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC;IA0J/E,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}