@staticn0va/wigolo 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (982) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +195 -73
  3. package/SKILL.md +382 -0
  4. package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
  5. package/assets/blocks/claude-code/wigolo-command.md +40 -0
  6. package/assets/blocks/cursor/wigolo.mdc +46 -0
  7. package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
  8. package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
  9. package/assets/skills/wigolo/SKILL.md +50 -0
  10. package/assets/skills/wigolo/rules/cache-first.md +30 -0
  11. package/assets/skills/wigolo/rules/synthesis.md +43 -0
  12. package/assets/skills/wigolo-agent/SKILL.md +73 -0
  13. package/assets/skills/wigolo-crawl/SKILL.md +60 -0
  14. package/assets/skills/wigolo-extract/SKILL.md +59 -0
  15. package/assets/skills/wigolo-fetch/SKILL.md +65 -0
  16. package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
  17. package/assets/skills/wigolo-research/SKILL.md +77 -0
  18. package/assets/skills/wigolo-search/SKILL.md +78 -0
  19. package/dist/agent/executor.d.ts +33 -0
  20. package/dist/agent/executor.d.ts.map +1 -0
  21. package/dist/agent/executor.js +233 -0
  22. package/dist/agent/executor.js.map +1 -0
  23. package/dist/agent/pipeline.d.ts +5 -0
  24. package/dist/agent/pipeline.d.ts.map +1 -0
  25. package/dist/agent/pipeline.js +208 -0
  26. package/dist/agent/pipeline.js.map +1 -0
  27. package/dist/agent/planner.d.ts +13 -0
  28. package/dist/agent/planner.d.ts.map +1 -0
  29. package/dist/agent/planner.js +271 -0
  30. package/dist/agent/planner.js.map +1 -0
  31. package/dist/agent/relevance.d.ts +15 -0
  32. package/dist/agent/relevance.d.ts.map +1 -0
  33. package/dist/agent/relevance.js +60 -0
  34. package/dist/agent/relevance.js.map +1 -0
  35. package/dist/cache/backfill-embeddings.d.ts +23 -0
  36. package/dist/cache/backfill-embeddings.d.ts.map +1 -0
  37. package/dist/cache/backfill-embeddings.js +105 -0
  38. package/dist/cache/backfill-embeddings.js.map +1 -0
  39. package/dist/cache/change-detector.d.ts +7 -0
  40. package/dist/cache/change-detector.d.ts.map +1 -0
  41. package/dist/cache/change-detector.js +43 -0
  42. package/dist/cache/change-detector.js.map +1 -0
  43. package/dist/cache/db.d.ts +1 -0
  44. package/dist/cache/db.d.ts.map +1 -1
  45. package/dist/cache/db.js +94 -22
  46. package/dist/cache/db.js.map +1 -1
  47. package/dist/cache/diff-summary.d.ts +2 -0
  48. package/dist/cache/diff-summary.d.ts.map +1 -0
  49. package/dist/cache/diff-summary.js +82 -0
  50. package/dist/cache/diff-summary.js.map +1 -0
  51. package/dist/cache/migrations/runner.d.ts +29 -0
  52. package/dist/cache/migrations/runner.d.ts.map +1 -0
  53. package/dist/cache/migrations/runner.js +147 -0
  54. package/dist/cache/migrations/runner.js.map +1 -0
  55. package/dist/cache/sqlite-vec-store.d.ts +42 -0
  56. package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
  57. package/dist/cache/sqlite-vec-store.js +176 -0
  58. package/dist/cache/sqlite-vec-store.js.map +1 -0
  59. package/dist/cache/store.d.ts +46 -1
  60. package/dist/cache/store.d.ts.map +1 -1
  61. package/dist/cache/store.js +362 -168
  62. package/dist/cache/store.js.map +1 -1
  63. package/dist/cli/agents/antigravity.d.ts +20 -0
  64. package/dist/cli/agents/antigravity.d.ts.map +1 -0
  65. package/dist/cli/agents/antigravity.js +49 -0
  66. package/dist/cli/agents/antigravity.js.map +1 -0
  67. package/dist/cli/agents/claude-code.d.ts +25 -0
  68. package/dist/cli/agents/claude-code.d.ts.map +1 -0
  69. package/dist/cli/agents/claude-code.js +111 -0
  70. package/dist/cli/agents/claude-code.js.map +1 -0
  71. package/dist/cli/agents/cursor.d.ts +21 -0
  72. package/dist/cli/agents/cursor.d.ts.map +1 -0
  73. package/dist/cli/agents/cursor.js +58 -0
  74. package/dist/cli/agents/cursor.js.map +1 -0
  75. package/dist/cli/agents/gemini-cli.d.ts +21 -0
  76. package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
  77. package/dist/cli/agents/gemini-cli.js +55 -0
  78. package/dist/cli/agents/gemini-cli.js.map +1 -0
  79. package/dist/cli/agents/registry.d.ts +21 -0
  80. package/dist/cli/agents/registry.d.ts.map +1 -0
  81. package/dist/cli/agents/registry.js +27 -0
  82. package/dist/cli/agents/registry.js.map +1 -0
  83. package/dist/cli/agents/utils.d.ts +26 -0
  84. package/dist/cli/agents/utils.d.ts.map +1 -0
  85. package/dist/cli/agents/utils.js +136 -0
  86. package/dist/cli/agents/utils.js.map +1 -0
  87. package/dist/cli/agents/vscode.d.ts +21 -0
  88. package/dist/cli/agents/vscode.d.ts.map +1 -0
  89. package/dist/cli/agents/vscode.js +62 -0
  90. package/dist/cli/agents/vscode.js.map +1 -0
  91. package/dist/cli/auth.d.ts +2 -0
  92. package/dist/cli/auth.d.ts.map +1 -0
  93. package/dist/cli/auth.js +94 -0
  94. package/dist/cli/auth.js.map +1 -0
  95. package/dist/cli/backfill.d.ts +2 -0
  96. package/dist/cli/backfill.d.ts.map +1 -0
  97. package/dist/cli/backfill.js +58 -0
  98. package/dist/cli/backfill.js.map +1 -0
  99. package/dist/cli/daemon.d.ts +6 -1
  100. package/dist/cli/daemon.d.ts.map +1 -1
  101. package/dist/cli/daemon.js +61 -3
  102. package/dist/cli/daemon.js.map +1 -1
  103. package/dist/cli/doctor.d.ts +8 -0
  104. package/dist/cli/doctor.d.ts.map +1 -0
  105. package/dist/cli/doctor.js +318 -0
  106. package/dist/cli/doctor.js.map +1 -0
  107. package/dist/cli/health.d.ts +1 -1
  108. package/dist/cli/health.d.ts.map +1 -1
  109. package/dist/cli/health.js +42 -3
  110. package/dist/cli/health.js.map +1 -1
  111. package/dist/cli/help.d.ts +6 -0
  112. package/dist/cli/help.d.ts.map +1 -0
  113. package/dist/cli/help.js +63 -0
  114. package/dist/cli/help.js.map +1 -0
  115. package/dist/cli/index.d.ts +1 -1
  116. package/dist/cli/index.d.ts.map +1 -1
  117. package/dist/cli/index.js +35 -7
  118. package/dist/cli/index.js.map +1 -1
  119. package/dist/cli/init.d.ts +2 -0
  120. package/dist/cli/init.d.ts.map +1 -0
  121. package/dist/cli/init.js +201 -0
  122. package/dist/cli/init.js.map +1 -0
  123. package/dist/cli/plugin.d.ts +5 -0
  124. package/dist/cli/plugin.d.ts.map +1 -0
  125. package/dist/cli/plugin.js +185 -0
  126. package/dist/cli/plugin.js.map +1 -0
  127. package/dist/cli/setup-mcp.d.ts +2 -0
  128. package/dist/cli/setup-mcp.d.ts.map +1 -0
  129. package/dist/cli/setup-mcp.js +114 -0
  130. package/dist/cli/setup-mcp.js.map +1 -0
  131. package/dist/cli/shell.d.ts +2 -0
  132. package/dist/cli/shell.d.ts.map +1 -0
  133. package/dist/cli/shell.js +86 -0
  134. package/dist/cli/shell.js.map +1 -0
  135. package/dist/cli/status.d.ts +2 -0
  136. package/dist/cli/status.d.ts.map +1 -0
  137. package/dist/cli/status.js +31 -0
  138. package/dist/cli/status.js.map +1 -0
  139. package/dist/cli/telemetry.d.ts +10 -0
  140. package/dist/cli/telemetry.d.ts.map +1 -0
  141. package/dist/cli/telemetry.js +56 -0
  142. package/dist/cli/telemetry.js.map +1 -0
  143. package/dist/cli/tui/agents-types.d.ts +28 -0
  144. package/dist/cli/tui/agents-types.d.ts.map +1 -0
  145. package/dist/cli/tui/agents-types.js +1 -0
  146. package/dist/cli/tui/agents-types.js.map +1 -0
  147. package/dist/cli/tui/agents.d.ts +11 -0
  148. package/dist/cli/tui/agents.d.ts.map +1 -0
  149. package/dist/cli/tui/agents.js +93 -0
  150. package/dist/cli/tui/agents.js.map +1 -0
  151. package/dist/cli/tui/banner.d.ts +3 -0
  152. package/dist/cli/tui/banner.d.ts.map +1 -0
  153. package/dist/cli/tui/banner.js +30 -0
  154. package/dist/cli/tui/banner.js.map +1 -0
  155. package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
  156. package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
  157. package/dist/cli/tui/components/AgentSelect.js +116 -0
  158. package/dist/cli/tui/components/AgentSelect.js.map +1 -0
  159. package/dist/cli/tui/components/Banner.d.ts +6 -0
  160. package/dist/cli/tui/components/Banner.d.ts.map +1 -0
  161. package/dist/cli/tui/components/Banner.js +25 -0
  162. package/dist/cli/tui/components/Banner.js.map +1 -0
  163. package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
  164. package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
  165. package/dist/cli/tui/components/BrowserSelect.js +19 -0
  166. package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
  167. package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
  168. package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
  169. package/dist/cli/tui/components/InstallProgress.js +67 -0
  170. package/dist/cli/tui/components/InstallProgress.js.map +1 -0
  171. package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
  172. package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
  173. package/dist/cli/tui/components/SkillInstall.js +94 -0
  174. package/dist/cli/tui/components/SkillInstall.js.map +1 -0
  175. package/dist/cli/tui/components/Summary.d.ts +22 -0
  176. package/dist/cli/tui/components/Summary.d.ts.map +1 -0
  177. package/dist/cli/tui/components/Summary.js +135 -0
  178. package/dist/cli/tui/components/Summary.js.map +1 -0
  179. package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
  180. package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
  181. package/dist/cli/tui/components/SystemCheck.js +71 -0
  182. package/dist/cli/tui/components/SystemCheck.js.map +1 -0
  183. package/dist/cli/tui/components/Verification.d.ts +8 -0
  184. package/dist/cli/tui/components/Verification.d.ts.map +1 -0
  185. package/dist/cli/tui/components/Verification.js +63 -0
  186. package/dist/cli/tui/components/Verification.js.map +1 -0
  187. package/dist/cli/tui/config-writer-cli.d.ts +12 -0
  188. package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
  189. package/dist/cli/tui/config-writer-cli.js +39 -0
  190. package/dist/cli/tui/config-writer-cli.js.map +1 -0
  191. package/dist/cli/tui/config-writer-json.d.ts +16 -0
  192. package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
  193. package/dist/cli/tui/config-writer-json.js +86 -0
  194. package/dist/cli/tui/config-writer-json.js.map +1 -0
  195. package/dist/cli/tui/config-writer-toml.d.ts +16 -0
  196. package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
  197. package/dist/cli/tui/config-writer-toml.js +83 -0
  198. package/dist/cli/tui/config-writer-toml.js.map +1 -0
  199. package/dist/cli/tui/config-writer.d.ts +25 -0
  200. package/dist/cli/tui/config-writer.d.ts.map +1 -0
  201. package/dist/cli/tui/config-writer.js +101 -0
  202. package/dist/cli/tui/config-writer.js.map +1 -0
  203. package/dist/cli/tui/detect-helpers.d.ts +6 -0
  204. package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
  205. package/dist/cli/tui/detect-helpers.js +45 -0
  206. package/dist/cli/tui/detect-helpers.js.map +1 -0
  207. package/dist/cli/tui/extras-prompt.d.ts +7 -0
  208. package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
  209. package/dist/cli/tui/extras-prompt.js +42 -0
  210. package/dist/cli/tui/extras-prompt.js.map +1 -0
  211. package/dist/cli/tui/flags-types.d.ts +19 -0
  212. package/dist/cli/tui/flags-types.d.ts.map +1 -0
  213. package/dist/cli/tui/flags-types.js +23 -0
  214. package/dist/cli/tui/flags-types.js.map +1 -0
  215. package/dist/cli/tui/flags.d.ts +5 -0
  216. package/dist/cli/tui/flags.d.ts.map +1 -0
  217. package/dist/cli/tui/flags.js +132 -0
  218. package/dist/cli/tui/flags.js.map +1 -0
  219. package/dist/cli/tui/format.d.ts +14 -0
  220. package/dist/cli/tui/format.d.ts.map +1 -0
  221. package/dist/cli/tui/format.js +37 -0
  222. package/dist/cli/tui/format.js.map +1 -0
  223. package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
  224. package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
  225. package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
  226. package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
  227. package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
  228. package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
  229. package/dist/cli/tui/hooks/useInstall.js +90 -0
  230. package/dist/cli/tui/hooks/useInstall.js.map +1 -0
  231. package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
  232. package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
  233. package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
  234. package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
  235. package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
  236. package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
  237. package/dist/cli/tui/hooks/useVerify.js +71 -0
  238. package/dist/cli/tui/hooks/useVerify.js.map +1 -0
  239. package/dist/cli/tui/ink-init.d.ts +2 -0
  240. package/dist/cli/tui/ink-init.d.ts.map +1 -0
  241. package/dist/cli/tui/ink-init.js +198 -0
  242. package/dist/cli/tui/ink-init.js.map +1 -0
  243. package/dist/cli/tui/reporter-auto.d.ts +7 -0
  244. package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
  245. package/dist/cli/tui/reporter-auto.js +15 -0
  246. package/dist/cli/tui/reporter-auto.js.map +1 -0
  247. package/dist/cli/tui/reporter.d.ts +26 -0
  248. package/dist/cli/tui/reporter.d.ts.map +1 -0
  249. package/dist/cli/tui/reporter.js +32 -0
  250. package/dist/cli/tui/reporter.js.map +1 -0
  251. package/dist/cli/tui/run-command.d.ts +14 -0
  252. package/dist/cli/tui/run-command.d.ts.map +1 -0
  253. package/dist/cli/tui/run-command.js +72 -0
  254. package/dist/cli/tui/run-command.js.map +1 -0
  255. package/dist/cli/tui/select-agents.d.ts +6 -0
  256. package/dist/cli/tui/select-agents.d.ts.map +1 -0
  257. package/dist/cli/tui/select-agents.js +32 -0
  258. package/dist/cli/tui/select-agents.js.map +1 -0
  259. package/dist/cli/tui/status-agents.d.ts +11 -0
  260. package/dist/cli/tui/status-agents.d.ts.map +1 -0
  261. package/dist/cli/tui/status-agents.js +53 -0
  262. package/dist/cli/tui/status-agents.js.map +1 -0
  263. package/dist/cli/tui/status-cache.d.ts +6 -0
  264. package/dist/cli/tui/status-cache.d.ts.map +1 -0
  265. package/dist/cli/tui/status-cache.js +39 -0
  266. package/dist/cli/tui/status-cache.js.map +1 -0
  267. package/dist/cli/tui/status-format.d.ts +14 -0
  268. package/dist/cli/tui/status-format.d.ts.map +1 -0
  269. package/dist/cli/tui/status-format.js +41 -0
  270. package/dist/cli/tui/status-format.js.map +1 -0
  271. package/dist/cli/tui/status-python.d.ts +6 -0
  272. package/dist/cli/tui/status-python.d.ts.map +1 -0
  273. package/dist/cli/tui/status-python.js +30 -0
  274. package/dist/cli/tui/status-python.js.map +1 -0
  275. package/dist/cli/tui/system-check.d.ts +24 -0
  276. package/dist/cli/tui/system-check.d.ts.map +1 -0
  277. package/dist/cli/tui/system-check.js +103 -0
  278. package/dist/cli/tui/system-check.js.map +1 -0
  279. package/dist/cli/tui/tui-reporter.d.ts +19 -0
  280. package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
  281. package/dist/cli/tui/tui-reporter.js +95 -0
  282. package/dist/cli/tui/tui-reporter.js.map +1 -0
  283. package/dist/cli/tui/utils/config-writer.d.ts +3 -0
  284. package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
  285. package/dist/cli/tui/utils/config-writer.js +22 -0
  286. package/dist/cli/tui/utils/config-writer.js.map +1 -0
  287. package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
  288. package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
  289. package/dist/cli/tui/utils/suppress-logs.js +11 -0
  290. package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
  291. package/dist/cli/tui/verify-suggestions.d.ts +5 -0
  292. package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
  293. package/dist/cli/tui/verify-suggestions.js +20 -0
  294. package/dist/cli/tui/verify-suggestions.js.map +1 -0
  295. package/dist/cli/tui/verify.d.ts +14 -0
  296. package/dist/cli/tui/verify.d.ts.map +1 -0
  297. package/dist/cli/tui/verify.js +101 -0
  298. package/dist/cli/tui/verify.js.map +1 -0
  299. package/dist/cli/tui/version.d.ts +2 -0
  300. package/dist/cli/tui/version.d.ts.map +1 -0
  301. package/dist/cli/tui/version.js +14 -0
  302. package/dist/cli/tui/version.js.map +1 -0
  303. package/dist/cli/uninstall.d.ts +2 -0
  304. package/dist/cli/uninstall.d.ts.map +1 -0
  305. package/dist/cli/uninstall.js +57 -0
  306. package/dist/cli/uninstall.js.map +1 -0
  307. package/dist/cli/warmup.d.ts +10 -2
  308. package/dist/cli/warmup.d.ts.map +1 -1
  309. package/dist/cli/warmup.js +226 -93
  310. package/dist/cli/warmup.js.map +1 -1
  311. package/dist/config.d.ts +28 -2
  312. package/dist/config.d.ts.map +1 -1
  313. package/dist/config.js +106 -56
  314. package/dist/config.js.map +1 -1
  315. package/dist/crawl/crawler.d.ts +6 -0
  316. package/dist/crawl/crawler.d.ts.map +1 -1
  317. package/dist/crawl/crawler.js +210 -209
  318. package/dist/crawl/crawler.js.map +1 -1
  319. package/dist/crawl/dedup.d.ts +1 -0
  320. package/dist/crawl/dedup.d.ts.map +1 -1
  321. package/dist/crawl/dedup.js +124 -81
  322. package/dist/crawl/dedup.js.map +1 -1
  323. package/dist/crawl/etag-incremental.d.ts +43 -0
  324. package/dist/crawl/etag-incremental.d.ts.map +1 -0
  325. package/dist/crawl/etag-incremental.js +94 -0
  326. package/dist/crawl/etag-incremental.js.map +1 -0
  327. package/dist/crawl/index-to-vec.d.ts +10 -0
  328. package/dist/crawl/index-to-vec.d.ts.map +1 -0
  329. package/dist/crawl/index-to-vec.js +44 -0
  330. package/dist/crawl/index-to-vec.js.map +1 -0
  331. package/dist/crawl/mapper.js +136 -164
  332. package/dist/crawl/mapper.js.map +1 -1
  333. package/dist/crawl/rate-limiter.js +63 -66
  334. package/dist/crawl/rate-limiter.js.map +1 -1
  335. package/dist/crawl/robots.js +58 -57
  336. package/dist/crawl/robots.js.map +1 -1
  337. package/dist/crawl/sitemap-first.d.ts +12 -0
  338. package/dist/crawl/sitemap-first.d.ts.map +1 -0
  339. package/dist/crawl/sitemap-first.js +47 -0
  340. package/dist/crawl/sitemap-first.js.map +1 -0
  341. package/dist/crawl/sitemap.js +33 -32
  342. package/dist/crawl/sitemap.js.map +1 -1
  343. package/dist/crawl/url-utils.d.ts +1 -0
  344. package/dist/crawl/url-utils.d.ts.map +1 -1
  345. package/dist/crawl/url-utils.js +49 -37
  346. package/dist/crawl/url-utils.js.map +1 -1
  347. package/dist/daemon/health-check.d.ts +16 -0
  348. package/dist/daemon/health-check.d.ts.map +1 -0
  349. package/dist/daemon/health-check.js +33 -0
  350. package/dist/daemon/health-check.js.map +1 -0
  351. package/dist/daemon/http-server.d.ts +26 -0
  352. package/dist/daemon/http-server.d.ts.map +1 -0
  353. package/dist/daemon/http-server.js +275 -0
  354. package/dist/daemon/http-server.js.map +1 -0
  355. package/dist/daemon/proxy.d.ts +10 -0
  356. package/dist/daemon/proxy.d.ts.map +1 -0
  357. package/dist/daemon/proxy.js +93 -0
  358. package/dist/daemon/proxy.js.map +1 -0
  359. package/dist/embedding/embed.d.ts +59 -0
  360. package/dist/embedding/embed.d.ts.map +1 -0
  361. package/dist/embedding/embed.js +233 -0
  362. package/dist/embedding/embed.js.map +1 -0
  363. package/dist/embedding/fastembed-provider.d.ts +19 -0
  364. package/dist/embedding/fastembed-provider.d.ts.map +1 -0
  365. package/dist/embedding/fastembed-provider.js +51 -0
  366. package/dist/embedding/fastembed-provider.js.map +1 -0
  367. package/dist/embedding/key-terms.d.ts +12 -0
  368. package/dist/embedding/key-terms.d.ts.map +1 -0
  369. package/dist/embedding/key-terms.js +234 -0
  370. package/dist/embedding/key-terms.js.map +1 -0
  371. package/dist/extraction/boilerplate.d.ts +15 -0
  372. package/dist/extraction/boilerplate.d.ts.map +1 -0
  373. package/dist/extraction/boilerplate.js +52 -0
  374. package/dist/extraction/boilerplate.js.map +1 -0
  375. package/dist/extraction/defuddle.d.ts.map +1 -1
  376. package/dist/extraction/defuddle.js +27 -23
  377. package/dist/extraction/defuddle.js.map +1 -1
  378. package/dist/extraction/extract.d.ts.map +1 -1
  379. package/dist/extraction/extract.js +76 -76
  380. package/dist/extraction/extract.js.map +1 -1
  381. package/dist/extraction/jsonld.js +50 -54
  382. package/dist/extraction/jsonld.js.map +1 -1
  383. package/dist/extraction/lang-hints.d.ts +2 -0
  384. package/dist/extraction/lang-hints.d.ts.map +1 -0
  385. package/dist/extraction/lang-hints.js +30 -0
  386. package/dist/extraction/lang-hints.js.map +1 -0
  387. package/dist/extraction/llm-fallback.d.ts +17 -0
  388. package/dist/extraction/llm-fallback.d.ts.map +1 -0
  389. package/dist/extraction/llm-fallback.js +130 -0
  390. package/dist/extraction/llm-fallback.js.map +1 -0
  391. package/dist/extraction/markdown-sanitize.d.ts +2 -0
  392. package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
  393. package/dist/extraction/markdown-sanitize.js +151 -0
  394. package/dist/extraction/markdown-sanitize.js.map +1 -0
  395. package/dist/extraction/markdown.d.ts +11 -0
  396. package/dist/extraction/markdown.d.ts.map +1 -1
  397. package/dist/extraction/markdown.js +195 -91
  398. package/dist/extraction/markdown.js.map +1 -1
  399. package/dist/extraction/pipeline.d.ts +8 -0
  400. package/dist/extraction/pipeline.d.ts.map +1 -1
  401. package/dist/extraction/pipeline.js +57 -91
  402. package/dist/extraction/pipeline.js.map +1 -1
  403. package/dist/extraction/readability.d.ts +1 -1
  404. package/dist/extraction/readability.d.ts.map +1 -1
  405. package/dist/extraction/readability.js +28 -29
  406. package/dist/extraction/readability.js.map +1 -1
  407. package/dist/extraction/schema.d.ts +12 -0
  408. package/dist/extraction/schema.d.ts.map +1 -1
  409. package/dist/extraction/schema.js +135 -72
  410. package/dist/extraction/schema.js.map +1 -1
  411. package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
  412. package/dist/extraction/site-extractors/docs-generic.js +81 -91
  413. package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
  414. package/dist/extraction/site-extractors/github.d.ts.map +1 -1
  415. package/dist/extraction/site-extractors/github.js +87 -95
  416. package/dist/extraction/site-extractors/github.js.map +1 -1
  417. package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
  418. package/dist/extraction/site-extractors/mdn.js +46 -54
  419. package/dist/extraction/site-extractors/mdn.js.map +1 -1
  420. package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
  421. package/dist/extraction/site-extractors/stackoverflow.js +71 -80
  422. package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
  423. package/dist/extraction/structured-data.d.ts +4 -0
  424. package/dist/extraction/structured-data.d.ts.map +1 -0
  425. package/dist/extraction/structured-data.js +173 -0
  426. package/dist/extraction/structured-data.js.map +1 -0
  427. package/dist/extraction/structured.d.ts +4 -0
  428. package/dist/extraction/structured.d.ts.map +1 -0
  429. package/dist/extraction/structured.js +163 -0
  430. package/dist/extraction/structured.js.map +1 -0
  431. package/dist/extraction/v1/classifier.d.ts +3 -0
  432. package/dist/extraction/v1/classifier.d.ts.map +1 -0
  433. package/dist/extraction/v1/classifier.js +110 -0
  434. package/dist/extraction/v1/classifier.js.map +1 -0
  435. package/dist/extraction/v1/extract-provider.d.ts +16 -0
  436. package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
  437. package/dist/extraction/v1/extract-provider.js +43 -0
  438. package/dist/extraction/v1/extract-provider.js.map +1 -0
  439. package/dist/extraction/v1/local-llm.d.ts +8 -0
  440. package/dist/extraction/v1/local-llm.d.ts.map +1 -0
  441. package/dist/extraction/v1/local-llm.js +58 -0
  442. package/dist/extraction/v1/local-llm.js.map +1 -0
  443. package/dist/extraction/v1/news.d.ts +3 -0
  444. package/dist/extraction/v1/news.d.ts.map +1 -0
  445. package/dist/extraction/v1/news.js +61 -0
  446. package/dist/extraction/v1/news.js.map +1 -0
  447. package/dist/extraction/v1/product.d.ts +3 -0
  448. package/dist/extraction/v1/product.d.ts.map +1 -0
  449. package/dist/extraction/v1/product.js +166 -0
  450. package/dist/extraction/v1/product.js.map +1 -0
  451. package/dist/extraction/v1/recipe.d.ts +3 -0
  452. package/dist/extraction/v1/recipe.d.ts.map +1 -0
  453. package/dist/extraction/v1/recipe.js +136 -0
  454. package/dist/extraction/v1/recipe.js.map +1 -0
  455. package/dist/extraction/v1/routed.d.ts +17 -0
  456. package/dist/extraction/v1/routed.d.ts.map +1 -0
  457. package/dist/extraction/v1/routed.js +68 -0
  458. package/dist/extraction/v1/routed.js.map +1 -0
  459. package/dist/extraction/v1/schemas/Article.d.ts +11 -0
  460. package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
  461. package/dist/extraction/v1/schemas/Article.js +23 -0
  462. package/dist/extraction/v1/schemas/Article.js.map +1 -0
  463. package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
  464. package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
  465. package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
  466. package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
  467. package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
  468. package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
  469. package/dist/extraction/v1/schemas/EventListing.js +122 -0
  470. package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
  471. package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
  472. package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
  473. package/dist/extraction/v1/schemas/Paper.js +156 -0
  474. package/dist/extraction/v1/schemas/Paper.js.map +1 -0
  475. package/dist/extraction/v1/schemas/Product.d.ts +17 -0
  476. package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
  477. package/dist/extraction/v1/schemas/Product.js +149 -0
  478. package/dist/extraction/v1/schemas/Product.js.map +1 -0
  479. package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
  480. package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
  481. package/dist/extraction/v1/schemas/Recipe.js +160 -0
  482. package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
  483. package/dist/extraction/v1/schemas/index.d.ts +13 -0
  484. package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
  485. package/dist/extraction/v1/schemas/index.js +44 -0
  486. package/dist/extraction/v1/schemas/index.js.map +1 -0
  487. package/dist/extraction/v1/site-extractors.d.ts +5 -0
  488. package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
  489. package/dist/extraction/v1/site-extractors.js +31 -0
  490. package/dist/extraction/v1/site-extractors.js.map +1 -0
  491. package/dist/fetch/action-executor.d.ts +28 -0
  492. package/dist/fetch/action-executor.d.ts.map +1 -0
  493. package/dist/fetch/action-executor.js +88 -0
  494. package/dist/fetch/action-executor.js.map +1 -0
  495. package/dist/fetch/auth.d.ts +2 -1
  496. package/dist/fetch/auth.d.ts.map +1 -1
  497. package/dist/fetch/auth.js +56 -26
  498. package/dist/fetch/auth.js.map +1 -1
  499. package/dist/fetch/browser-pool.d.ts +30 -11
  500. package/dist/fetch/browser-pool.d.ts.map +1 -1
  501. package/dist/fetch/browser-pool.js +303 -127
  502. package/dist/fetch/browser-pool.js.map +1 -1
  503. package/dist/fetch/browser-selector.d.ts +17 -0
  504. package/dist/fetch/browser-selector.d.ts.map +1 -0
  505. package/dist/fetch/browser-selector.js +72 -0
  506. package/dist/fetch/browser-selector.js.map +1 -0
  507. package/dist/fetch/browser-types.d.ts +3 -0
  508. package/dist/fetch/browser-types.d.ts.map +1 -0
  509. package/dist/fetch/browser-types.js +45 -0
  510. package/dist/fetch/browser-types.js.map +1 -0
  511. package/dist/fetch/cdp-client.d.ts +9 -0
  512. package/dist/fetch/cdp-client.d.ts.map +1 -0
  513. package/dist/fetch/cdp-client.js +89 -0
  514. package/dist/fetch/cdp-client.js.map +1 -0
  515. package/dist/fetch/content-check.js +39 -46
  516. package/dist/fetch/content-check.js.map +1 -1
  517. package/dist/fetch/http-client.d.ts +4 -0
  518. package/dist/fetch/http-client.d.ts.map +1 -1
  519. package/dist/fetch/http-client.js +147 -128
  520. package/dist/fetch/http-client.js.map +1 -1
  521. package/dist/fetch/lightpanda.d.ts +28 -0
  522. package/dist/fetch/lightpanda.d.ts.map +1 -0
  523. package/dist/fetch/lightpanda.js +174 -0
  524. package/dist/fetch/lightpanda.js.map +1 -0
  525. package/dist/fetch/playwright-tier.d.ts +19 -0
  526. package/dist/fetch/playwright-tier.d.ts.map +1 -0
  527. package/dist/fetch/playwright-tier.js +76 -0
  528. package/dist/fetch/playwright-tier.js.map +1 -0
  529. package/dist/fetch/router.d.ts +49 -3
  530. package/dist/fetch/router.d.ts.map +1 -1
  531. package/dist/fetch/router.js +185 -81
  532. package/dist/fetch/router.js.map +1 -1
  533. package/dist/index.js +97 -17
  534. package/dist/index.js.map +1 -1
  535. package/dist/instructions.d.ts +31 -0
  536. package/dist/instructions.d.ts.map +1 -0
  537. package/dist/instructions.js +245 -0
  538. package/dist/instructions.js.map +1 -0
  539. package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
  540. package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
  541. package/dist/integrations/cloud/llm/anthropic.js +41 -0
  542. package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
  543. package/dist/integrations/cloud/llm/cache.d.ts +5 -0
  544. package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
  545. package/dist/integrations/cloud/llm/cache.js +49 -0
  546. package/dist/integrations/cloud/llm/cache.js.map +1 -0
  547. package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
  548. package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
  549. package/dist/integrations/cloud/llm/gemini.js +37 -0
  550. package/dist/integrations/cloud/llm/gemini.js.map +1 -0
  551. package/dist/integrations/cloud/llm/groq.d.ts +3 -0
  552. package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
  553. package/dist/integrations/cloud/llm/groq.js +74 -0
  554. package/dist/integrations/cloud/llm/groq.js.map +1 -0
  555. package/dist/integrations/cloud/llm/hash.d.ts +3 -0
  556. package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
  557. package/dist/integrations/cloud/llm/hash.js +26 -0
  558. package/dist/integrations/cloud/llm/hash.js.map +1 -0
  559. package/dist/integrations/cloud/llm/openai.d.ts +3 -0
  560. package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
  561. package/dist/integrations/cloud/llm/openai.js +43 -0
  562. package/dist/integrations/cloud/llm/openai.js.map +1 -0
  563. package/dist/integrations/cloud/llm/select.d.ts +5 -0
  564. package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
  565. package/dist/integrations/cloud/llm/select.js +30 -0
  566. package/dist/integrations/cloud/llm/select.js.map +1 -0
  567. package/dist/integrations/cloud/llm/types.d.ts +24 -0
  568. package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
  569. package/dist/integrations/cloud/llm/types.js +1 -0
  570. package/dist/integrations/cloud/llm/types.js.map +1 -0
  571. package/dist/integrations/cloud/llm/validate.d.ts +6 -0
  572. package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
  573. package/dist/integrations/cloud/llm/validate.js +63 -0
  574. package/dist/integrations/cloud/llm/validate.js.map +1 -0
  575. package/dist/logger.d.ts +4 -1
  576. package/dist/logger.d.ts.map +1 -1
  577. package/dist/logger.js +71 -30
  578. package/dist/logger.js.map +1 -1
  579. package/dist/pdf-parse.d.js +1 -0
  580. package/dist/pdf-parse.d.js.map +1 -0
  581. package/dist/plugins/loader.d.ts +20 -0
  582. package/dist/plugins/loader.d.ts.map +1 -0
  583. package/dist/plugins/loader.js +157 -0
  584. package/dist/plugins/loader.js.map +1 -0
  585. package/dist/plugins/registry.d.ts +26 -0
  586. package/dist/plugins/registry.d.ts.map +1 -0
  587. package/dist/plugins/registry.js +71 -0
  588. package/dist/plugins/registry.js.map +1 -0
  589. package/dist/plugins/validate.d.ts +9 -0
  590. package/dist/plugins/validate.d.ts.map +1 -0
  591. package/dist/plugins/validate.js +79 -0
  592. package/dist/plugins/validate.js.map +1 -0
  593. package/dist/providers/embed-provider.d.ts +11 -0
  594. package/dist/providers/embed-provider.d.ts.map +1 -0
  595. package/dist/providers/embed-provider.js +24 -0
  596. package/dist/providers/embed-provider.js.map +1 -0
  597. package/dist/providers/extract-provider.d.ts +23 -0
  598. package/dist/providers/extract-provider.d.ts.map +1 -0
  599. package/dist/providers/extract-provider.js +25 -0
  600. package/dist/providers/extract-provider.js.map +1 -0
  601. package/dist/providers/rerank-provider.d.ts +16 -0
  602. package/dist/providers/rerank-provider.d.ts.map +1 -0
  603. package/dist/providers/rerank-provider.js +28 -0
  604. package/dist/providers/rerank-provider.js.map +1 -0
  605. package/dist/providers/search-provider.d.ts +25 -0
  606. package/dist/providers/search-provider.d.ts.map +1 -0
  607. package/dist/providers/search-provider.js +44 -0
  608. package/dist/providers/search-provider.js.map +1 -0
  609. package/dist/providers/vector-store.d.ts +27 -0
  610. package/dist/providers/vector-store.d.ts.map +1 -0
  611. package/dist/providers/vector-store.js +27 -0
  612. package/dist/providers/vector-store.js.map +1 -0
  613. package/dist/python-env.d.ts +9 -0
  614. package/dist/python-env.d.ts.map +1 -0
  615. package/dist/python-env.js +13 -0
  616. package/dist/python-env.js.map +1 -0
  617. package/dist/repl/commands/agent.d.ts +5 -0
  618. package/dist/repl/commands/agent.d.ts.map +1 -0
  619. package/dist/repl/commands/agent.js +62 -0
  620. package/dist/repl/commands/agent.js.map +1 -0
  621. package/dist/repl/commands/cache.d.ts +4 -0
  622. package/dist/repl/commands/cache.d.ts.map +1 -0
  623. package/dist/repl/commands/cache.js +43 -0
  624. package/dist/repl/commands/cache.js.map +1 -0
  625. package/dist/repl/commands/crawl.d.ts +7 -0
  626. package/dist/repl/commands/crawl.d.ts.map +1 -0
  627. package/dist/repl/commands/crawl.js +44 -0
  628. package/dist/repl/commands/crawl.js.map +1 -0
  629. package/dist/repl/commands/extract.d.ts +5 -0
  630. package/dist/repl/commands/extract.d.ts.map +1 -0
  631. package/dist/repl/commands/extract.js +47 -0
  632. package/dist/repl/commands/extract.js.map +1 -0
  633. package/dist/repl/commands/fetch.d.ts +5 -0
  634. package/dist/repl/commands/fetch.d.ts.map +1 -0
  635. package/dist/repl/commands/fetch.js +67 -0
  636. package/dist/repl/commands/fetch.js.map +1 -0
  637. package/dist/repl/commands/find-similar.d.ts +5 -0
  638. package/dist/repl/commands/find-similar.d.ts.map +1 -0
  639. package/dist/repl/commands/find-similar.js +74 -0
  640. package/dist/repl/commands/find-similar.js.map +1 -0
  641. package/dist/repl/commands/research.d.ts +5 -0
  642. package/dist/repl/commands/research.d.ts.map +1 -0
  643. package/dist/repl/commands/research.js +65 -0
  644. package/dist/repl/commands/research.js.map +1 -0
  645. package/dist/repl/commands/search.d.ts +5 -0
  646. package/dist/repl/commands/search.d.ts.map +1 -0
  647. package/dist/repl/commands/search.js +74 -0
  648. package/dist/repl/commands/search.js.map +1 -0
  649. package/dist/repl/commands/types.d.ts +9 -0
  650. package/dist/repl/commands/types.d.ts.map +1 -0
  651. package/dist/repl/commands/types.js +1 -0
  652. package/dist/repl/commands/types.js.map +1 -0
  653. package/dist/repl/formatters.d.ts +13 -0
  654. package/dist/repl/formatters.d.ts.map +1 -0
  655. package/dist/repl/formatters.js +283 -0
  656. package/dist/repl/formatters.js.map +1 -0
  657. package/dist/repl/parser.d.ts +9 -0
  658. package/dist/repl/parser.d.ts.map +1 -0
  659. package/dist/repl/parser.js +86 -0
  660. package/dist/repl/parser.js.map +1 -0
  661. package/dist/repl/shell.d.ts +8 -0
  662. package/dist/repl/shell.d.ts.map +1 -0
  663. package/dist/repl/shell.js +184 -0
  664. package/dist/repl/shell.js.map +1 -0
  665. package/dist/research/branch-exploration.d.ts +14 -0
  666. package/dist/research/branch-exploration.d.ts.map +1 -0
  667. package/dist/research/branch-exploration.js +100 -0
  668. package/dist/research/branch-exploration.js.map +1 -0
  669. package/dist/research/brief.d.ts +5 -0
  670. package/dist/research/brief.d.ts.map +1 -0
  671. package/dist/research/brief.js +242 -0
  672. package/dist/research/brief.js.map +1 -0
  673. package/dist/research/citation-graph.d.ts +9 -0
  674. package/dist/research/citation-graph.d.ts.map +1 -0
  675. package/dist/research/citation-graph.js +114 -0
  676. package/dist/research/citation-graph.js.map +1 -0
  677. package/dist/research/decompose.d.ts +14 -0
  678. package/dist/research/decompose.d.ts.map +1 -0
  679. package/dist/research/decompose.js +439 -0
  680. package/dist/research/decompose.js.map +1 -0
  681. package/dist/research/pipeline.d.ts +5 -0
  682. package/dist/research/pipeline.d.ts.map +1 -0
  683. package/dist/research/pipeline.js +269 -0
  684. package/dist/research/pipeline.js.map +1 -0
  685. package/dist/research/synthesis-local.d.ts +16 -0
  686. package/dist/research/synthesis-local.d.ts.map +1 -0
  687. package/dist/research/synthesis-local.js +73 -0
  688. package/dist/research/synthesis-local.js.map +1 -0
  689. package/dist/research/synthesize.d.ts +10 -0
  690. package/dist/research/synthesize.d.ts.map +1 -0
  691. package/dist/research/synthesize.js +137 -0
  692. package/dist/research/synthesize.js.map +1 -0
  693. package/dist/search/answer-synthesis.d.ts +33 -0
  694. package/dist/search/answer-synthesis.d.ts.map +1 -0
  695. package/dist/search/answer-synthesis.js +244 -0
  696. package/dist/search/answer-synthesis.js.map +1 -0
  697. package/dist/search/context-formatter.d.ts +3 -0
  698. package/dist/search/context-formatter.d.ts.map +1 -0
  699. package/dist/search/context-formatter.js +56 -0
  700. package/dist/search/context-formatter.js.map +1 -0
  701. package/dist/search/dedup.d.ts +1 -0
  702. package/dist/search/dedup.d.ts.map +1 -1
  703. package/dist/search/dedup.js +40 -32
  704. package/dist/search/dedup.js.map +1 -1
  705. package/dist/search/engines/arxiv.d.ts +7 -0
  706. package/dist/search/engines/arxiv.d.ts.map +1 -0
  707. package/dist/search/engines/arxiv.js +70 -0
  708. package/dist/search/engines/arxiv.js.map +1 -0
  709. package/dist/search/engines/bing-news.d.ts +7 -0
  710. package/dist/search/engines/bing-news.d.ts.map +1 -0
  711. package/dist/search/engines/bing-news.js +97 -0
  712. package/dist/search/engines/bing-news.js.map +1 -0
  713. package/dist/search/engines/bing.d.ts +1 -0
  714. package/dist/search/engines/bing.d.ts.map +1 -1
  715. package/dist/search/engines/bing.js +100 -44
  716. package/dist/search/engines/bing.js.map +1 -1
  717. package/dist/search/engines/devdocs.d.ts +6 -0
  718. package/dist/search/engines/devdocs.d.ts.map +1 -0
  719. package/dist/search/engines/devdocs.js +56 -0
  720. package/dist/search/engines/devdocs.js.map +1 -0
  721. package/dist/search/engines/duckduckgo.d.ts.map +1 -1
  722. package/dist/search/engines/duckduckgo.js +56 -44
  723. package/dist/search/engines/duckduckgo.js.map +1 -1
  724. package/dist/search/engines/github-code.d.ts +7 -0
  725. package/dist/search/engines/github-code.d.ts.map +1 -0
  726. package/dist/search/engines/github-code.js +55 -0
  727. package/dist/search/engines/github-code.js.map +1 -0
  728. package/dist/search/engines/hn-algolia.d.ts +7 -0
  729. package/dist/search/engines/hn-algolia.d.ts.map +1 -0
  730. package/dist/search/engines/hn-algolia.js +76 -0
  731. package/dist/search/engines/hn-algolia.js.map +1 -0
  732. package/dist/search/engines/lobsters.d.ts +7 -0
  733. package/dist/search/engines/lobsters.d.ts.map +1 -0
  734. package/dist/search/engines/lobsters.js +83 -0
  735. package/dist/search/engines/lobsters.js.map +1 -0
  736. package/dist/search/engines/mdn.d.ts +7 -0
  737. package/dist/search/engines/mdn.d.ts.map +1 -0
  738. package/dist/search/engines/mdn.js +48 -0
  739. package/dist/search/engines/mdn.js.map +1 -0
  740. package/dist/search/engines/semantic-scholar.d.ts +7 -0
  741. package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
  742. package/dist/search/engines/semantic-scholar.js +69 -0
  743. package/dist/search/engines/semantic-scholar.js.map +1 -0
  744. package/dist/search/engines/stackoverflow.d.ts +7 -0
  745. package/dist/search/engines/stackoverflow.d.ts.map +1 -0
  746. package/dist/search/engines/stackoverflow.js +73 -0
  747. package/dist/search/engines/stackoverflow.js.map +1 -0
  748. package/dist/search/engines/startpage.d.ts.map +1 -1
  749. package/dist/search/engines/startpage.js +65 -46
  750. package/dist/search/engines/startpage.js.map +1 -1
  751. package/dist/search/evidence.d.ts +25 -0
  752. package/dist/search/evidence.d.ts.map +1 -0
  753. package/dist/search/evidence.js +220 -0
  754. package/dist/search/evidence.js.map +1 -0
  755. package/dist/search/filters.js +49 -55
  756. package/dist/search/filters.js.map +1 -1
  757. package/dist/search/find-similar/crawl-rank.d.ts +9 -0
  758. package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
  759. package/dist/search/find-similar/crawl-rank.js +272 -0
  760. package/dist/search/find-similar/crawl-rank.js.map +1 -0
  761. package/dist/search/find-similar/mode.d.ts +4 -0
  762. package/dist/search/find-similar/mode.d.ts.map +1 -0
  763. package/dist/search/find-similar/mode.js +12 -0
  764. package/dist/search/find-similar/mode.js.map +1 -0
  765. package/dist/search/find-similar.d.ts +5 -0
  766. package/dist/search/find-similar.d.ts.map +1 -0
  767. package/dist/search/find-similar.js +509 -0
  768. package/dist/search/find-similar.js.map +1 -0
  769. package/dist/search/highlights.d.ts +19 -0
  770. package/dist/search/highlights.d.ts.map +1 -0
  771. package/dist/search/highlights.js +167 -0
  772. package/dist/search/highlights.js.map +1 -0
  773. package/dist/search/language-filter.d.ts +29 -0
  774. package/dist/search/language-filter.d.ts.map +1 -0
  775. package/dist/search/language-filter.js +126 -0
  776. package/dist/search/language-filter.js.map +1 -0
  777. package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
  778. package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
  779. package/dist/search/legacy/searxng-orchestrator.js +501 -0
  780. package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
  781. package/dist/search/legacy/searxng-provider.d.ts +7 -0
  782. package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
  783. package/dist/search/legacy/searxng-provider.js +11 -0
  784. package/dist/search/legacy/searxng-provider.js.map +1 -0
  785. package/dist/search/multi-query.d.ts +25 -0
  786. package/dist/search/multi-query.d.ts.map +1 -0
  787. package/dist/search/multi-query.js +228 -0
  788. package/dist/search/multi-query.js.map +1 -0
  789. package/dist/search/query.js +32 -34
  790. package/dist/search/query.js.map +1 -1
  791. package/dist/search/rerank.d.ts +3 -1
  792. package/dist/search/rerank.d.ts.map +1 -1
  793. package/dist/search/rerank.js +44 -35
  794. package/dist/search/rerank.js.map +1 -1
  795. package/dist/search/reranker/authority-boost.d.ts +3 -0
  796. package/dist/search/reranker/authority-boost.d.ts.map +1 -0
  797. package/dist/search/reranker/authority-boost.js +179 -0
  798. package/dist/search/reranker/authority-boost.js.map +1 -0
  799. package/dist/search/reranker/consensus-boost.d.ts +3 -0
  800. package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
  801. package/dist/search/reranker/consensus-boost.js +27 -0
  802. package/dist/search/reranker/consensus-boost.js.map +1 -0
  803. package/dist/search/reranker/recency-boost.d.ts +3 -0
  804. package/dist/search/reranker/recency-boost.d.ts.map +1 -0
  805. package/dist/search/reranker/recency-boost.js +13 -0
  806. package/dist/search/reranker/recency-boost.js.map +1 -0
  807. package/dist/search/reranker/recency.d.ts +3 -0
  808. package/dist/search/reranker/recency.d.ts.map +1 -0
  809. package/dist/search/reranker/recency.js +23 -0
  810. package/dist/search/reranker/recency.js.map +1 -0
  811. package/dist/search/reranker/transformers-rerank-provider.d.ts +12 -0
  812. package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
  813. package/dist/search/reranker/transformers-rerank-provider.js +78 -0
  814. package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
  815. package/dist/search/rrf.d.ts +17 -0
  816. package/dist/search/rrf.d.ts.map +1 -0
  817. package/dist/search/rrf.js +39 -0
  818. package/dist/search/rrf.js.map +1 -0
  819. package/dist/search/sampling.d.ts +25 -0
  820. package/dist/search/sampling.d.ts.map +1 -0
  821. package/dist/search/sampling.js +52 -0
  822. package/dist/search/sampling.js.map +1 -0
  823. package/dist/search/searxng.d.ts.map +1 -1
  824. package/dist/search/searxng.js +69 -79
  825. package/dist/search/searxng.js.map +1 -1
  826. package/dist/search/tokens.d.ts +3 -0
  827. package/dist/search/tokens.d.ts.map +1 -0
  828. package/dist/search/tokens.js +39 -0
  829. package/dist/search/tokens.js.map +1 -0
  830. package/dist/search/truncate.d.ts +6 -0
  831. package/dist/search/truncate.d.ts.map +1 -0
  832. package/dist/search/truncate.js +26 -0
  833. package/dist/search/truncate.js.map +1 -0
  834. package/dist/search/url-unwrap.d.ts +3 -0
  835. package/dist/search/url-unwrap.d.ts.map +1 -0
  836. package/dist/search/url-unwrap.js +43 -0
  837. package/dist/search/url-unwrap.js.map +1 -0
  838. package/dist/search/v1/context-rank.d.ts +13 -0
  839. package/dist/search/v1/context-rank.d.ts.map +1 -0
  840. package/dist/search/v1/context-rank.js +74 -0
  841. package/dist/search/v1/context-rank.js.map +1 -0
  842. package/dist/search/v1/engine-base.d.ts +27 -0
  843. package/dist/search/v1/engine-base.d.ts.map +1 -0
  844. package/dist/search/v1/engine-base.js +110 -0
  845. package/dist/search/v1/engine-base.js.map +1 -0
  846. package/dist/search/v1/intent-router.d.ts +22 -0
  847. package/dist/search/v1/intent-router.d.ts.map +1 -0
  848. package/dist/search/v1/intent-router.js +138 -0
  849. package/dist/search/v1/intent-router.js.map +1 -0
  850. package/dist/search/v1/orchestrator.d.ts +24 -0
  851. package/dist/search/v1/orchestrator.d.ts.map +1 -0
  852. package/dist/search/v1/orchestrator.js +163 -0
  853. package/dist/search/v1/orchestrator.js.map +1 -0
  854. package/dist/search/v1/recency-boost.d.ts +9 -0
  855. package/dist/search/v1/recency-boost.d.ts.map +1 -0
  856. package/dist/search/v1/recency-boost.js +37 -0
  857. package/dist/search/v1/recency-boost.js.map +1 -0
  858. package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
  859. package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
  860. package/dist/search/v1/recent-cache-dedup.js +85 -0
  861. package/dist/search/v1/recent-cache-dedup.js.map +1 -0
  862. package/dist/search/v1/rss/feed-config.d.ts +21 -0
  863. package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
  864. package/dist/search/v1/rss/feed-config.js +90 -0
  865. package/dist/search/v1/rss/feed-config.js.map +1 -0
  866. package/dist/search/v1/rss/feed-parser.d.ts +14 -0
  867. package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
  868. package/dist/search/v1/rss/feed-parser.js +104 -0
  869. package/dist/search/v1/rss/feed-parser.js.map +1 -0
  870. package/dist/search/v1/rss/feed-poller.d.ts +22 -0
  871. package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
  872. package/dist/search/v1/rss/feed-poller.js +102 -0
  873. package/dist/search/v1/rss/feed-poller.js.map +1 -0
  874. package/dist/search/v1/rss/feed-store.d.ts +30 -0
  875. package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
  876. package/dist/search/v1/rss/feed-store.js +134 -0
  877. package/dist/search/v1/rss/feed-store.js.map +1 -0
  878. package/dist/search/v1/rss/rss-engine.d.ts +6 -0
  879. package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
  880. package/dist/search/v1/rss/rss-engine.js +28 -0
  881. package/dist/search/v1/rss/rss-engine.js.map +1 -0
  882. package/dist/search/v1/v1-provider.d.ts +7 -0
  883. package/dist/search/v1/v1-provider.d.ts.map +1 -0
  884. package/dist/search/v1/v1-provider.js +68 -0
  885. package/dist/search/v1/v1-provider.js.map +1 -0
  886. package/dist/search/v1/verticals/code.d.ts +4 -0
  887. package/dist/search/v1/verticals/code.d.ts.map +1 -0
  888. package/dist/search/v1/verticals/code.js +20 -0
  889. package/dist/search/v1/verticals/code.js.map +1 -0
  890. package/dist/search/v1/verticals/docs.d.ts +4 -0
  891. package/dist/search/v1/verticals/docs.d.ts.map +1 -0
  892. package/dist/search/v1/verticals/docs.js +20 -0
  893. package/dist/search/v1/verticals/docs.js.map +1 -0
  894. package/dist/search/v1/verticals/general.d.ts +4 -0
  895. package/dist/search/v1/verticals/general.d.ts.map +1 -0
  896. package/dist/search/v1/verticals/general.js +22 -0
  897. package/dist/search/v1/verticals/general.js.map +1 -0
  898. package/dist/search/v1/verticals/news.d.ts +10 -0
  899. package/dist/search/v1/verticals/news.d.ts.map +1 -0
  900. package/dist/search/v1/verticals/news.js +52 -0
  901. package/dist/search/v1/verticals/news.js.map +1 -0
  902. package/dist/search/v1/verticals/papers.d.ts +4 -0
  903. package/dist/search/v1/verticals/papers.d.ts.map +1 -0
  904. package/dist/search/v1/verticals/papers.js +23 -0
  905. package/dist/search/v1/verticals/papers.js.map +1 -0
  906. package/dist/search/validator.js +31 -31
  907. package/dist/search/validator.js.map +1 -1
  908. package/dist/searxng/bootstrap.d.ts +30 -0
  909. package/dist/searxng/bootstrap.d.ts.map +1 -1
  910. package/dist/searxng/bootstrap.js +223 -85
  911. package/dist/searxng/bootstrap.js.map +1 -1
  912. package/dist/searxng/docker.d.ts.map +1 -1
  913. package/dist/searxng/docker.js +69 -60
  914. package/dist/searxng/docker.js.map +1 -1
  915. package/dist/searxng/process.d.ts +13 -1
  916. package/dist/searxng/process.d.ts.map +1 -1
  917. package/dist/searxng/process.js +231 -164
  918. package/dist/searxng/process.js.map +1 -1
  919. package/dist/server/backend-status.d.ts +13 -0
  920. package/dist/server/backend-status.d.ts.map +1 -0
  921. package/dist/server/backend-status.js +40 -0
  922. package/dist/server/backend-status.js.map +1 -0
  923. package/dist/server/tool-schemas.d.ts +549 -0
  924. package/dist/server/tool-schemas.d.ts.map +1 -0
  925. package/dist/server/tool-schemas.js +464 -0
  926. package/dist/server/tool-schemas.js.map +1 -0
  927. package/dist/server/warmup-on-start.d.ts +9 -0
  928. package/dist/server/warmup-on-start.d.ts.map +1 -0
  929. package/dist/server/warmup-on-start.js +55 -0
  930. package/dist/server/warmup-on-start.js.map +1 -0
  931. package/dist/server.d.ts +17 -0
  932. package/dist/server.d.ts.map +1 -1
  933. package/dist/server.js +454 -297
  934. package/dist/server.js.map +1 -1
  935. package/dist/tools/agent.d.ts +5 -0
  936. package/dist/tools/agent.d.ts.map +1 -0
  937. package/dist/tools/agent.js +128 -0
  938. package/dist/tools/agent.js.map +1 -0
  939. package/dist/tools/cache.d.ts +2 -1
  940. package/dist/tools/cache.d.ts.map +1 -1
  941. package/dist/tools/cache.js +175 -44
  942. package/dist/tools/cache.js.map +1 -1
  943. package/dist/tools/crawl.d.ts.map +1 -1
  944. package/dist/tools/crawl.js +171 -88
  945. package/dist/tools/crawl.js.map +1 -1
  946. package/dist/tools/extract.d.ts +2 -2
  947. package/dist/tools/extract.d.ts.map +1 -1
  948. package/dist/tools/extract.js +175 -59
  949. package/dist/tools/extract.js.map +1 -1
  950. package/dist/tools/fetch.d.ts +2 -2
  951. package/dist/tools/fetch.d.ts.map +1 -1
  952. package/dist/tools/fetch.js +161 -68
  953. package/dist/tools/fetch.js.map +1 -1
  954. package/dist/tools/find-similar.d.ts +5 -0
  955. package/dist/tools/find-similar.d.ts.map +1 -0
  956. package/dist/tools/find-similar.js +127 -0
  957. package/dist/tools/find-similar.js.map +1 -0
  958. package/dist/tools/research.d.ts +5 -0
  959. package/dist/tools/research.d.ts.map +1 -0
  960. package/dist/tools/research.js +107 -0
  961. package/dist/tools/research.js.map +1 -0
  962. package/dist/tools/search.d.ts +10 -2
  963. package/dist/tools/search.d.ts.map +1 -1
  964. package/dist/tools/search.js +13 -158
  965. package/dist/tools/search.js.map +1 -1
  966. package/dist/types.d.ts +350 -7
  967. package/dist/types.d.ts.map +1 -1
  968. package/dist/types.js +6 -1
  969. package/dist/types.js.map +1 -1
  970. package/dist/util/mode.d.ts +4 -0
  971. package/dist/util/mode.d.ts.map +1 -0
  972. package/dist/util/mode.js +34 -0
  973. package/dist/util/mode.js.map +1 -0
  974. package/package.json +78 -8
  975. package/dist/extraction/trafilatura.d.ts +0 -6
  976. package/dist/extraction/trafilatura.d.ts.map +0 -1
  977. package/dist/extraction/trafilatura.js +0 -105
  978. package/dist/extraction/trafilatura.js.map +0 -1
  979. package/dist/search/flashrank.d.ts +0 -12
  980. package/dist/search/flashrank.d.ts.map +0 -1
  981. package/dist/search/flashrank.js +0 -63
  982. package/dist/search/flashrank.js.map +0 -1
@@ -0,0 +1,7 @@
1
+ import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
2
+ export declare class HnAlgoliaEngine implements SearchEngine {
3
+ name: string;
4
+ search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
5
+ private parseHits;
6
+ }
7
+ //# sourceMappingURL=hn-algolia.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hn-algolia.d.ts","sourceRoot":"","sources":["../../../src/search/engines/hn-algolia.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAoCzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAgB;IAEd,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiC1F,OAAO,CAAC,SAAS;CAmClB"}
@@ -0,0 +1,76 @@
1
+ import { createLogger } from "../../logger.js";
2
+ const log = createLogger("search");
3
+ const SNIPPET_LIMIT = 200;
4
+ function toEpoch(dateStr) {
5
+ const d = new Date(dateStr);
6
+ const t = d.getTime();
7
+ if (isNaN(t)) return 0;
8
+ return Math.floor(t / 1e3);
9
+ }
10
+ function asString(v) {
11
+ return typeof v === "string" && v.length > 0 ? v : void 0;
12
+ }
13
+ function asNumber(v) {
14
+ return typeof v === "number" && isFinite(v) ? v : void 0;
15
+ }
16
+ class HnAlgoliaEngine {
17
+ name = "hn-algolia";
18
+ async search(query, options = {}) {
19
+ const timeoutMs = options.timeoutMs ?? 1e4;
20
+ const maxResults = options.maxResults ?? 10;
21
+ const params = new URLSearchParams({
22
+ query,
23
+ hitsPerPage: String(maxResults),
24
+ tags: "story"
25
+ });
26
+ if (options.fromDate || options.toDate) {
27
+ const filters = [];
28
+ if (options.fromDate) {
29
+ const epoch = toEpoch(options.fromDate);
30
+ if (epoch > 0) filters.push(`created_at_i>${epoch}`);
31
+ }
32
+ if (options.toDate) {
33
+ const epoch = toEpoch(options.toDate);
34
+ if (epoch > 0) filters.push(`created_at_i<${epoch}`);
35
+ }
36
+ if (filters.length) params.set("numericFilters", filters.join(","));
37
+ }
38
+ const url = `https://hn.algolia.com/api/v1/search?${params}`;
39
+ log.debug("hn algolia search", { query });
40
+ const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
41
+ if (!response.ok) throw new Error(`HN Algolia returned ${response.status}`);
42
+ const data = await response.json();
43
+ return this.parseHits(data.hits ?? []);
44
+ }
45
+ parseHits(hits) {
46
+ const results = [];
47
+ const total = hits.length;
48
+ for (let i = 0; i < total; i++) {
49
+ const hit = hits[i];
50
+ const title = asString(hit.title);
51
+ if (!title) continue;
52
+ const objectID = asString(hit.objectID);
53
+ const url = asString(hit.url) ?? (objectID ? `https://news.ycombinator.com/item?id=${objectID}` : void 0);
54
+ if (!url) continue;
55
+ const storyText = asString(hit.story_text);
56
+ const points = asNumber(hit.points) ?? 0;
57
+ const comments = asNumber(hit.num_comments) ?? 0;
58
+ const snippet = storyText ? storyText.slice(0, SNIPPET_LIMIT) : `${points} points \xB7 ${comments} comments`;
59
+ const createdAt = asNumber(hit.created_at_i);
60
+ const published_date = createdAt ? new Date(createdAt * 1e3).toISOString() : void 0;
61
+ results.push({
62
+ title,
63
+ url,
64
+ snippet,
65
+ relevance_score: 1 - i / Math.max(total, 1),
66
+ engine: "hn-algolia",
67
+ ...published_date ? { published_date } : {}
68
+ });
69
+ }
70
+ return results;
71
+ }
72
+ }
73
+ export {
74
+ HnAlgoliaEngine
75
+ };
76
+ //# sourceMappingURL=hn-algolia.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/search/engines/hn-algolia.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface HnHit {\n objectID?: unknown;\n title?: unknown;\n url?: unknown;\n story_text?: unknown;\n points?: unknown;\n num_comments?: unknown;\n created_at_i?: unknown;\n}\n\ninterface HnResponse {\n hits?: HnHit[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction toEpoch(dateStr: string): number {\n const d = new Date(dateStr);\n const t = d.getTime();\n if (isNaN(t)) return 0;\n return Math.floor(t / 1000);\n}\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nexport class HnAlgoliaEngine implements SearchEngine {\n name = 'hn-algolia';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n query,\n hitsPerPage: String(maxResults),\n tags: 'story',\n });\n\n if (options.fromDate || options.toDate) {\n const filters: string[] = [];\n if (options.fromDate) {\n const epoch = toEpoch(options.fromDate);\n if (epoch > 0) filters.push(`created_at_i>${epoch}`);\n }\n if (options.toDate) {\n const epoch = toEpoch(options.toDate);\n if (epoch > 0) filters.push(`created_at_i<${epoch}`);\n }\n if (filters.length) params.set('numericFilters', filters.join(','));\n }\n\n const url = `https://hn.algolia.com/api/v1/search?${params}`;\n log.debug('hn algolia search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`HN Algolia returned ${response.status}`);\n\n const data = (await response.json()) as HnResponse;\n return this.parseHits(data.hits ?? []);\n }\n\n private parseHits(hits: HnHit[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = hits.length;\n\n for (let i = 0; i < total; i++) {\n const hit = hits[i];\n const title = asString(hit.title);\n if (!title) continue;\n\n const objectID = asString(hit.objectID);\n const url = asString(hit.url) ?? (objectID ? `https://news.ycombinator.com/item?id=${objectID}` : undefined);\n if (!url) continue;\n\n const storyText = asString(hit.story_text);\n const points = asNumber(hit.points) ?? 0;\n const comments = asNumber(hit.num_comments) ?? 0;\n const snippet = storyText\n ? storyText.slice(0, SNIPPET_LIMIT)\n : `${points} points · ${comments} comments`;\n\n const createdAt = asNumber(hit.created_at_i);\n const published_date = createdAt ? new Date(createdAt * 1000).toISOString() : undefined;\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'hn-algolia',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAgBjC,MAAM,gBAAgB;AAEtB,SAAS,QAAQ,SAAyB;AACxC,QAAM,IAAI,IAAI,KAAK,OAAO;AAC1B,QAAM,IAAI,EAAE,QAAQ;AACpB,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,KAAK,MAAM,IAAI,GAAI;AAC5B;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEO,MAAM,gBAAwC;AAAA,EACnD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC;AAAA,MACA,aAAa,OAAO,UAAU;AAAA,MAC9B,MAAM;AAAA,IACR,CAAC;AAED,QAAI,QAAQ,YAAY,QAAQ,QAAQ;AACtC,YAAM,UAAoB,CAAC;AAC3B,UAAI,QAAQ,UAAU;AACpB,cAAM,QAAQ,QAAQ,QAAQ,QAAQ;AACtC,YAAI,QAAQ,EAAG,SAAQ,KAAK,gBAAgB,KAAK,EAAE;AAAA,MACrD;AACA,UAAI,QAAQ,QAAQ;AAClB,cAAM,QAAQ,QAAQ,QAAQ,MAAM;AACpC,YAAI,QAAQ,EAAG,SAAQ,KAAK,gBAAgB,KAAK,EAAE;AAAA,MACrD;AACA,UAAI,QAAQ,OAAQ,QAAO,IAAI,kBAAkB,QAAQ,KAAK,GAAG,CAAC;AAAA,IACpE;AAEA,UAAM,MAAM,wCAAwC,MAAM;AAC1D,QAAI,MAAM,qBAAqB,EAAE,MAAM,CAAC;AAExC,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,uBAAuB,SAAS,MAAM,EAAE;AAE1E,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,UAAU,KAAK,QAAQ,CAAC,CAAC;AAAA,EACvC;AAAA,EAEQ,UAAU,MAAkC;AAClD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,UAAI,CAAC,MAAO;AAEZ,YAAM,WAAW,SAAS,IAAI,QAAQ;AACtC,YAAM,MAAM,SAAS,IAAI,GAAG,MAAM,WAAW,wCAAwC,QAAQ,KAAK;AAClG,UAAI,CAAC,IAAK;AAEV,YAAM,YAAY,SAAS,IAAI,UAAU;AACzC,YAAM,SAAS,SAAS,IAAI,MAAM,KAAK;AACvC,YAAM,WAAW,SAAS,IAAI,YAAY,KAAK;AAC/C,YAAM,UAAU,YACZ,UAAU,MAAM,GAAG,aAAa,IAChC,GAAG,MAAM,gBAAa,QAAQ;AAElC,YAAM,YAAY,SAAS,IAAI,YAAY;AAC3C,YAAM,iBAAiB,YAAY,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY,IAAI;AAE9E,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
@@ -0,0 +1,7 @@
1
+ import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
2
+ export declare class LobstersEngine implements SearchEngine {
3
+ name: string;
4
+ search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
5
+ private parseHits;
6
+ }
7
+ //# sourceMappingURL=lobsters.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lobsters.d.ts","sourceRoot":"","sources":["../../../src/search/engines/lobsters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqCzF,qBAAa,cAAe,YAAW,YAAY;IACjD,IAAI,SAAc;IAEZ,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAqB1F,OAAO,CAAC,SAAS;CAsClB"}
@@ -0,0 +1,83 @@
1
+ import { createLogger } from "../../logger.js";
2
+ const log = createLogger("search");
3
+ const SNIPPET_LIMIT = 200;
4
+ function asString(v) {
5
+ return typeof v === "string" && v.length > 0 ? v : void 0;
6
+ }
7
+ function asNumber(v) {
8
+ return typeof v === "number" && isFinite(v) ? v : void 0;
9
+ }
10
+ function extractHits(data) {
11
+ if (Array.isArray(data)) return data;
12
+ if (data && typeof data === "object" && "results" in data) {
13
+ const r = data.results;
14
+ if (Array.isArray(r)) return r;
15
+ }
16
+ return [];
17
+ }
18
+ class LobstersEngine {
19
+ name = "lobsters";
20
+ async search(query, options = {}) {
21
+ const timeoutMs = options.timeoutMs ?? 1e4;
22
+ const maxResults = options.maxResults ?? 10;
23
+ const params = new URLSearchParams({ q: query, what: "stories" });
24
+ const url = `https://lobste.rs/search.json?${params}`;
25
+ log.debug("lobsters search", { query });
26
+ const response = await fetch(url, {
27
+ signal: AbortSignal.timeout(timeoutMs),
28
+ headers: { Accept: "application/json" }
29
+ });
30
+ if (!response.ok) throw new Error(`Lobsters returned ${response.status}`);
31
+ const data = await response.json();
32
+ const hits = extractHits(data);
33
+ const mapped = this.parseHits(hits);
34
+ const filtered = applyDateFilter(mapped, options);
35
+ return filtered.slice(0, maxResults);
36
+ }
37
+ parseHits(hits) {
38
+ const results = [];
39
+ const total = hits.length;
40
+ for (let i = 0; i < total; i++) {
41
+ const hit = hits[i];
42
+ const title = asString(hit.title);
43
+ if (!title) continue;
44
+ const url = asString(hit.url) ?? asString(hit.short_id_url);
45
+ if (!url) continue;
46
+ const description = asString(hit.description);
47
+ const score = asNumber(hit.score) ?? 0;
48
+ const comments = asNumber(hit.comment_count) ?? 0;
49
+ const snippet = description ? description.slice(0, SNIPPET_LIMIT) : `${score} score \xB7 ${comments} comments`;
50
+ const createdAt = asString(hit.created_at);
51
+ let published_date;
52
+ if (createdAt) {
53
+ const d = new Date(createdAt);
54
+ if (!isNaN(d.getTime())) published_date = d.toISOString();
55
+ }
56
+ results.push({
57
+ title,
58
+ url,
59
+ snippet,
60
+ relevance_score: 1 - i / Math.max(total, 1),
61
+ engine: "lobsters",
62
+ ...published_date ? { published_date } : {}
63
+ });
64
+ }
65
+ return results;
66
+ }
67
+ }
68
+ function applyDateFilter(results, options) {
69
+ if (!options.fromDate && !options.toDate) return results;
70
+ const from = options.fromDate ? new Date(options.fromDate).getTime() : -Infinity;
71
+ const to = options.toDate ? new Date(options.toDate).getTime() : Infinity;
72
+ if (isNaN(from) || isNaN(to)) return results;
73
+ return results.filter((r) => {
74
+ if (!r.published_date) return false;
75
+ const t = new Date(r.published_date).getTime();
76
+ if (isNaN(t)) return false;
77
+ return t >= from && t <= to;
78
+ });
79
+ }
80
+ export {
81
+ LobstersEngine
82
+ };
83
+ //# sourceMappingURL=lobsters.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/search/engines/lobsters.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\n// Lobste.rs /search.json sometimes returns an array directly, sometimes a\n// { results: [...] } envelope. Both shapes are handled defensively.\ninterface LobsterHit {\n short_id?: unknown;\n title?: unknown;\n url?: unknown;\n score?: unknown;\n description?: unknown;\n comment_count?: unknown;\n created_at?: unknown;\n short_id_url?: unknown;\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction extractHits(data: unknown): LobsterHit[] {\n if (Array.isArray(data)) return data as LobsterHit[];\n if (data && typeof data === 'object' && 'results' in data) {\n const r = (data as { results?: unknown }).results;\n if (Array.isArray(r)) return r as LobsterHit[];\n }\n return [];\n}\n\nexport class LobstersEngine implements SearchEngine {\n name = 'lobsters';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({ q: query, what: 'stories' });\n const url = `https://lobste.rs/search.json?${params}`;\n log.debug('lobsters search', { query });\n\n const response = await fetch(url, {\n signal: AbortSignal.timeout(timeoutMs),\n headers: { Accept: 'application/json' },\n });\n if (!response.ok) throw new Error(`Lobsters returned ${response.status}`);\n\n const data = (await response.json()) as unknown;\n const hits = extractHits(data);\n const mapped = this.parseHits(hits);\n const filtered = applyDateFilter(mapped, options);\n return filtered.slice(0, maxResults);\n }\n\n private parseHits(hits: LobsterHit[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = hits.length;\n\n for (let i = 0; i < total; i++) {\n const hit = hits[i];\n const title = asString(hit.title);\n if (!title) continue;\n\n const url = asString(hit.url) ?? asString(hit.short_id_url);\n if (!url) continue;\n\n const description = asString(hit.description);\n const score = asNumber(hit.score) ?? 0;\n const comments = asNumber(hit.comment_count) ?? 0;\n const snippet = description\n ? description.slice(0, SNIPPET_LIMIT)\n : `${score} score · ${comments} comments`;\n\n const createdAt = asString(hit.created_at);\n let published_date: string | undefined;\n if (createdAt) {\n const d = new Date(createdAt);\n if (!isNaN(d.getTime())) published_date = d.toISOString();\n }\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'lobsters',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n\nfunction applyDateFilter(results: RawSearchResult[], options: SearchEngineOptions): RawSearchResult[] {\n if (!options.fromDate && !options.toDate) return results;\n const from = options.fromDate ? new Date(options.fromDate).getTime() : -Infinity;\n const to = options.toDate ? new Date(options.toDate).getTime() : Infinity;\n if (isNaN(from) || isNaN(to)) return results;\n return results.filter((r) => {\n if (!r.published_date) return false;\n const t = new Date(r.published_date).getTime();\n if (isNaN(t)) return false;\n return t >= from && t <= to;\n });\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAejC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,YAAY,MAA6B;AAChD,MAAI,MAAM,QAAQ,IAAI,EAAG,QAAO;AAChC,MAAI,QAAQ,OAAO,SAAS,YAAY,aAAa,MAAM;AACzD,UAAM,IAAK,KAA+B;AAC1C,QAAI,MAAM,QAAQ,CAAC,EAAG,QAAO;AAAA,EAC/B;AACA,SAAO,CAAC;AACV;AAEO,MAAM,eAAuC;AAAA,EAClD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB,EAAE,GAAG,OAAO,MAAM,UAAU,CAAC;AAChE,UAAM,MAAM,iCAAiC,MAAM;AACnD,QAAI,MAAM,mBAAmB,EAAE,MAAM,CAAC;AAEtC,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,QAAQ,mBAAmB;AAAA,IACxC,CAAC;AACD,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,qBAAqB,SAAS,MAAM,EAAE;AAExE,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,SAAS,KAAK,UAAU,IAAI;AAClC,UAAM,WAAW,gBAAgB,QAAQ,OAAO;AAChD,WAAO,SAAS,MAAM,GAAG,UAAU;AAAA,EACrC;AAAA,EAEQ,UAAU,MAAuC;AACvD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,UAAI,CAAC,MAAO;AAEZ,YAAM,MAAM,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,YAAY;AAC1D,UAAI,CAAC,IAAK;AAEV,YAAM,cAAc,SAAS,IAAI,WAAW;AAC5C,YAAM,QAAQ,SAAS,IAAI,KAAK,KAAK;AACrC,YAAM,WAAW,SAAS,IAAI,aAAa,KAAK;AAChD,YAAM,UAAU,cACZ,YAAY,MAAM,GAAG,aAAa,IAClC,GAAG,KAAK,eAAY,QAAQ;AAEhC,YAAM,YAAY,SAAS,IAAI,UAAU;AACzC,UAAI;AACJ,UAAI,WAAW;AACb,cAAM,IAAI,IAAI,KAAK,SAAS;AAC5B,YAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAG,kBAAiB,EAAE,YAAY;AAAA,MAC1D;AAEA,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;AAEA,SAAS,gBAAgB,SAA4B,SAAiD;AACpG,MAAI,CAAC,QAAQ,YAAY,CAAC,QAAQ,OAAQ,QAAO;AACjD,QAAM,OAAO,QAAQ,WAAW,IAAI,KAAK,QAAQ,QAAQ,EAAE,QAAQ,IAAI;AACvE,QAAM,KAAK,QAAQ,SAAS,IAAI,KAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI;AACjE,MAAI,MAAM,IAAI,KAAK,MAAM,EAAE,EAAG,QAAO;AACrC,SAAO,QAAQ,OAAO,CAAC,MAAM;AAC3B,QAAI,CAAC,EAAE,eAAgB,QAAO;AAC9B,UAAM,IAAI,IAAI,KAAK,EAAE,cAAc,EAAE,QAAQ;AAC7C,QAAI,MAAM,CAAC,EAAG,QAAO;AACrB,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B,CAAC;AACH;","names":[]}
@@ -0,0 +1,7 @@
1
+ import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
2
+ export declare class MdnEngine implements SearchEngine {
3
+ name: string;
4
+ search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
5
+ private parseDocs;
6
+ }
7
+ //# sourceMappingURL=mdn.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mdn.d.ts","sourceRoot":"","sources":["../../../src/search/engines/mdn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqBzF,qBAAa,SAAU,YAAW,YAAY;IAC5C,IAAI,SAAS;IAEP,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAmB1F,OAAO,CAAC,SAAS;CAwBlB"}
@@ -0,0 +1,48 @@
1
+ import { createLogger } from "../../logger.js";
2
+ const log = createLogger("search");
3
+ const MDN_HOST = "https://developer.mozilla.org";
4
+ function asString(v) {
5
+ return typeof v === "string" && v.length > 0 ? v : void 0;
6
+ }
7
+ class MdnEngine {
8
+ name = "mdn";
9
+ async search(query, options = {}) {
10
+ const timeoutMs = options.timeoutMs ?? 1e4;
11
+ const maxResults = options.maxResults ?? 10;
12
+ const params = new URLSearchParams({
13
+ q: query,
14
+ locale: "en-US",
15
+ size: String(maxResults)
16
+ });
17
+ const url = `${MDN_HOST}/api/v1/search?${params}`;
18
+ log.debug("mdn search", { query });
19
+ const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
20
+ if (!response.ok) throw new Error(`MDN returned ${response.status}`);
21
+ const data = await response.json();
22
+ return this.parseDocs(data.documents ?? []);
23
+ }
24
+ parseDocs(docs) {
25
+ const results = [];
26
+ const total = docs.length;
27
+ for (let i = 0; i < total; i++) {
28
+ const doc = docs[i];
29
+ const title = asString(doc.title);
30
+ const path = asString(doc.mdn_url);
31
+ if (!title || !path) continue;
32
+ const summary = asString(doc.summary) ?? "";
33
+ const url = path.startsWith("http") ? path : `${MDN_HOST}${path}`;
34
+ results.push({
35
+ title,
36
+ url,
37
+ snippet: summary,
38
+ relevance_score: 1 - i / Math.max(total, 1),
39
+ engine: "mdn"
40
+ });
41
+ }
42
+ return results;
43
+ }
44
+ }
45
+ export {
46
+ MdnEngine
47
+ };
48
+ //# sourceMappingURL=mdn.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/search/engines/mdn.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface MdnDoc {\n mdn_url?: unknown;\n title?: unknown;\n summary?: unknown;\n}\n\ninterface MdnResponse {\n documents?: MdnDoc[];\n}\n\nconst MDN_HOST = 'https://developer.mozilla.org';\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nexport class MdnEngine implements SearchEngine {\n name = 'mdn';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n q: query,\n locale: 'en-US',\n size: String(maxResults),\n });\n const url = `${MDN_HOST}/api/v1/search?${params}`;\n log.debug('mdn search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`MDN returned ${response.status}`);\n\n const data = (await response.json()) as MdnResponse;\n return this.parseDocs(data.documents ?? []);\n }\n\n private parseDocs(docs: MdnDoc[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = docs.length;\n\n for (let i = 0; i < total; i++) {\n const doc = docs[i];\n const title = asString(doc.title);\n const path = asString(doc.mdn_url);\n if (!title || !path) continue;\n\n const summary = asString(doc.summary) ?? '';\n const url = path.startsWith('http') ? path : `${MDN_HOST}${path}`;\n\n results.push({\n title,\n url,\n snippet: summary,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'mdn',\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAYjC,MAAM,WAAW;AAEjB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEO,MAAM,UAAkC;AAAA,EAC7C,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,GAAG;AAAA,MACH,QAAQ;AAAA,MACR,MAAM,OAAO,UAAU;AAAA,IACzB,CAAC;AACD,UAAM,MAAM,GAAG,QAAQ,kBAAkB,MAAM;AAC/C,QAAI,MAAM,cAAc,EAAE,MAAM,CAAC;AAEjC,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,gBAAgB,SAAS,MAAM,EAAE;AAEnE,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,UAAU,KAAK,aAAa,CAAC,CAAC;AAAA,EAC5C;AAAA,EAEQ,UAAU,MAAmC;AACnD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,YAAM,OAAO,SAAS,IAAI,OAAO;AACjC,UAAI,CAAC,SAAS,CAAC,KAAM;AAErB,YAAM,UAAU,SAAS,IAAI,OAAO,KAAK;AACzC,YAAM,MAAM,KAAK,WAAW,MAAM,IAAI,OAAO,GAAG,QAAQ,GAAG,IAAI;AAE/D,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA,SAAS;AAAA,QACT,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
@@ -0,0 +1,7 @@
1
+ import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
2
+ export declare class SemanticScholarEngine implements SearchEngine {
3
+ name: string;
4
+ search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
5
+ private parsePapers;
6
+ }
7
+ //# sourceMappingURL=semantic-scholar.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"semantic-scholar.d.ts","sourceRoot":"","sources":["../../../src/search/engines/semantic-scholar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAsCzF,qBAAa,qBAAsB,YAAW,YAAY;IACxD,IAAI,SAAsB;IAEpB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA4B1F,OAAO,CAAC,WAAW;CAgCpB"}
@@ -0,0 +1,69 @@
1
+ import { createLogger } from "../../logger.js";
2
+ const log = createLogger("search");
3
+ const SNIPPET_LIMIT = 200;
4
+ function asString(v) {
5
+ return typeof v === "string" && v.length > 0 ? v : void 0;
6
+ }
7
+ function asNumber(v) {
8
+ return typeof v === "number" && isFinite(v) ? v : void 0;
9
+ }
10
+ function yearOf(iso) {
11
+ const t = new Date(iso).getTime();
12
+ if (isNaN(t)) return void 0;
13
+ return new Date(t).getUTCFullYear();
14
+ }
15
+ class SemanticScholarEngine {
16
+ name = "semantic-scholar";
17
+ async search(query, options = {}) {
18
+ const timeoutMs = options.timeoutMs ?? 1e4;
19
+ const maxResults = options.maxResults ?? 10;
20
+ const params = new URLSearchParams({
21
+ query,
22
+ limit: String(maxResults),
23
+ fields: "title,abstract,year,url,authors,externalIds,openAccessPdf"
24
+ });
25
+ if (options.fromDate || options.toDate) {
26
+ const fromY = options.fromDate ? yearOf(options.fromDate) : void 0;
27
+ const toY = options.toDate ? yearOf(options.toDate) : void 0;
28
+ if (fromY !== void 0 || toY !== void 0) {
29
+ params.set("year", `${fromY ?? ""}-${toY ?? ""}`);
30
+ }
31
+ }
32
+ const url = `https://api.semanticscholar.org/graph/v1/paper/search?${params}`;
33
+ log.debug("semantic-scholar search", { query });
34
+ const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
35
+ if (!response.ok) throw new Error(`Semantic Scholar returned ${response.status}`);
36
+ const data = await response.json();
37
+ return this.parsePapers(data.data ?? []);
38
+ }
39
+ parsePapers(papers) {
40
+ const results = [];
41
+ const total = papers.length;
42
+ for (let i = 0; i < total; i++) {
43
+ const p = papers[i];
44
+ const title = asString(p.title);
45
+ if (!title) continue;
46
+ const pdfUrl = asString(p.openAccessPdf?.url);
47
+ const baseUrl = asString(p.url);
48
+ const url = pdfUrl ?? baseUrl;
49
+ if (!url) continue;
50
+ const abstract = asString(p.abstract) ?? "";
51
+ const snippet = abstract.slice(0, SNIPPET_LIMIT);
52
+ const year = asNumber(p.year);
53
+ const published_date = year ? `${year}-01-01T00:00:00.000Z` : void 0;
54
+ results.push({
55
+ title,
56
+ url,
57
+ snippet,
58
+ relevance_score: 1 - i / Math.max(total, 1),
59
+ engine: "semantic-scholar",
60
+ ...published_date ? { published_date } : {}
61
+ });
62
+ }
63
+ return results;
64
+ }
65
+ }
66
+ export {
67
+ SemanticScholarEngine
68
+ };
69
+ //# sourceMappingURL=semantic-scholar.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/search/engines/semantic-scholar.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface OpenAccessPdf {\n url?: unknown;\n}\n\ninterface S2Paper {\n paperId?: unknown;\n title?: unknown;\n abstract?: unknown;\n year?: unknown;\n url?: unknown;\n openAccessPdf?: OpenAccessPdf;\n}\n\ninterface S2Response {\n data?: S2Paper[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction yearOf(iso: string): number | undefined {\n const t = new Date(iso).getTime();\n if (isNaN(t)) return undefined;\n return new Date(t).getUTCFullYear();\n}\n\nexport class SemanticScholarEngine implements SearchEngine {\n name = 'semantic-scholar';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n query,\n limit: String(maxResults),\n fields: 'title,abstract,year,url,authors,externalIds,openAccessPdf',\n });\n\n if (options.fromDate || options.toDate) {\n const fromY = options.fromDate ? yearOf(options.fromDate) : undefined;\n const toY = options.toDate ? yearOf(options.toDate) : undefined;\n if (fromY !== undefined || toY !== undefined) {\n params.set('year', `${fromY ?? ''}-${toY ?? ''}`);\n }\n }\n\n const url = `https://api.semanticscholar.org/graph/v1/paper/search?${params}`;\n log.debug('semantic-scholar search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`Semantic Scholar returned ${response.status}`);\n\n const data = (await response.json()) as S2Response;\n return this.parsePapers(data.data ?? []);\n }\n\n private parsePapers(papers: S2Paper[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = papers.length;\n\n for (let i = 0; i < total; i++) {\n const p = papers[i];\n const title = asString(p.title);\n if (!title) continue;\n\n const pdfUrl = asString(p.openAccessPdf?.url);\n const baseUrl = asString(p.url);\n const url = pdfUrl ?? baseUrl;\n if (!url) continue;\n\n const abstract = asString(p.abstract) ?? '';\n const snippet = abstract.slice(0, SNIPPET_LIMIT);\n\n const year = asNumber(p.year);\n const published_date = year ? `${year}-01-01T00:00:00.000Z` : undefined;\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'semantic-scholar',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAmBjC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,OAAO,KAAiC;AAC/C,QAAM,IAAI,IAAI,KAAK,GAAG,EAAE,QAAQ;AAChC,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,IAAI,KAAK,CAAC,EAAE,eAAe;AACpC;AAEO,MAAM,sBAA8C;AAAA,EACzD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC;AAAA,MACA,OAAO,OAAO,UAAU;AAAA,MACxB,QAAQ;AAAA,IACV,CAAC;AAED,QAAI,QAAQ,YAAY,QAAQ,QAAQ;AACtC,YAAM,QAAQ,QAAQ,WAAW,OAAO,QAAQ,QAAQ,IAAI;AAC5D,YAAM,MAAM,QAAQ,SAAS,OAAO,QAAQ,MAAM,IAAI;AACtD,UAAI,UAAU,UAAa,QAAQ,QAAW;AAC5C,eAAO,IAAI,QAAQ,GAAG,SAAS,EAAE,IAAI,OAAO,EAAE,EAAE;AAAA,MAClD;AAAA,IACF;AAEA,UAAM,MAAM,yDAAyD,MAAM;AAC3E,QAAI,MAAM,2BAA2B,EAAE,MAAM,CAAC;AAE9C,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,6BAA6B,SAAS,MAAM,EAAE;AAEhF,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,YAAY,KAAK,QAAQ,CAAC,CAAC;AAAA,EACzC;AAAA,EAEQ,YAAY,QAAsC;AACxD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,OAAO;AAErB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,IAAI,OAAO,CAAC;AAClB,YAAM,QAAQ,SAAS,EAAE,KAAK;AAC9B,UAAI,CAAC,MAAO;AAEZ,YAAM,SAAS,SAAS,EAAE,eAAe,GAAG;AAC5C,YAAM,UAAU,SAAS,EAAE,GAAG;AAC9B,YAAM,MAAM,UAAU;AACtB,UAAI,CAAC,IAAK;AAEV,YAAM,WAAW,SAAS,EAAE,QAAQ,KAAK;AACzC,YAAM,UAAU,SAAS,MAAM,GAAG,aAAa;AAE/C,YAAM,OAAO,SAAS,EAAE,IAAI;AAC5B,YAAM,iBAAiB,OAAO,GAAG,IAAI,yBAAyB;AAE9D,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
@@ -0,0 +1,7 @@
1
+ import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
2
+ export declare class StackOverflowEngine implements SearchEngine {
3
+ name: string;
4
+ search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
5
+ private parseItems;
6
+ }
7
+ //# sourceMappingURL=stackoverflow.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"stackoverflow.d.ts","sourceRoot":"","sources":["../../../src/search/engines/stackoverflow.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAuCzF,qBAAa,mBAAoB,YAAW,YAAY;IACtD,IAAI,SAAmB;IAEjB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgC1F,OAAO,CAAC,UAAU;CA4BnB"}
@@ -0,0 +1,73 @@
1
+ import { createLogger } from "../../logger.js";
2
+ const log = createLogger("search");
3
+ const SNIPPET_LIMIT = 200;
4
+ function asString(v) {
5
+ return typeof v === "string" && v.length > 0 ? v : void 0;
6
+ }
7
+ function asNumber(v) {
8
+ return typeof v === "number" && isFinite(v) ? v : void 0;
9
+ }
10
+ function stripHtml(html) {
11
+ return html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
12
+ }
13
+ function toEpochSeconds(iso) {
14
+ const t = new Date(iso).getTime();
15
+ if (isNaN(t)) return void 0;
16
+ return Math.floor(t / 1e3);
17
+ }
18
+ class StackOverflowEngine {
19
+ name = "stackoverflow";
20
+ async search(query, options = {}) {
21
+ const timeoutMs = options.timeoutMs ?? 1e4;
22
+ const maxResults = options.maxResults ?? 10;
23
+ const params = new URLSearchParams({
24
+ order: "desc",
25
+ sort: "relevance",
26
+ q: query,
27
+ site: "stackoverflow",
28
+ filter: "withbody",
29
+ pagesize: String(maxResults)
30
+ });
31
+ if (options.fromDate) {
32
+ const epoch = toEpochSeconds(options.fromDate);
33
+ if (epoch !== void 0) params.set("fromdate", String(epoch));
34
+ }
35
+ if (options.toDate) {
36
+ const epoch = toEpochSeconds(options.toDate);
37
+ if (epoch !== void 0) params.set("todate", String(epoch));
38
+ }
39
+ const url = `https://api.stackexchange.com/2.3/search/advanced?${params}`;
40
+ log.debug("stackoverflow search", { query });
41
+ const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
42
+ if (!response.ok) throw new Error(`StackOverflow returned ${response.status}`);
43
+ const data = await response.json();
44
+ return this.parseItems(data.items ?? []);
45
+ }
46
+ parseItems(items) {
47
+ const results = [];
48
+ const total = items.length;
49
+ for (let i = 0; i < total; i++) {
50
+ const item = items[i];
51
+ const title = asString(item.title);
52
+ const link = asString(item.link);
53
+ if (!title || !link) continue;
54
+ const bodyHtml = asString(item.body) ?? "";
55
+ const snippet = stripHtml(bodyHtml).slice(0, SNIPPET_LIMIT);
56
+ const createdAt = asNumber(item.creation_date);
57
+ const published_date = createdAt ? new Date(createdAt * 1e3).toISOString() : void 0;
58
+ results.push({
59
+ title,
60
+ url: link,
61
+ snippet,
62
+ relevance_score: 1 - i / Math.max(total, 1),
63
+ engine: "stackoverflow",
64
+ ...published_date ? { published_date } : {}
65
+ });
66
+ }
67
+ return results;
68
+ }
69
+ }
70
+ export {
71
+ StackOverflowEngine
72
+ };
73
+ //# sourceMappingURL=stackoverflow.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/search/engines/stackoverflow.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface SoItem {\n title?: unknown;\n link?: unknown;\n body?: unknown;\n creation_date?: unknown;\n}\n\ninterface SoResponse {\n items?: SoItem[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction stripHtml(html: string): string {\n return html\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\nfunction toEpochSeconds(iso: string): number | undefined {\n const t = new Date(iso).getTime();\n if (isNaN(t)) return undefined;\n return Math.floor(t / 1000);\n}\n\nexport class StackOverflowEngine implements SearchEngine {\n name = 'stackoverflow';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n order: 'desc',\n sort: 'relevance',\n q: query,\n site: 'stackoverflow',\n filter: 'withbody',\n pagesize: String(maxResults),\n });\n\n if (options.fromDate) {\n const epoch = toEpochSeconds(options.fromDate);\n if (epoch !== undefined) params.set('fromdate', String(epoch));\n }\n if (options.toDate) {\n const epoch = toEpochSeconds(options.toDate);\n if (epoch !== undefined) params.set('todate', String(epoch));\n }\n\n const url = `https://api.stackexchange.com/2.3/search/advanced?${params}`;\n log.debug('stackoverflow search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`StackOverflow returned ${response.status}`);\n\n const data = (await response.json()) as SoResponse;\n return this.parseItems(data.items ?? []);\n }\n\n private parseItems(items: SoItem[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = items.length;\n\n for (let i = 0; i < total; i++) {\n const item = items[i];\n const title = asString(item.title);\n const link = asString(item.link);\n if (!title || !link) continue;\n\n const bodyHtml = asString(item.body) ?? '';\n const snippet = stripHtml(bodyHtml).slice(0, SNIPPET_LIMIT);\n\n const createdAt = asNumber(item.creation_date);\n const published_date = createdAt ? new Date(createdAt * 1000).toISOString() : undefined;\n\n results.push({\n title,\n url: link,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'stackoverflow',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAajC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,UAAU,MAAsB;AACvC,SAAO,KACJ,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEA,SAAS,eAAe,KAAiC;AACvD,QAAM,IAAI,IAAI,KAAK,GAAG,EAAE,QAAQ;AAChC,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,KAAK,MAAM,IAAI,GAAI;AAC5B;AAEO,MAAM,oBAA4C;AAAA,EACvD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,OAAO;AAAA,MACP,MAAM;AAAA,MACN,GAAG;AAAA,MACH,MAAM;AAAA,MACN,QAAQ;AAAA,MACR,UAAU,OAAO,UAAU;AAAA,IAC7B,CAAC;AAED,QAAI,QAAQ,UAAU;AACpB,YAAM,QAAQ,eAAe,QAAQ,QAAQ;AAC7C,UAAI,UAAU,OAAW,QAAO,IAAI,YAAY,OAAO,KAAK,CAAC;AAAA,IAC/D;AACA,QAAI,QAAQ,QAAQ;AAClB,YAAM,QAAQ,eAAe,QAAQ,MAAM;AAC3C,UAAI,UAAU,OAAW,QAAO,IAAI,UAAU,OAAO,KAAK,CAAC;AAAA,IAC7D;AAEA,UAAM,MAAM,qDAAqD,MAAM;AACvE,QAAI,MAAM,wBAAwB,EAAE,MAAM,CAAC;AAE3C,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,0BAA0B,SAAS,MAAM,EAAE;AAE7E,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,WAAW,KAAK,SAAS,CAAC,CAAC;AAAA,EACzC;AAAA,EAEQ,WAAW,OAAoC;AACrD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,MAAM;AAEpB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,OAAO,MAAM,CAAC;AACpB,YAAM,QAAQ,SAAS,KAAK,KAAK;AACjC,YAAM,OAAO,SAAS,KAAK,IAAI;AAC/B,UAAI,CAAC,SAAS,CAAC,KAAM;AAErB,YAAM,WAAW,SAAS,KAAK,IAAI,KAAK;AACxC,YAAM,UAAU,UAAU,QAAQ,EAAE,MAAM,GAAG,aAAa;AAE1D,YAAM,YAAY,SAAS,KAAK,aAAa;AAC7C,YAAM,iBAAiB,YAAY,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY,IAAI;AAE9E,cAAQ,KAAK;AAAA,QACX;AAAA,QACA,KAAK;AAAA,QACL;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
@@ -1 +1 @@
1
- {"version":3,"file":"startpage.d.ts","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAKzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAe;IAEb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CA4BlE"}
1
+ {"version":3,"file":"startpage.d.ts","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqBzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAe;IAEb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CAkClE"}
@@ -1,50 +1,69 @@
1
- import { parseHTML } from 'linkedom';
2
- import { createLogger } from '../../logger.js';
3
- const log = createLogger('search');
4
- export class StartpageEngine {
5
- name = 'startpage';
6
- async search(query, options = {}) {
7
- const timeoutMs = options.timeoutMs ?? 10000;
8
- const maxResults = options.maxResults ?? 10;
9
- const params = new URLSearchParams({ query, cat: 'web' });
10
- const url = `https://www.startpage.com/sp/search?${params}`;
11
- log.debug('scraping startpage', { query });
12
- const response = await fetch(url, {
13
- method: 'POST',
14
- headers: {
15
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
16
- 'Content-Type': 'application/x-www-form-urlencoded',
17
- },
18
- body: params.toString(),
19
- signal: AbortSignal.timeout(timeoutMs),
1
+ import { parseHTML } from "linkedom";
2
+ import { createLogger } from "../../logger.js";
3
+ const log = createLogger("search");
4
+ const DATE_SNIPPET_PATTERN = /^(\w{3}\s+\d{1,2},\s+\d{4}|\d{4}-\d{2}-\d{2})\s*[·—–-]/;
5
+ function parseDateFromEl(el) {
6
+ if (!el?.textContent) return void 0;
7
+ const text = el.textContent.trim();
8
+ const d = new Date(text);
9
+ return isNaN(d.getTime()) ? void 0 : d.toISOString();
10
+ }
11
+ function parseDateFromSnippet(snippet) {
12
+ const match = snippet.trim().match(DATE_SNIPPET_PATTERN);
13
+ if (!match) return void 0;
14
+ const d = new Date(match[1]);
15
+ return isNaN(d.getTime()) ? void 0 : d.toISOString();
16
+ }
17
+ class StartpageEngine {
18
+ name = "startpage";
19
+ async search(query, options = {}) {
20
+ const timeoutMs = options.timeoutMs ?? 1e4;
21
+ const maxResults = options.maxResults ?? 10;
22
+ const params = new URLSearchParams({ query, cat: "web" });
23
+ const url = `https://www.startpage.com/sp/search?${params}`;
24
+ log.debug("scraping startpage", { query });
25
+ const response = await fetch(url, {
26
+ method: "POST",
27
+ headers: {
28
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
29
+ "Content-Type": "application/x-www-form-urlencoded"
30
+ },
31
+ body: params.toString(),
32
+ signal: AbortSignal.timeout(timeoutMs)
33
+ });
34
+ if (!response.ok) throw new Error(`Startpage returned ${response.status}`);
35
+ const html = await response.text();
36
+ return this.parseResults(html, maxResults);
37
+ }
38
+ parseResults(html, maxResults) {
39
+ const { document } = parseHTML(html);
40
+ const results = [];
41
+ const items = document.querySelectorAll(".w-gl__result");
42
+ const total = Math.min(items.length, maxResults);
43
+ for (let i = 0; i < total; i++) {
44
+ const item = items[i];
45
+ const link = item.querySelector(".w-gl__result-title, a.w-gl__result-url");
46
+ const snippetEl = item.querySelector(".w-gl__description");
47
+ const href = link?.getAttribute("href");
48
+ const title = link?.textContent?.trim();
49
+ if (href && title) {
50
+ const dateEl = item.querySelector(".w-gl__result-date, time");
51
+ const snippetText = snippetEl?.textContent?.trim() ?? "";
52
+ const published_date = parseDateFromEl(dateEl) ?? parseDateFromSnippet(snippetText);
53
+ results.push({
54
+ title,
55
+ url: href,
56
+ snippet: snippetText,
57
+ relevance_score: 1 - i / Math.max(items.length, 1),
58
+ engine: "startpage",
59
+ ...published_date ? { published_date } : {}
20
60
  });
21
- if (!response.ok)
22
- throw new Error(`Startpage returned ${response.status}`);
23
- const html = await response.text();
24
- return this.parseResults(html, maxResults);
25
- }
26
- parseResults(html, maxResults) {
27
- const { document } = parseHTML(html);
28
- const results = [];
29
- const items = document.querySelectorAll('.w-gl__result');
30
- const total = Math.min(items.length, maxResults);
31
- for (let i = 0; i < total; i++) {
32
- const item = items[i];
33
- const link = item.querySelector('.w-gl__result-title, a.w-gl__result-url');
34
- const snippetEl = item.querySelector('.w-gl__description');
35
- const href = link?.getAttribute('href');
36
- const title = link?.textContent?.trim();
37
- if (href && title) {
38
- results.push({
39
- title,
40
- url: href,
41
- snippet: snippetEl?.textContent?.trim() ?? '',
42
- relevance_score: 1 - i / Math.max(items.length, 1),
43
- engine: 'startpage',
44
- });
45
- }
46
- }
47
- return results;
61
+ }
48
62
  }
63
+ return results;
64
+ }
49
65
  }
66
+ export {
67
+ StartpageEngine
68
+ };
50
69
  //# sourceMappingURL=startpage.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"startpage.js","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,OAAO,eAAe;IAC1B,IAAI,GAAG,WAAW,CAAC;IAEnB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA+B,EAAE;QAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,uCAAuC,MAAM,EAAE,CAAC;QAE5D,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,YAAY,EAAE,uHAAuH;gBACrI,cAAc,EAAE,mCAAmC;aACpD;YACD,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;YACvB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3E,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,UAAkB;QAC3C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,yCAAyC,CAAC,CAAC;YAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;YAE3D,MAAM,IAAI,GAAG,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAExC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,GAAG,EAAE,IAAI;oBACT,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;oBAC7C,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClD,MAAM,EAAE,WAAW;iBACpB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
1
+ {"version":3,"sources":["../../../src/search/engines/startpage.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\nconst DATE_SNIPPET_PATTERN = /^(\\w{3}\\s+\\d{1,2},\\s+\\d{4}|\\d{4}-\\d{2}-\\d{2})\\s*[·—–-]/;\n\nfunction parseDateFromEl(el: { textContent?: string | null } | null): string | undefined {\n if (!el?.textContent) return undefined;\n const text = el.textContent.trim();\n const d = new Date(text);\n return isNaN(d.getTime()) ? undefined : d.toISOString();\n}\n\nfunction parseDateFromSnippet(snippet: string): string | undefined {\n const match = snippet.trim().match(DATE_SNIPPET_PATTERN);\n if (!match) return undefined;\n const d = new Date(match[1]);\n return isNaN(d.getTime()) ? undefined : d.toISOString();\n}\n\nexport class StartpageEngine implements SearchEngine {\n name = 'startpage';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({ query, cat: 'web' });\n const url = `https://www.startpage.com/sp/search?${params}`;\n\n log.debug('scraping startpage', { query });\n\n const response = await fetch(url, {\n method: 'POST',\n headers: {\n 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',\n 'Content-Type': 'application/x-www-form-urlencoded',\n },\n body: params.toString(),\n signal: AbortSignal.timeout(timeoutMs),\n });\n\n if (!response.ok) throw new Error(`Startpage returned ${response.status}`);\n\n const html = await response.text();\n return this.parseResults(html, maxResults);\n }\n\n parseResults(html: string, maxResults: number): RawSearchResult[] {\n const { document } = parseHTML(html);\n const results: RawSearchResult[] = [];\n\n const items = document.querySelectorAll('.w-gl__result');\n const total = Math.min(items.length, maxResults);\n\n for (let i = 0; i < total; i++) {\n const item = items[i];\n const link = item.querySelector('.w-gl__result-title, a.w-gl__result-url');\n const snippetEl = item.querySelector('.w-gl__description');\n\n const href = link?.getAttribute('href');\n const title = link?.textContent?.trim();\n\n if (href && title) {\n // Startpage sometimes shows dates in a dedicated element or snippet prefix\n const dateEl = item.querySelector('.w-gl__result-date, time');\n const snippetText = snippetEl?.textContent?.trim() ?? '';\n const published_date = parseDateFromEl(dateEl) ?? parseDateFromSnippet(snippetText);\n\n results.push({\n title,\n url: href,\n snippet: snippetText,\n relevance_score: 1 - i / Math.max(items.length, 1),\n engine: 'startpage',\n ...(published_date ? { published_date } : {}),\n });\n }\n }\n\n return results;\n }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAE1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,uBAAuB;AAE7B,SAAS,gBAAgB,IAAgE;AACvF,MAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,QAAM,OAAO,GAAG,YAAY,KAAK;AACjC,QAAM,IAAI,IAAI,KAAK,IAAI;AACvB,SAAO,MAAM,EAAE,QAAQ,CAAC,IAAI,SAAY,EAAE,YAAY;AACxD;AAEA,SAAS,qBAAqB,SAAqC;AACjE,QAAM,QAAQ,QAAQ,KAAK,EAAE,MAAM,oBAAoB;AACvD,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC;AAC3B,SAAO,MAAM,EAAE,QAAQ,CAAC,IAAI,SAAY,EAAE,YAAY;AACxD;AAEO,MAAM,gBAAwC;AAAA,EACnD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB,EAAE,OAAO,KAAK,MAAM,CAAC;AACxD,UAAM,MAAM,uCAAuC,MAAM;AAEzD,QAAI,MAAM,sBAAsB,EAAE,MAAM,CAAC;AAEzC,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc;AAAA,QACd,gBAAgB;AAAA,MAClB;AAAA,MACA,MAAM,OAAO,SAAS;AAAA,MACtB,QAAQ,YAAY,QAAQ,SAAS;AAAA,IACvC,CAAC;AAED,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,sBAAsB,SAAS,MAAM,EAAE;AAEzE,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,WAAO,KAAK,aAAa,MAAM,UAAU;AAAA,EAC3C;AAAA,EAEA,aAAa,MAAc,YAAuC;AAChE,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAA6B,CAAC;AAEpC,UAAM,QAAQ,SAAS,iBAAiB,eAAe;AACvD,UAAM,QAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU;AAE/C,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,OAAO,MAAM,CAAC;AACpB,YAAM,OAAO,KAAK,cAAc,yCAAyC;AACzE,YAAM,YAAY,KAAK,cAAc,oBAAoB;AAEzD,YAAM,OAAO,MAAM,aAAa,MAAM;AACtC,YAAM,QAAQ,MAAM,aAAa,KAAK;AAEtC,UAAI,QAAQ,OAAO;AAEjB,cAAM,SAAS,KAAK,cAAc,0BAA0B;AAC5D,cAAM,cAAc,WAAW,aAAa,KAAK,KAAK;AACtD,cAAM,iBAAiB,gBAAgB,MAAM,KAAK,qBAAqB,WAAW;AAElF,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA,KAAK;AAAA,UACL,SAAS;AAAA,UACT,iBAAiB,IAAI,IAAI,KAAK,IAAI,MAAM,QAAQ,CAAC;AAAA,UACjD,QAAQ;AAAA,UACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,QAC7C,CAAC;AAAA,MACH;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}