@staticn0va/wigolo 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (982) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +195 -73
  3. package/SKILL.md +382 -0
  4. package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
  5. package/assets/blocks/claude-code/wigolo-command.md +40 -0
  6. package/assets/blocks/cursor/wigolo.mdc +46 -0
  7. package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
  8. package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
  9. package/assets/skills/wigolo/SKILL.md +50 -0
  10. package/assets/skills/wigolo/rules/cache-first.md +30 -0
  11. package/assets/skills/wigolo/rules/synthesis.md +43 -0
  12. package/assets/skills/wigolo-agent/SKILL.md +73 -0
  13. package/assets/skills/wigolo-crawl/SKILL.md +60 -0
  14. package/assets/skills/wigolo-extract/SKILL.md +59 -0
  15. package/assets/skills/wigolo-fetch/SKILL.md +65 -0
  16. package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
  17. package/assets/skills/wigolo-research/SKILL.md +77 -0
  18. package/assets/skills/wigolo-search/SKILL.md +78 -0
  19. package/dist/agent/executor.d.ts +33 -0
  20. package/dist/agent/executor.d.ts.map +1 -0
  21. package/dist/agent/executor.js +233 -0
  22. package/dist/agent/executor.js.map +1 -0
  23. package/dist/agent/pipeline.d.ts +5 -0
  24. package/dist/agent/pipeline.d.ts.map +1 -0
  25. package/dist/agent/pipeline.js +208 -0
  26. package/dist/agent/pipeline.js.map +1 -0
  27. package/dist/agent/planner.d.ts +13 -0
  28. package/dist/agent/planner.d.ts.map +1 -0
  29. package/dist/agent/planner.js +271 -0
  30. package/dist/agent/planner.js.map +1 -0
  31. package/dist/agent/relevance.d.ts +15 -0
  32. package/dist/agent/relevance.d.ts.map +1 -0
  33. package/dist/agent/relevance.js +60 -0
  34. package/dist/agent/relevance.js.map +1 -0
  35. package/dist/cache/backfill-embeddings.d.ts +23 -0
  36. package/dist/cache/backfill-embeddings.d.ts.map +1 -0
  37. package/dist/cache/backfill-embeddings.js +105 -0
  38. package/dist/cache/backfill-embeddings.js.map +1 -0
  39. package/dist/cache/change-detector.d.ts +7 -0
  40. package/dist/cache/change-detector.d.ts.map +1 -0
  41. package/dist/cache/change-detector.js +43 -0
  42. package/dist/cache/change-detector.js.map +1 -0
  43. package/dist/cache/db.d.ts +1 -0
  44. package/dist/cache/db.d.ts.map +1 -1
  45. package/dist/cache/db.js +94 -22
  46. package/dist/cache/db.js.map +1 -1
  47. package/dist/cache/diff-summary.d.ts +2 -0
  48. package/dist/cache/diff-summary.d.ts.map +1 -0
  49. package/dist/cache/diff-summary.js +82 -0
  50. package/dist/cache/diff-summary.js.map +1 -0
  51. package/dist/cache/migrations/runner.d.ts +29 -0
  52. package/dist/cache/migrations/runner.d.ts.map +1 -0
  53. package/dist/cache/migrations/runner.js +147 -0
  54. package/dist/cache/migrations/runner.js.map +1 -0
  55. package/dist/cache/sqlite-vec-store.d.ts +42 -0
  56. package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
  57. package/dist/cache/sqlite-vec-store.js +176 -0
  58. package/dist/cache/sqlite-vec-store.js.map +1 -0
  59. package/dist/cache/store.d.ts +46 -1
  60. package/dist/cache/store.d.ts.map +1 -1
  61. package/dist/cache/store.js +362 -168
  62. package/dist/cache/store.js.map +1 -1
  63. package/dist/cli/agents/antigravity.d.ts +20 -0
  64. package/dist/cli/agents/antigravity.d.ts.map +1 -0
  65. package/dist/cli/agents/antigravity.js +49 -0
  66. package/dist/cli/agents/antigravity.js.map +1 -0
  67. package/dist/cli/agents/claude-code.d.ts +25 -0
  68. package/dist/cli/agents/claude-code.d.ts.map +1 -0
  69. package/dist/cli/agents/claude-code.js +111 -0
  70. package/dist/cli/agents/claude-code.js.map +1 -0
  71. package/dist/cli/agents/cursor.d.ts +21 -0
  72. package/dist/cli/agents/cursor.d.ts.map +1 -0
  73. package/dist/cli/agents/cursor.js +58 -0
  74. package/dist/cli/agents/cursor.js.map +1 -0
  75. package/dist/cli/agents/gemini-cli.d.ts +21 -0
  76. package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
  77. package/dist/cli/agents/gemini-cli.js +55 -0
  78. package/dist/cli/agents/gemini-cli.js.map +1 -0
  79. package/dist/cli/agents/registry.d.ts +21 -0
  80. package/dist/cli/agents/registry.d.ts.map +1 -0
  81. package/dist/cli/agents/registry.js +27 -0
  82. package/dist/cli/agents/registry.js.map +1 -0
  83. package/dist/cli/agents/utils.d.ts +26 -0
  84. package/dist/cli/agents/utils.d.ts.map +1 -0
  85. package/dist/cli/agents/utils.js +136 -0
  86. package/dist/cli/agents/utils.js.map +1 -0
  87. package/dist/cli/agents/vscode.d.ts +21 -0
  88. package/dist/cli/agents/vscode.d.ts.map +1 -0
  89. package/dist/cli/agents/vscode.js +62 -0
  90. package/dist/cli/agents/vscode.js.map +1 -0
  91. package/dist/cli/auth.d.ts +2 -0
  92. package/dist/cli/auth.d.ts.map +1 -0
  93. package/dist/cli/auth.js +94 -0
  94. package/dist/cli/auth.js.map +1 -0
  95. package/dist/cli/backfill.d.ts +2 -0
  96. package/dist/cli/backfill.d.ts.map +1 -0
  97. package/dist/cli/backfill.js +58 -0
  98. package/dist/cli/backfill.js.map +1 -0
  99. package/dist/cli/daemon.d.ts +6 -1
  100. package/dist/cli/daemon.d.ts.map +1 -1
  101. package/dist/cli/daemon.js +61 -3
  102. package/dist/cli/daemon.js.map +1 -1
  103. package/dist/cli/doctor.d.ts +8 -0
  104. package/dist/cli/doctor.d.ts.map +1 -0
  105. package/dist/cli/doctor.js +318 -0
  106. package/dist/cli/doctor.js.map +1 -0
  107. package/dist/cli/health.d.ts +1 -1
  108. package/dist/cli/health.d.ts.map +1 -1
  109. package/dist/cli/health.js +42 -3
  110. package/dist/cli/health.js.map +1 -1
  111. package/dist/cli/help.d.ts +6 -0
  112. package/dist/cli/help.d.ts.map +1 -0
  113. package/dist/cli/help.js +63 -0
  114. package/dist/cli/help.js.map +1 -0
  115. package/dist/cli/index.d.ts +1 -1
  116. package/dist/cli/index.d.ts.map +1 -1
  117. package/dist/cli/index.js +35 -7
  118. package/dist/cli/index.js.map +1 -1
  119. package/dist/cli/init.d.ts +2 -0
  120. package/dist/cli/init.d.ts.map +1 -0
  121. package/dist/cli/init.js +201 -0
  122. package/dist/cli/init.js.map +1 -0
  123. package/dist/cli/plugin.d.ts +5 -0
  124. package/dist/cli/plugin.d.ts.map +1 -0
  125. package/dist/cli/plugin.js +185 -0
  126. package/dist/cli/plugin.js.map +1 -0
  127. package/dist/cli/setup-mcp.d.ts +2 -0
  128. package/dist/cli/setup-mcp.d.ts.map +1 -0
  129. package/dist/cli/setup-mcp.js +114 -0
  130. package/dist/cli/setup-mcp.js.map +1 -0
  131. package/dist/cli/shell.d.ts +2 -0
  132. package/dist/cli/shell.d.ts.map +1 -0
  133. package/dist/cli/shell.js +86 -0
  134. package/dist/cli/shell.js.map +1 -0
  135. package/dist/cli/status.d.ts +2 -0
  136. package/dist/cli/status.d.ts.map +1 -0
  137. package/dist/cli/status.js +31 -0
  138. package/dist/cli/status.js.map +1 -0
  139. package/dist/cli/telemetry.d.ts +10 -0
  140. package/dist/cli/telemetry.d.ts.map +1 -0
  141. package/dist/cli/telemetry.js +56 -0
  142. package/dist/cli/telemetry.js.map +1 -0
  143. package/dist/cli/tui/agents-types.d.ts +28 -0
  144. package/dist/cli/tui/agents-types.d.ts.map +1 -0
  145. package/dist/cli/tui/agents-types.js +1 -0
  146. package/dist/cli/tui/agents-types.js.map +1 -0
  147. package/dist/cli/tui/agents.d.ts +11 -0
  148. package/dist/cli/tui/agents.d.ts.map +1 -0
  149. package/dist/cli/tui/agents.js +93 -0
  150. package/dist/cli/tui/agents.js.map +1 -0
  151. package/dist/cli/tui/banner.d.ts +3 -0
  152. package/dist/cli/tui/banner.d.ts.map +1 -0
  153. package/dist/cli/tui/banner.js +30 -0
  154. package/dist/cli/tui/banner.js.map +1 -0
  155. package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
  156. package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
  157. package/dist/cli/tui/components/AgentSelect.js +116 -0
  158. package/dist/cli/tui/components/AgentSelect.js.map +1 -0
  159. package/dist/cli/tui/components/Banner.d.ts +6 -0
  160. package/dist/cli/tui/components/Banner.d.ts.map +1 -0
  161. package/dist/cli/tui/components/Banner.js +25 -0
  162. package/dist/cli/tui/components/Banner.js.map +1 -0
  163. package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
  164. package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
  165. package/dist/cli/tui/components/BrowserSelect.js +19 -0
  166. package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
  167. package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
  168. package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
  169. package/dist/cli/tui/components/InstallProgress.js +67 -0
  170. package/dist/cli/tui/components/InstallProgress.js.map +1 -0
  171. package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
  172. package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
  173. package/dist/cli/tui/components/SkillInstall.js +94 -0
  174. package/dist/cli/tui/components/SkillInstall.js.map +1 -0
  175. package/dist/cli/tui/components/Summary.d.ts +22 -0
  176. package/dist/cli/tui/components/Summary.d.ts.map +1 -0
  177. package/dist/cli/tui/components/Summary.js +135 -0
  178. package/dist/cli/tui/components/Summary.js.map +1 -0
  179. package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
  180. package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
  181. package/dist/cli/tui/components/SystemCheck.js +71 -0
  182. package/dist/cli/tui/components/SystemCheck.js.map +1 -0
  183. package/dist/cli/tui/components/Verification.d.ts +8 -0
  184. package/dist/cli/tui/components/Verification.d.ts.map +1 -0
  185. package/dist/cli/tui/components/Verification.js +63 -0
  186. package/dist/cli/tui/components/Verification.js.map +1 -0
  187. package/dist/cli/tui/config-writer-cli.d.ts +12 -0
  188. package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
  189. package/dist/cli/tui/config-writer-cli.js +39 -0
  190. package/dist/cli/tui/config-writer-cli.js.map +1 -0
  191. package/dist/cli/tui/config-writer-json.d.ts +16 -0
  192. package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
  193. package/dist/cli/tui/config-writer-json.js +86 -0
  194. package/dist/cli/tui/config-writer-json.js.map +1 -0
  195. package/dist/cli/tui/config-writer-toml.d.ts +16 -0
  196. package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
  197. package/dist/cli/tui/config-writer-toml.js +83 -0
  198. package/dist/cli/tui/config-writer-toml.js.map +1 -0
  199. package/dist/cli/tui/config-writer.d.ts +25 -0
  200. package/dist/cli/tui/config-writer.d.ts.map +1 -0
  201. package/dist/cli/tui/config-writer.js +101 -0
  202. package/dist/cli/tui/config-writer.js.map +1 -0
  203. package/dist/cli/tui/detect-helpers.d.ts +6 -0
  204. package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
  205. package/dist/cli/tui/detect-helpers.js +45 -0
  206. package/dist/cli/tui/detect-helpers.js.map +1 -0
  207. package/dist/cli/tui/extras-prompt.d.ts +7 -0
  208. package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
  209. package/dist/cli/tui/extras-prompt.js +42 -0
  210. package/dist/cli/tui/extras-prompt.js.map +1 -0
  211. package/dist/cli/tui/flags-types.d.ts +19 -0
  212. package/dist/cli/tui/flags-types.d.ts.map +1 -0
  213. package/dist/cli/tui/flags-types.js +23 -0
  214. package/dist/cli/tui/flags-types.js.map +1 -0
  215. package/dist/cli/tui/flags.d.ts +5 -0
  216. package/dist/cli/tui/flags.d.ts.map +1 -0
  217. package/dist/cli/tui/flags.js +132 -0
  218. package/dist/cli/tui/flags.js.map +1 -0
  219. package/dist/cli/tui/format.d.ts +14 -0
  220. package/dist/cli/tui/format.d.ts.map +1 -0
  221. package/dist/cli/tui/format.js +37 -0
  222. package/dist/cli/tui/format.js.map +1 -0
  223. package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
  224. package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
  225. package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
  226. package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
  227. package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
  228. package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
  229. package/dist/cli/tui/hooks/useInstall.js +90 -0
  230. package/dist/cli/tui/hooks/useInstall.js.map +1 -0
  231. package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
  232. package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
  233. package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
  234. package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
  235. package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
  236. package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
  237. package/dist/cli/tui/hooks/useVerify.js +71 -0
  238. package/dist/cli/tui/hooks/useVerify.js.map +1 -0
  239. package/dist/cli/tui/ink-init.d.ts +2 -0
  240. package/dist/cli/tui/ink-init.d.ts.map +1 -0
  241. package/dist/cli/tui/ink-init.js +198 -0
  242. package/dist/cli/tui/ink-init.js.map +1 -0
  243. package/dist/cli/tui/reporter-auto.d.ts +7 -0
  244. package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
  245. package/dist/cli/tui/reporter-auto.js +15 -0
  246. package/dist/cli/tui/reporter-auto.js.map +1 -0
  247. package/dist/cli/tui/reporter.d.ts +26 -0
  248. package/dist/cli/tui/reporter.d.ts.map +1 -0
  249. package/dist/cli/tui/reporter.js +32 -0
  250. package/dist/cli/tui/reporter.js.map +1 -0
  251. package/dist/cli/tui/run-command.d.ts +14 -0
  252. package/dist/cli/tui/run-command.d.ts.map +1 -0
  253. package/dist/cli/tui/run-command.js +72 -0
  254. package/dist/cli/tui/run-command.js.map +1 -0
  255. package/dist/cli/tui/select-agents.d.ts +6 -0
  256. package/dist/cli/tui/select-agents.d.ts.map +1 -0
  257. package/dist/cli/tui/select-agents.js +32 -0
  258. package/dist/cli/tui/select-agents.js.map +1 -0
  259. package/dist/cli/tui/status-agents.d.ts +11 -0
  260. package/dist/cli/tui/status-agents.d.ts.map +1 -0
  261. package/dist/cli/tui/status-agents.js +53 -0
  262. package/dist/cli/tui/status-agents.js.map +1 -0
  263. package/dist/cli/tui/status-cache.d.ts +6 -0
  264. package/dist/cli/tui/status-cache.d.ts.map +1 -0
  265. package/dist/cli/tui/status-cache.js +39 -0
  266. package/dist/cli/tui/status-cache.js.map +1 -0
  267. package/dist/cli/tui/status-format.d.ts +14 -0
  268. package/dist/cli/tui/status-format.d.ts.map +1 -0
  269. package/dist/cli/tui/status-format.js +41 -0
  270. package/dist/cli/tui/status-format.js.map +1 -0
  271. package/dist/cli/tui/status-python.d.ts +6 -0
  272. package/dist/cli/tui/status-python.d.ts.map +1 -0
  273. package/dist/cli/tui/status-python.js +30 -0
  274. package/dist/cli/tui/status-python.js.map +1 -0
  275. package/dist/cli/tui/system-check.d.ts +24 -0
  276. package/dist/cli/tui/system-check.d.ts.map +1 -0
  277. package/dist/cli/tui/system-check.js +103 -0
  278. package/dist/cli/tui/system-check.js.map +1 -0
  279. package/dist/cli/tui/tui-reporter.d.ts +19 -0
  280. package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
  281. package/dist/cli/tui/tui-reporter.js +95 -0
  282. package/dist/cli/tui/tui-reporter.js.map +1 -0
  283. package/dist/cli/tui/utils/config-writer.d.ts +3 -0
  284. package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
  285. package/dist/cli/tui/utils/config-writer.js +22 -0
  286. package/dist/cli/tui/utils/config-writer.js.map +1 -0
  287. package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
  288. package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
  289. package/dist/cli/tui/utils/suppress-logs.js +11 -0
  290. package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
  291. package/dist/cli/tui/verify-suggestions.d.ts +5 -0
  292. package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
  293. package/dist/cli/tui/verify-suggestions.js +20 -0
  294. package/dist/cli/tui/verify-suggestions.js.map +1 -0
  295. package/dist/cli/tui/verify.d.ts +14 -0
  296. package/dist/cli/tui/verify.d.ts.map +1 -0
  297. package/dist/cli/tui/verify.js +101 -0
  298. package/dist/cli/tui/verify.js.map +1 -0
  299. package/dist/cli/tui/version.d.ts +2 -0
  300. package/dist/cli/tui/version.d.ts.map +1 -0
  301. package/dist/cli/tui/version.js +14 -0
  302. package/dist/cli/tui/version.js.map +1 -0
  303. package/dist/cli/uninstall.d.ts +2 -0
  304. package/dist/cli/uninstall.d.ts.map +1 -0
  305. package/dist/cli/uninstall.js +57 -0
  306. package/dist/cli/uninstall.js.map +1 -0
  307. package/dist/cli/warmup.d.ts +10 -2
  308. package/dist/cli/warmup.d.ts.map +1 -1
  309. package/dist/cli/warmup.js +226 -93
  310. package/dist/cli/warmup.js.map +1 -1
  311. package/dist/config.d.ts +28 -2
  312. package/dist/config.d.ts.map +1 -1
  313. package/dist/config.js +106 -56
  314. package/dist/config.js.map +1 -1
  315. package/dist/crawl/crawler.d.ts +6 -0
  316. package/dist/crawl/crawler.d.ts.map +1 -1
  317. package/dist/crawl/crawler.js +210 -209
  318. package/dist/crawl/crawler.js.map +1 -1
  319. package/dist/crawl/dedup.d.ts +1 -0
  320. package/dist/crawl/dedup.d.ts.map +1 -1
  321. package/dist/crawl/dedup.js +124 -81
  322. package/dist/crawl/dedup.js.map +1 -1
  323. package/dist/crawl/etag-incremental.d.ts +43 -0
  324. package/dist/crawl/etag-incremental.d.ts.map +1 -0
  325. package/dist/crawl/etag-incremental.js +94 -0
  326. package/dist/crawl/etag-incremental.js.map +1 -0
  327. package/dist/crawl/index-to-vec.d.ts +10 -0
  328. package/dist/crawl/index-to-vec.d.ts.map +1 -0
  329. package/dist/crawl/index-to-vec.js +44 -0
  330. package/dist/crawl/index-to-vec.js.map +1 -0
  331. package/dist/crawl/mapper.js +136 -164
  332. package/dist/crawl/mapper.js.map +1 -1
  333. package/dist/crawl/rate-limiter.js +63 -66
  334. package/dist/crawl/rate-limiter.js.map +1 -1
  335. package/dist/crawl/robots.js +58 -57
  336. package/dist/crawl/robots.js.map +1 -1
  337. package/dist/crawl/sitemap-first.d.ts +12 -0
  338. package/dist/crawl/sitemap-first.d.ts.map +1 -0
  339. package/dist/crawl/sitemap-first.js +47 -0
  340. package/dist/crawl/sitemap-first.js.map +1 -0
  341. package/dist/crawl/sitemap.js +33 -32
  342. package/dist/crawl/sitemap.js.map +1 -1
  343. package/dist/crawl/url-utils.d.ts +1 -0
  344. package/dist/crawl/url-utils.d.ts.map +1 -1
  345. package/dist/crawl/url-utils.js +49 -37
  346. package/dist/crawl/url-utils.js.map +1 -1
  347. package/dist/daemon/health-check.d.ts +16 -0
  348. package/dist/daemon/health-check.d.ts.map +1 -0
  349. package/dist/daemon/health-check.js +33 -0
  350. package/dist/daemon/health-check.js.map +1 -0
  351. package/dist/daemon/http-server.d.ts +26 -0
  352. package/dist/daemon/http-server.d.ts.map +1 -0
  353. package/dist/daemon/http-server.js +275 -0
  354. package/dist/daemon/http-server.js.map +1 -0
  355. package/dist/daemon/proxy.d.ts +10 -0
  356. package/dist/daemon/proxy.d.ts.map +1 -0
  357. package/dist/daemon/proxy.js +93 -0
  358. package/dist/daemon/proxy.js.map +1 -0
  359. package/dist/embedding/embed.d.ts +59 -0
  360. package/dist/embedding/embed.d.ts.map +1 -0
  361. package/dist/embedding/embed.js +233 -0
  362. package/dist/embedding/embed.js.map +1 -0
  363. package/dist/embedding/fastembed-provider.d.ts +19 -0
  364. package/dist/embedding/fastembed-provider.d.ts.map +1 -0
  365. package/dist/embedding/fastembed-provider.js +51 -0
  366. package/dist/embedding/fastembed-provider.js.map +1 -0
  367. package/dist/embedding/key-terms.d.ts +12 -0
  368. package/dist/embedding/key-terms.d.ts.map +1 -0
  369. package/dist/embedding/key-terms.js +234 -0
  370. package/dist/embedding/key-terms.js.map +1 -0
  371. package/dist/extraction/boilerplate.d.ts +15 -0
  372. package/dist/extraction/boilerplate.d.ts.map +1 -0
  373. package/dist/extraction/boilerplate.js +52 -0
  374. package/dist/extraction/boilerplate.js.map +1 -0
  375. package/dist/extraction/defuddle.d.ts.map +1 -1
  376. package/dist/extraction/defuddle.js +27 -23
  377. package/dist/extraction/defuddle.js.map +1 -1
  378. package/dist/extraction/extract.d.ts.map +1 -1
  379. package/dist/extraction/extract.js +76 -76
  380. package/dist/extraction/extract.js.map +1 -1
  381. package/dist/extraction/jsonld.js +50 -54
  382. package/dist/extraction/jsonld.js.map +1 -1
  383. package/dist/extraction/lang-hints.d.ts +2 -0
  384. package/dist/extraction/lang-hints.d.ts.map +1 -0
  385. package/dist/extraction/lang-hints.js +30 -0
  386. package/dist/extraction/lang-hints.js.map +1 -0
  387. package/dist/extraction/llm-fallback.d.ts +17 -0
  388. package/dist/extraction/llm-fallback.d.ts.map +1 -0
  389. package/dist/extraction/llm-fallback.js +130 -0
  390. package/dist/extraction/llm-fallback.js.map +1 -0
  391. package/dist/extraction/markdown-sanitize.d.ts +2 -0
  392. package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
  393. package/dist/extraction/markdown-sanitize.js +151 -0
  394. package/dist/extraction/markdown-sanitize.js.map +1 -0
  395. package/dist/extraction/markdown.d.ts +11 -0
  396. package/dist/extraction/markdown.d.ts.map +1 -1
  397. package/dist/extraction/markdown.js +195 -91
  398. package/dist/extraction/markdown.js.map +1 -1
  399. package/dist/extraction/pipeline.d.ts +8 -0
  400. package/dist/extraction/pipeline.d.ts.map +1 -1
  401. package/dist/extraction/pipeline.js +57 -91
  402. package/dist/extraction/pipeline.js.map +1 -1
  403. package/dist/extraction/readability.d.ts +1 -1
  404. package/dist/extraction/readability.d.ts.map +1 -1
  405. package/dist/extraction/readability.js +28 -29
  406. package/dist/extraction/readability.js.map +1 -1
  407. package/dist/extraction/schema.d.ts +12 -0
  408. package/dist/extraction/schema.d.ts.map +1 -1
  409. package/dist/extraction/schema.js +135 -72
  410. package/dist/extraction/schema.js.map +1 -1
  411. package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
  412. package/dist/extraction/site-extractors/docs-generic.js +81 -91
  413. package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
  414. package/dist/extraction/site-extractors/github.d.ts.map +1 -1
  415. package/dist/extraction/site-extractors/github.js +87 -95
  416. package/dist/extraction/site-extractors/github.js.map +1 -1
  417. package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
  418. package/dist/extraction/site-extractors/mdn.js +46 -54
  419. package/dist/extraction/site-extractors/mdn.js.map +1 -1
  420. package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
  421. package/dist/extraction/site-extractors/stackoverflow.js +71 -80
  422. package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
  423. package/dist/extraction/structured-data.d.ts +4 -0
  424. package/dist/extraction/structured-data.d.ts.map +1 -0
  425. package/dist/extraction/structured-data.js +173 -0
  426. package/dist/extraction/structured-data.js.map +1 -0
  427. package/dist/extraction/structured.d.ts +4 -0
  428. package/dist/extraction/structured.d.ts.map +1 -0
  429. package/dist/extraction/structured.js +163 -0
  430. package/dist/extraction/structured.js.map +1 -0
  431. package/dist/extraction/v1/classifier.d.ts +3 -0
  432. package/dist/extraction/v1/classifier.d.ts.map +1 -0
  433. package/dist/extraction/v1/classifier.js +110 -0
  434. package/dist/extraction/v1/classifier.js.map +1 -0
  435. package/dist/extraction/v1/extract-provider.d.ts +16 -0
  436. package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
  437. package/dist/extraction/v1/extract-provider.js +43 -0
  438. package/dist/extraction/v1/extract-provider.js.map +1 -0
  439. package/dist/extraction/v1/local-llm.d.ts +8 -0
  440. package/dist/extraction/v1/local-llm.d.ts.map +1 -0
  441. package/dist/extraction/v1/local-llm.js +58 -0
  442. package/dist/extraction/v1/local-llm.js.map +1 -0
  443. package/dist/extraction/v1/news.d.ts +3 -0
  444. package/dist/extraction/v1/news.d.ts.map +1 -0
  445. package/dist/extraction/v1/news.js +61 -0
  446. package/dist/extraction/v1/news.js.map +1 -0
  447. package/dist/extraction/v1/product.d.ts +3 -0
  448. package/dist/extraction/v1/product.d.ts.map +1 -0
  449. package/dist/extraction/v1/product.js +166 -0
  450. package/dist/extraction/v1/product.js.map +1 -0
  451. package/dist/extraction/v1/recipe.d.ts +3 -0
  452. package/dist/extraction/v1/recipe.d.ts.map +1 -0
  453. package/dist/extraction/v1/recipe.js +136 -0
  454. package/dist/extraction/v1/recipe.js.map +1 -0
  455. package/dist/extraction/v1/routed.d.ts +17 -0
  456. package/dist/extraction/v1/routed.d.ts.map +1 -0
  457. package/dist/extraction/v1/routed.js +68 -0
  458. package/dist/extraction/v1/routed.js.map +1 -0
  459. package/dist/extraction/v1/schemas/Article.d.ts +11 -0
  460. package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
  461. package/dist/extraction/v1/schemas/Article.js +23 -0
  462. package/dist/extraction/v1/schemas/Article.js.map +1 -0
  463. package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
  464. package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
  465. package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
  466. package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
  467. package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
  468. package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
  469. package/dist/extraction/v1/schemas/EventListing.js +122 -0
  470. package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
  471. package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
  472. package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
  473. package/dist/extraction/v1/schemas/Paper.js +156 -0
  474. package/dist/extraction/v1/schemas/Paper.js.map +1 -0
  475. package/dist/extraction/v1/schemas/Product.d.ts +17 -0
  476. package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
  477. package/dist/extraction/v1/schemas/Product.js +149 -0
  478. package/dist/extraction/v1/schemas/Product.js.map +1 -0
  479. package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
  480. package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
  481. package/dist/extraction/v1/schemas/Recipe.js +160 -0
  482. package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
  483. package/dist/extraction/v1/schemas/index.d.ts +13 -0
  484. package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
  485. package/dist/extraction/v1/schemas/index.js +44 -0
  486. package/dist/extraction/v1/schemas/index.js.map +1 -0
  487. package/dist/extraction/v1/site-extractors.d.ts +5 -0
  488. package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
  489. package/dist/extraction/v1/site-extractors.js +31 -0
  490. package/dist/extraction/v1/site-extractors.js.map +1 -0
  491. package/dist/fetch/action-executor.d.ts +28 -0
  492. package/dist/fetch/action-executor.d.ts.map +1 -0
  493. package/dist/fetch/action-executor.js +88 -0
  494. package/dist/fetch/action-executor.js.map +1 -0
  495. package/dist/fetch/auth.d.ts +2 -1
  496. package/dist/fetch/auth.d.ts.map +1 -1
  497. package/dist/fetch/auth.js +56 -26
  498. package/dist/fetch/auth.js.map +1 -1
  499. package/dist/fetch/browser-pool.d.ts +30 -11
  500. package/dist/fetch/browser-pool.d.ts.map +1 -1
  501. package/dist/fetch/browser-pool.js +303 -127
  502. package/dist/fetch/browser-pool.js.map +1 -1
  503. package/dist/fetch/browser-selector.d.ts +17 -0
  504. package/dist/fetch/browser-selector.d.ts.map +1 -0
  505. package/dist/fetch/browser-selector.js +72 -0
  506. package/dist/fetch/browser-selector.js.map +1 -0
  507. package/dist/fetch/browser-types.d.ts +3 -0
  508. package/dist/fetch/browser-types.d.ts.map +1 -0
  509. package/dist/fetch/browser-types.js +45 -0
  510. package/dist/fetch/browser-types.js.map +1 -0
  511. package/dist/fetch/cdp-client.d.ts +9 -0
  512. package/dist/fetch/cdp-client.d.ts.map +1 -0
  513. package/dist/fetch/cdp-client.js +89 -0
  514. package/dist/fetch/cdp-client.js.map +1 -0
  515. package/dist/fetch/content-check.js +39 -46
  516. package/dist/fetch/content-check.js.map +1 -1
  517. package/dist/fetch/http-client.d.ts +4 -0
  518. package/dist/fetch/http-client.d.ts.map +1 -1
  519. package/dist/fetch/http-client.js +147 -128
  520. package/dist/fetch/http-client.js.map +1 -1
  521. package/dist/fetch/lightpanda.d.ts +28 -0
  522. package/dist/fetch/lightpanda.d.ts.map +1 -0
  523. package/dist/fetch/lightpanda.js +174 -0
  524. package/dist/fetch/lightpanda.js.map +1 -0
  525. package/dist/fetch/playwright-tier.d.ts +19 -0
  526. package/dist/fetch/playwright-tier.d.ts.map +1 -0
  527. package/dist/fetch/playwright-tier.js +76 -0
  528. package/dist/fetch/playwright-tier.js.map +1 -0
  529. package/dist/fetch/router.d.ts +49 -3
  530. package/dist/fetch/router.d.ts.map +1 -1
  531. package/dist/fetch/router.js +185 -81
  532. package/dist/fetch/router.js.map +1 -1
  533. package/dist/index.js +97 -17
  534. package/dist/index.js.map +1 -1
  535. package/dist/instructions.d.ts +31 -0
  536. package/dist/instructions.d.ts.map +1 -0
  537. package/dist/instructions.js +245 -0
  538. package/dist/instructions.js.map +1 -0
  539. package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
  540. package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
  541. package/dist/integrations/cloud/llm/anthropic.js +41 -0
  542. package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
  543. package/dist/integrations/cloud/llm/cache.d.ts +5 -0
  544. package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
  545. package/dist/integrations/cloud/llm/cache.js +49 -0
  546. package/dist/integrations/cloud/llm/cache.js.map +1 -0
  547. package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
  548. package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
  549. package/dist/integrations/cloud/llm/gemini.js +37 -0
  550. package/dist/integrations/cloud/llm/gemini.js.map +1 -0
  551. package/dist/integrations/cloud/llm/groq.d.ts +3 -0
  552. package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
  553. package/dist/integrations/cloud/llm/groq.js +74 -0
  554. package/dist/integrations/cloud/llm/groq.js.map +1 -0
  555. package/dist/integrations/cloud/llm/hash.d.ts +3 -0
  556. package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
  557. package/dist/integrations/cloud/llm/hash.js +26 -0
  558. package/dist/integrations/cloud/llm/hash.js.map +1 -0
  559. package/dist/integrations/cloud/llm/openai.d.ts +3 -0
  560. package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
  561. package/dist/integrations/cloud/llm/openai.js +43 -0
  562. package/dist/integrations/cloud/llm/openai.js.map +1 -0
  563. package/dist/integrations/cloud/llm/select.d.ts +5 -0
  564. package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
  565. package/dist/integrations/cloud/llm/select.js +30 -0
  566. package/dist/integrations/cloud/llm/select.js.map +1 -0
  567. package/dist/integrations/cloud/llm/types.d.ts +24 -0
  568. package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
  569. package/dist/integrations/cloud/llm/types.js +1 -0
  570. package/dist/integrations/cloud/llm/types.js.map +1 -0
  571. package/dist/integrations/cloud/llm/validate.d.ts +6 -0
  572. package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
  573. package/dist/integrations/cloud/llm/validate.js +63 -0
  574. package/dist/integrations/cloud/llm/validate.js.map +1 -0
  575. package/dist/logger.d.ts +4 -1
  576. package/dist/logger.d.ts.map +1 -1
  577. package/dist/logger.js +71 -30
  578. package/dist/logger.js.map +1 -1
  579. package/dist/pdf-parse.d.js +1 -0
  580. package/dist/pdf-parse.d.js.map +1 -0
  581. package/dist/plugins/loader.d.ts +20 -0
  582. package/dist/plugins/loader.d.ts.map +1 -0
  583. package/dist/plugins/loader.js +157 -0
  584. package/dist/plugins/loader.js.map +1 -0
  585. package/dist/plugins/registry.d.ts +26 -0
  586. package/dist/plugins/registry.d.ts.map +1 -0
  587. package/dist/plugins/registry.js +71 -0
  588. package/dist/plugins/registry.js.map +1 -0
  589. package/dist/plugins/validate.d.ts +9 -0
  590. package/dist/plugins/validate.d.ts.map +1 -0
  591. package/dist/plugins/validate.js +79 -0
  592. package/dist/plugins/validate.js.map +1 -0
  593. package/dist/providers/embed-provider.d.ts +11 -0
  594. package/dist/providers/embed-provider.d.ts.map +1 -0
  595. package/dist/providers/embed-provider.js +24 -0
  596. package/dist/providers/embed-provider.js.map +1 -0
  597. package/dist/providers/extract-provider.d.ts +23 -0
  598. package/dist/providers/extract-provider.d.ts.map +1 -0
  599. package/dist/providers/extract-provider.js +25 -0
  600. package/dist/providers/extract-provider.js.map +1 -0
  601. package/dist/providers/rerank-provider.d.ts +16 -0
  602. package/dist/providers/rerank-provider.d.ts.map +1 -0
  603. package/dist/providers/rerank-provider.js +28 -0
  604. package/dist/providers/rerank-provider.js.map +1 -0
  605. package/dist/providers/search-provider.d.ts +25 -0
  606. package/dist/providers/search-provider.d.ts.map +1 -0
  607. package/dist/providers/search-provider.js +44 -0
  608. package/dist/providers/search-provider.js.map +1 -0
  609. package/dist/providers/vector-store.d.ts +27 -0
  610. package/dist/providers/vector-store.d.ts.map +1 -0
  611. package/dist/providers/vector-store.js +27 -0
  612. package/dist/providers/vector-store.js.map +1 -0
  613. package/dist/python-env.d.ts +9 -0
  614. package/dist/python-env.d.ts.map +1 -0
  615. package/dist/python-env.js +13 -0
  616. package/dist/python-env.js.map +1 -0
  617. package/dist/repl/commands/agent.d.ts +5 -0
  618. package/dist/repl/commands/agent.d.ts.map +1 -0
  619. package/dist/repl/commands/agent.js +62 -0
  620. package/dist/repl/commands/agent.js.map +1 -0
  621. package/dist/repl/commands/cache.d.ts +4 -0
  622. package/dist/repl/commands/cache.d.ts.map +1 -0
  623. package/dist/repl/commands/cache.js +43 -0
  624. package/dist/repl/commands/cache.js.map +1 -0
  625. package/dist/repl/commands/crawl.d.ts +7 -0
  626. package/dist/repl/commands/crawl.d.ts.map +1 -0
  627. package/dist/repl/commands/crawl.js +44 -0
  628. package/dist/repl/commands/crawl.js.map +1 -0
  629. package/dist/repl/commands/extract.d.ts +5 -0
  630. package/dist/repl/commands/extract.d.ts.map +1 -0
  631. package/dist/repl/commands/extract.js +47 -0
  632. package/dist/repl/commands/extract.js.map +1 -0
  633. package/dist/repl/commands/fetch.d.ts +5 -0
  634. package/dist/repl/commands/fetch.d.ts.map +1 -0
  635. package/dist/repl/commands/fetch.js +67 -0
  636. package/dist/repl/commands/fetch.js.map +1 -0
  637. package/dist/repl/commands/find-similar.d.ts +5 -0
  638. package/dist/repl/commands/find-similar.d.ts.map +1 -0
  639. package/dist/repl/commands/find-similar.js +74 -0
  640. package/dist/repl/commands/find-similar.js.map +1 -0
  641. package/dist/repl/commands/research.d.ts +5 -0
  642. package/dist/repl/commands/research.d.ts.map +1 -0
  643. package/dist/repl/commands/research.js +65 -0
  644. package/dist/repl/commands/research.js.map +1 -0
  645. package/dist/repl/commands/search.d.ts +5 -0
  646. package/dist/repl/commands/search.d.ts.map +1 -0
  647. package/dist/repl/commands/search.js +74 -0
  648. package/dist/repl/commands/search.js.map +1 -0
  649. package/dist/repl/commands/types.d.ts +9 -0
  650. package/dist/repl/commands/types.d.ts.map +1 -0
  651. package/dist/repl/commands/types.js +1 -0
  652. package/dist/repl/commands/types.js.map +1 -0
  653. package/dist/repl/formatters.d.ts +13 -0
  654. package/dist/repl/formatters.d.ts.map +1 -0
  655. package/dist/repl/formatters.js +283 -0
  656. package/dist/repl/formatters.js.map +1 -0
  657. package/dist/repl/parser.d.ts +9 -0
  658. package/dist/repl/parser.d.ts.map +1 -0
  659. package/dist/repl/parser.js +86 -0
  660. package/dist/repl/parser.js.map +1 -0
  661. package/dist/repl/shell.d.ts +8 -0
  662. package/dist/repl/shell.d.ts.map +1 -0
  663. package/dist/repl/shell.js +184 -0
  664. package/dist/repl/shell.js.map +1 -0
  665. package/dist/research/branch-exploration.d.ts +14 -0
  666. package/dist/research/branch-exploration.d.ts.map +1 -0
  667. package/dist/research/branch-exploration.js +100 -0
  668. package/dist/research/branch-exploration.js.map +1 -0
  669. package/dist/research/brief.d.ts +5 -0
  670. package/dist/research/brief.d.ts.map +1 -0
  671. package/dist/research/brief.js +242 -0
  672. package/dist/research/brief.js.map +1 -0
  673. package/dist/research/citation-graph.d.ts +9 -0
  674. package/dist/research/citation-graph.d.ts.map +1 -0
  675. package/dist/research/citation-graph.js +114 -0
  676. package/dist/research/citation-graph.js.map +1 -0
  677. package/dist/research/decompose.d.ts +14 -0
  678. package/dist/research/decompose.d.ts.map +1 -0
  679. package/dist/research/decompose.js +439 -0
  680. package/dist/research/decompose.js.map +1 -0
  681. package/dist/research/pipeline.d.ts +5 -0
  682. package/dist/research/pipeline.d.ts.map +1 -0
  683. package/dist/research/pipeline.js +269 -0
  684. package/dist/research/pipeline.js.map +1 -0
  685. package/dist/research/synthesis-local.d.ts +16 -0
  686. package/dist/research/synthesis-local.d.ts.map +1 -0
  687. package/dist/research/synthesis-local.js +73 -0
  688. package/dist/research/synthesis-local.js.map +1 -0
  689. package/dist/research/synthesize.d.ts +10 -0
  690. package/dist/research/synthesize.d.ts.map +1 -0
  691. package/dist/research/synthesize.js +137 -0
  692. package/dist/research/synthesize.js.map +1 -0
  693. package/dist/search/answer-synthesis.d.ts +33 -0
  694. package/dist/search/answer-synthesis.d.ts.map +1 -0
  695. package/dist/search/answer-synthesis.js +244 -0
  696. package/dist/search/answer-synthesis.js.map +1 -0
  697. package/dist/search/context-formatter.d.ts +3 -0
  698. package/dist/search/context-formatter.d.ts.map +1 -0
  699. package/dist/search/context-formatter.js +56 -0
  700. package/dist/search/context-formatter.js.map +1 -0
  701. package/dist/search/dedup.d.ts +1 -0
  702. package/dist/search/dedup.d.ts.map +1 -1
  703. package/dist/search/dedup.js +40 -32
  704. package/dist/search/dedup.js.map +1 -1
  705. package/dist/search/engines/arxiv.d.ts +7 -0
  706. package/dist/search/engines/arxiv.d.ts.map +1 -0
  707. package/dist/search/engines/arxiv.js +70 -0
  708. package/dist/search/engines/arxiv.js.map +1 -0
  709. package/dist/search/engines/bing-news.d.ts +7 -0
  710. package/dist/search/engines/bing-news.d.ts.map +1 -0
  711. package/dist/search/engines/bing-news.js +97 -0
  712. package/dist/search/engines/bing-news.js.map +1 -0
  713. package/dist/search/engines/bing.d.ts +1 -0
  714. package/dist/search/engines/bing.d.ts.map +1 -1
  715. package/dist/search/engines/bing.js +100 -44
  716. package/dist/search/engines/bing.js.map +1 -1
  717. package/dist/search/engines/devdocs.d.ts +6 -0
  718. package/dist/search/engines/devdocs.d.ts.map +1 -0
  719. package/dist/search/engines/devdocs.js +56 -0
  720. package/dist/search/engines/devdocs.js.map +1 -0
  721. package/dist/search/engines/duckduckgo.d.ts.map +1 -1
  722. package/dist/search/engines/duckduckgo.js +56 -44
  723. package/dist/search/engines/duckduckgo.js.map +1 -1
  724. package/dist/search/engines/github-code.d.ts +7 -0
  725. package/dist/search/engines/github-code.d.ts.map +1 -0
  726. package/dist/search/engines/github-code.js +55 -0
  727. package/dist/search/engines/github-code.js.map +1 -0
  728. package/dist/search/engines/hn-algolia.d.ts +7 -0
  729. package/dist/search/engines/hn-algolia.d.ts.map +1 -0
  730. package/dist/search/engines/hn-algolia.js +76 -0
  731. package/dist/search/engines/hn-algolia.js.map +1 -0
  732. package/dist/search/engines/lobsters.d.ts +7 -0
  733. package/dist/search/engines/lobsters.d.ts.map +1 -0
  734. package/dist/search/engines/lobsters.js +83 -0
  735. package/dist/search/engines/lobsters.js.map +1 -0
  736. package/dist/search/engines/mdn.d.ts +7 -0
  737. package/dist/search/engines/mdn.d.ts.map +1 -0
  738. package/dist/search/engines/mdn.js +48 -0
  739. package/dist/search/engines/mdn.js.map +1 -0
  740. package/dist/search/engines/semantic-scholar.d.ts +7 -0
  741. package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
  742. package/dist/search/engines/semantic-scholar.js +69 -0
  743. package/dist/search/engines/semantic-scholar.js.map +1 -0
  744. package/dist/search/engines/stackoverflow.d.ts +7 -0
  745. package/dist/search/engines/stackoverflow.d.ts.map +1 -0
  746. package/dist/search/engines/stackoverflow.js +73 -0
  747. package/dist/search/engines/stackoverflow.js.map +1 -0
  748. package/dist/search/engines/startpage.d.ts.map +1 -1
  749. package/dist/search/engines/startpage.js +65 -46
  750. package/dist/search/engines/startpage.js.map +1 -1
  751. package/dist/search/evidence.d.ts +25 -0
  752. package/dist/search/evidence.d.ts.map +1 -0
  753. package/dist/search/evidence.js +220 -0
  754. package/dist/search/evidence.js.map +1 -0
  755. package/dist/search/filters.js +49 -55
  756. package/dist/search/filters.js.map +1 -1
  757. package/dist/search/find-similar/crawl-rank.d.ts +9 -0
  758. package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
  759. package/dist/search/find-similar/crawl-rank.js +272 -0
  760. package/dist/search/find-similar/crawl-rank.js.map +1 -0
  761. package/dist/search/find-similar/mode.d.ts +4 -0
  762. package/dist/search/find-similar/mode.d.ts.map +1 -0
  763. package/dist/search/find-similar/mode.js +12 -0
  764. package/dist/search/find-similar/mode.js.map +1 -0
  765. package/dist/search/find-similar.d.ts +5 -0
  766. package/dist/search/find-similar.d.ts.map +1 -0
  767. package/dist/search/find-similar.js +509 -0
  768. package/dist/search/find-similar.js.map +1 -0
  769. package/dist/search/highlights.d.ts +19 -0
  770. package/dist/search/highlights.d.ts.map +1 -0
  771. package/dist/search/highlights.js +167 -0
  772. package/dist/search/highlights.js.map +1 -0
  773. package/dist/search/language-filter.d.ts +29 -0
  774. package/dist/search/language-filter.d.ts.map +1 -0
  775. package/dist/search/language-filter.js +126 -0
  776. package/dist/search/language-filter.js.map +1 -0
  777. package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
  778. package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
  779. package/dist/search/legacy/searxng-orchestrator.js +501 -0
  780. package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
  781. package/dist/search/legacy/searxng-provider.d.ts +7 -0
  782. package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
  783. package/dist/search/legacy/searxng-provider.js +11 -0
  784. package/dist/search/legacy/searxng-provider.js.map +1 -0
  785. package/dist/search/multi-query.d.ts +25 -0
  786. package/dist/search/multi-query.d.ts.map +1 -0
  787. package/dist/search/multi-query.js +228 -0
  788. package/dist/search/multi-query.js.map +1 -0
  789. package/dist/search/query.js +32 -34
  790. package/dist/search/query.js.map +1 -1
  791. package/dist/search/rerank.d.ts +3 -1
  792. package/dist/search/rerank.d.ts.map +1 -1
  793. package/dist/search/rerank.js +44 -35
  794. package/dist/search/rerank.js.map +1 -1
  795. package/dist/search/reranker/authority-boost.d.ts +3 -0
  796. package/dist/search/reranker/authority-boost.d.ts.map +1 -0
  797. package/dist/search/reranker/authority-boost.js +179 -0
  798. package/dist/search/reranker/authority-boost.js.map +1 -0
  799. package/dist/search/reranker/consensus-boost.d.ts +3 -0
  800. package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
  801. package/dist/search/reranker/consensus-boost.js +27 -0
  802. package/dist/search/reranker/consensus-boost.js.map +1 -0
  803. package/dist/search/reranker/recency-boost.d.ts +3 -0
  804. package/dist/search/reranker/recency-boost.d.ts.map +1 -0
  805. package/dist/search/reranker/recency-boost.js +13 -0
  806. package/dist/search/reranker/recency-boost.js.map +1 -0
  807. package/dist/search/reranker/recency.d.ts +3 -0
  808. package/dist/search/reranker/recency.d.ts.map +1 -0
  809. package/dist/search/reranker/recency.js +23 -0
  810. package/dist/search/reranker/recency.js.map +1 -0
  811. package/dist/search/reranker/transformers-rerank-provider.d.ts +12 -0
  812. package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
  813. package/dist/search/reranker/transformers-rerank-provider.js +78 -0
  814. package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
  815. package/dist/search/rrf.d.ts +17 -0
  816. package/dist/search/rrf.d.ts.map +1 -0
  817. package/dist/search/rrf.js +39 -0
  818. package/dist/search/rrf.js.map +1 -0
  819. package/dist/search/sampling.d.ts +25 -0
  820. package/dist/search/sampling.d.ts.map +1 -0
  821. package/dist/search/sampling.js +52 -0
  822. package/dist/search/sampling.js.map +1 -0
  823. package/dist/search/searxng.d.ts.map +1 -1
  824. package/dist/search/searxng.js +69 -79
  825. package/dist/search/searxng.js.map +1 -1
  826. package/dist/search/tokens.d.ts +3 -0
  827. package/dist/search/tokens.d.ts.map +1 -0
  828. package/dist/search/tokens.js +39 -0
  829. package/dist/search/tokens.js.map +1 -0
  830. package/dist/search/truncate.d.ts +6 -0
  831. package/dist/search/truncate.d.ts.map +1 -0
  832. package/dist/search/truncate.js +26 -0
  833. package/dist/search/truncate.js.map +1 -0
  834. package/dist/search/url-unwrap.d.ts +3 -0
  835. package/dist/search/url-unwrap.d.ts.map +1 -0
  836. package/dist/search/url-unwrap.js +43 -0
  837. package/dist/search/url-unwrap.js.map +1 -0
  838. package/dist/search/v1/context-rank.d.ts +13 -0
  839. package/dist/search/v1/context-rank.d.ts.map +1 -0
  840. package/dist/search/v1/context-rank.js +74 -0
  841. package/dist/search/v1/context-rank.js.map +1 -0
  842. package/dist/search/v1/engine-base.d.ts +27 -0
  843. package/dist/search/v1/engine-base.d.ts.map +1 -0
  844. package/dist/search/v1/engine-base.js +110 -0
  845. package/dist/search/v1/engine-base.js.map +1 -0
  846. package/dist/search/v1/intent-router.d.ts +22 -0
  847. package/dist/search/v1/intent-router.d.ts.map +1 -0
  848. package/dist/search/v1/intent-router.js +138 -0
  849. package/dist/search/v1/intent-router.js.map +1 -0
  850. package/dist/search/v1/orchestrator.d.ts +24 -0
  851. package/dist/search/v1/orchestrator.d.ts.map +1 -0
  852. package/dist/search/v1/orchestrator.js +163 -0
  853. package/dist/search/v1/orchestrator.js.map +1 -0
  854. package/dist/search/v1/recency-boost.d.ts +9 -0
  855. package/dist/search/v1/recency-boost.d.ts.map +1 -0
  856. package/dist/search/v1/recency-boost.js +37 -0
  857. package/dist/search/v1/recency-boost.js.map +1 -0
  858. package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
  859. package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
  860. package/dist/search/v1/recent-cache-dedup.js +85 -0
  861. package/dist/search/v1/recent-cache-dedup.js.map +1 -0
  862. package/dist/search/v1/rss/feed-config.d.ts +21 -0
  863. package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
  864. package/dist/search/v1/rss/feed-config.js +90 -0
  865. package/dist/search/v1/rss/feed-config.js.map +1 -0
  866. package/dist/search/v1/rss/feed-parser.d.ts +14 -0
  867. package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
  868. package/dist/search/v1/rss/feed-parser.js +104 -0
  869. package/dist/search/v1/rss/feed-parser.js.map +1 -0
  870. package/dist/search/v1/rss/feed-poller.d.ts +22 -0
  871. package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
  872. package/dist/search/v1/rss/feed-poller.js +102 -0
  873. package/dist/search/v1/rss/feed-poller.js.map +1 -0
  874. package/dist/search/v1/rss/feed-store.d.ts +30 -0
  875. package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
  876. package/dist/search/v1/rss/feed-store.js +134 -0
  877. package/dist/search/v1/rss/feed-store.js.map +1 -0
  878. package/dist/search/v1/rss/rss-engine.d.ts +6 -0
  879. package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
  880. package/dist/search/v1/rss/rss-engine.js +28 -0
  881. package/dist/search/v1/rss/rss-engine.js.map +1 -0
  882. package/dist/search/v1/v1-provider.d.ts +7 -0
  883. package/dist/search/v1/v1-provider.d.ts.map +1 -0
  884. package/dist/search/v1/v1-provider.js +68 -0
  885. package/dist/search/v1/v1-provider.js.map +1 -0
  886. package/dist/search/v1/verticals/code.d.ts +4 -0
  887. package/dist/search/v1/verticals/code.d.ts.map +1 -0
  888. package/dist/search/v1/verticals/code.js +20 -0
  889. package/dist/search/v1/verticals/code.js.map +1 -0
  890. package/dist/search/v1/verticals/docs.d.ts +4 -0
  891. package/dist/search/v1/verticals/docs.d.ts.map +1 -0
  892. package/dist/search/v1/verticals/docs.js +20 -0
  893. package/dist/search/v1/verticals/docs.js.map +1 -0
  894. package/dist/search/v1/verticals/general.d.ts +4 -0
  895. package/dist/search/v1/verticals/general.d.ts.map +1 -0
  896. package/dist/search/v1/verticals/general.js +22 -0
  897. package/dist/search/v1/verticals/general.js.map +1 -0
  898. package/dist/search/v1/verticals/news.d.ts +10 -0
  899. package/dist/search/v1/verticals/news.d.ts.map +1 -0
  900. package/dist/search/v1/verticals/news.js +52 -0
  901. package/dist/search/v1/verticals/news.js.map +1 -0
  902. package/dist/search/v1/verticals/papers.d.ts +4 -0
  903. package/dist/search/v1/verticals/papers.d.ts.map +1 -0
  904. package/dist/search/v1/verticals/papers.js +23 -0
  905. package/dist/search/v1/verticals/papers.js.map +1 -0
  906. package/dist/search/validator.js +31 -31
  907. package/dist/search/validator.js.map +1 -1
  908. package/dist/searxng/bootstrap.d.ts +30 -0
  909. package/dist/searxng/bootstrap.d.ts.map +1 -1
  910. package/dist/searxng/bootstrap.js +223 -85
  911. package/dist/searxng/bootstrap.js.map +1 -1
  912. package/dist/searxng/docker.d.ts.map +1 -1
  913. package/dist/searxng/docker.js +69 -60
  914. package/dist/searxng/docker.js.map +1 -1
  915. package/dist/searxng/process.d.ts +13 -1
  916. package/dist/searxng/process.d.ts.map +1 -1
  917. package/dist/searxng/process.js +231 -164
  918. package/dist/searxng/process.js.map +1 -1
  919. package/dist/server/backend-status.d.ts +13 -0
  920. package/dist/server/backend-status.d.ts.map +1 -0
  921. package/dist/server/backend-status.js +40 -0
  922. package/dist/server/backend-status.js.map +1 -0
  923. package/dist/server/tool-schemas.d.ts +549 -0
  924. package/dist/server/tool-schemas.d.ts.map +1 -0
  925. package/dist/server/tool-schemas.js +464 -0
  926. package/dist/server/tool-schemas.js.map +1 -0
  927. package/dist/server/warmup-on-start.d.ts +9 -0
  928. package/dist/server/warmup-on-start.d.ts.map +1 -0
  929. package/dist/server/warmup-on-start.js +55 -0
  930. package/dist/server/warmup-on-start.js.map +1 -0
  931. package/dist/server.d.ts +17 -0
  932. package/dist/server.d.ts.map +1 -1
  933. package/dist/server.js +454 -297
  934. package/dist/server.js.map +1 -1
  935. package/dist/tools/agent.d.ts +5 -0
  936. package/dist/tools/agent.d.ts.map +1 -0
  937. package/dist/tools/agent.js +128 -0
  938. package/dist/tools/agent.js.map +1 -0
  939. package/dist/tools/cache.d.ts +2 -1
  940. package/dist/tools/cache.d.ts.map +1 -1
  941. package/dist/tools/cache.js +175 -44
  942. package/dist/tools/cache.js.map +1 -1
  943. package/dist/tools/crawl.d.ts.map +1 -1
  944. package/dist/tools/crawl.js +171 -88
  945. package/dist/tools/crawl.js.map +1 -1
  946. package/dist/tools/extract.d.ts +2 -2
  947. package/dist/tools/extract.d.ts.map +1 -1
  948. package/dist/tools/extract.js +175 -59
  949. package/dist/tools/extract.js.map +1 -1
  950. package/dist/tools/fetch.d.ts +2 -2
  951. package/dist/tools/fetch.d.ts.map +1 -1
  952. package/dist/tools/fetch.js +161 -68
  953. package/dist/tools/fetch.js.map +1 -1
  954. package/dist/tools/find-similar.d.ts +5 -0
  955. package/dist/tools/find-similar.d.ts.map +1 -0
  956. package/dist/tools/find-similar.js +127 -0
  957. package/dist/tools/find-similar.js.map +1 -0
  958. package/dist/tools/research.d.ts +5 -0
  959. package/dist/tools/research.d.ts.map +1 -0
  960. package/dist/tools/research.js +107 -0
  961. package/dist/tools/research.js.map +1 -0
  962. package/dist/tools/search.d.ts +10 -2
  963. package/dist/tools/search.d.ts.map +1 -1
  964. package/dist/tools/search.js +13 -158
  965. package/dist/tools/search.js.map +1 -1
  966. package/dist/types.d.ts +350 -7
  967. package/dist/types.d.ts.map +1 -1
  968. package/dist/types.js +6 -1
  969. package/dist/types.js.map +1 -1
  970. package/dist/util/mode.d.ts +4 -0
  971. package/dist/util/mode.d.ts.map +1 -0
  972. package/dist/util/mode.js +34 -0
  973. package/dist/util/mode.js.map +1 -0
  974. package/package.json +78 -8
  975. package/dist/extraction/trafilatura.d.ts +0 -6
  976. package/dist/extraction/trafilatura.d.ts.map +0 -1
  977. package/dist/extraction/trafilatura.js +0 -105
  978. package/dist/extraction/trafilatura.js.map +0 -1
  979. package/dist/search/flashrank.d.ts +0 -12
  980. package/dist/search/flashrank.d.ts.map +0 -1
  981. package/dist/search/flashrank.js +0 -63
  982. package/dist/search/flashrank.js.map +0 -1
@@ -1,202 +1,307 @@
1
- import { createHash } from 'node:crypto';
2
- import { getDatabase } from './db.js';
3
- import { getConfig } from '../config.js';
4
- const TRACKING_PARAMS = new Set([
5
- 'utm_source',
6
- 'utm_medium',
7
- 'utm_campaign',
8
- 'utm_content',
9
- 'utm_term',
10
- 'utm_id',
11
- 'fbclid',
12
- 'gclid',
13
- 'msclkid',
14
- 'mc_cid',
15
- 'mc_eid',
1
+ import { createHash } from "node:crypto";
2
+ import { getDatabase } from "./db.js";
3
+ import { getConfig } from "../config.js";
4
+ import { createLogger } from "../logger.js";
5
+ const log = createLogger("cache");
6
+ function sanitizeFtsQuery(q) {
7
+ const trimmed = q.trim();
8
+ if (!trimmed) return "";
9
+ if (/^".*"$/.test(trimmed)) return trimmed;
10
+ const tokens = trimmed.match(/"[^"]*"|\S+/g) ?? [];
11
+ const RESERVED = /* @__PURE__ */ new Set(["AND", "OR", "NOT", "(", ")"]);
12
+ return tokens.map((tok) => {
13
+ if (tok.startsWith('"') && tok.endsWith('"')) return tok;
14
+ if (RESERVED.has(tok)) return tok;
15
+ if (/^\w+\*?$/.test(tok)) return tok;
16
+ return `"${tok.replace(/"/g, '""')}"`;
17
+ }).join(" ");
18
+ }
19
+ const TRACKING_PARAMS = /* @__PURE__ */ new Set([
20
+ "utm_source",
21
+ "utm_medium",
22
+ "utm_campaign",
23
+ "utm_content",
24
+ "utm_term",
25
+ "utm_id",
26
+ "fbclid",
27
+ "gclid",
28
+ "msclkid",
29
+ "mc_cid",
30
+ "mc_eid"
16
31
  ]);
17
- export function normalizeUrl(url) {
18
- const parsed = new URL(url);
19
- parsed.protocol = parsed.protocol.toLowerCase();
20
- parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\./, '');
21
- for (const key of [...parsed.searchParams.keys()]) {
22
- if (TRACKING_PARAMS.has(key) || key.startsWith('utm_')) {
23
- parsed.searchParams.delete(key);
24
- }
25
- }
26
- parsed.searchParams.sort();
27
- let result = parsed.toString();
28
- // Strip trailing slash from path (but not root)
29
- if (parsed.pathname !== '/' && result.endsWith('/')) {
30
- result = result.slice(0, -1);
32
+ function normalizeUrl(url) {
33
+ const parsed = new URL(url);
34
+ parsed.protocol = parsed.protocol.toLowerCase();
35
+ parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
36
+ for (const key of [...parsed.searchParams.keys()]) {
37
+ if (TRACKING_PARAMS.has(key) || key.startsWith("utm_")) {
38
+ parsed.searchParams.delete(key);
31
39
  }
32
- // Remove trailing slash from origin-only URLs too
33
- if (parsed.pathname === '/' && !parsed.search && !parsed.hash) {
34
- result = result.replace(/\/$/, '');
35
- }
36
- return result;
40
+ }
41
+ parsed.searchParams.sort();
42
+ let result = parsed.toString();
43
+ if (parsed.pathname !== "/" && result.endsWith("/")) {
44
+ result = result.slice(0, -1);
45
+ }
46
+ if (parsed.pathname === "/" && !parsed.search && !parsed.hash) {
47
+ result = result.replace(/\/$/, "");
48
+ }
49
+ return result;
37
50
  }
38
51
  function toIsoSeconds(date) {
39
- return date.toISOString().replace('T', ' ').replace(/\.\d+Z$/, '');
52
+ return date.toISOString().replace("T", " ").replace(/\.\d+Z$/, "");
40
53
  }
41
- export function cacheContent(result, extraction) {
54
+ function cacheContent(result, extraction) {
55
+ try {
42
56
  const db = getDatabase();
43
57
  const config = getConfig();
44
58
  const normalizedUrl = normalizeUrl(result.finalUrl || result.url);
45
- const contentHash = createHash('sha256').update(extraction.markdown).digest('hex');
46
- const now = new Date();
47
- const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1000);
59
+ const contentHash = createHash("sha256").update(extraction.markdown).digest("hex");
60
+ const now = /* @__PURE__ */ new Date();
61
+ const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1e3);
48
62
  const stmt = db.prepare(`
49
- INSERT OR REPLACE INTO url_cache (
50
- url, normalized_url, title, markdown, raw_html,
51
- metadata, links, images, fetch_method, extractor_used,
52
- content_hash, fetched_at, expires_at
53
- )
54
- VALUES (
55
- @url, @normalizedUrl, @title, @markdown, @rawHtml,
56
- @metadata, @links, @images, @fetchMethod, @extractorUsed,
57
- @contentHash, @fetchedAt, @expiresAt
58
- )
59
- `);
63
+ INSERT OR REPLACE INTO url_cache (
64
+ url, normalized_url, title, markdown, raw_html,
65
+ metadata, links, images, fetch_method, extractor_used,
66
+ content_hash, fetched_at, expires_at
67
+ )
68
+ VALUES (
69
+ @url, @normalizedUrl, @title, @markdown, @rawHtml,
70
+ @metadata, @links, @images, @fetchMethod, @extractorUsed,
71
+ @contentHash, @fetchedAt, @expiresAt
72
+ )
73
+ `);
60
74
  stmt.run({
61
- url: result.url,
62
- normalizedUrl,
63
- title: extraction.title,
64
- markdown: extraction.markdown,
65
- rawHtml: result.html,
66
- metadata: JSON.stringify(extraction.metadata),
67
- links: JSON.stringify(extraction.links),
68
- images: JSON.stringify(extraction.images),
69
- fetchMethod: result.method,
70
- extractorUsed: extraction.extractor,
71
- contentHash: contentHash,
72
- fetchedAt: toIsoSeconds(now),
73
- expiresAt: toIsoSeconds(expiresAt),
75
+ url: result.url,
76
+ normalizedUrl,
77
+ title: extraction.title,
78
+ markdown: extraction.markdown,
79
+ rawHtml: result.html,
80
+ metadata: JSON.stringify(extraction.metadata),
81
+ links: JSON.stringify(extraction.links),
82
+ images: JSON.stringify(extraction.images),
83
+ fetchMethod: result.method,
84
+ extractorUsed: extraction.extractor,
85
+ contentHash,
86
+ fetchedAt: toIsoSeconds(now),
87
+ expiresAt: toIsoSeconds(expiresAt)
74
88
  });
89
+ } catch (err) {
90
+ log.warn("cacheContent failed", {
91
+ url: result.url,
92
+ finalUrl: result.finalUrl,
93
+ error: err instanceof Error ? err.message : String(err)
94
+ });
95
+ }
75
96
  }
76
97
  function rowToCachedContent(row) {
77
- return {
78
- id: row.id,
79
- url: row.url,
80
- normalizedUrl: row.normalized_url,
81
- title: row.title,
82
- markdown: row.markdown,
83
- rawHtml: row.raw_html,
84
- metadata: row.metadata,
85
- links: row.links,
86
- images: row.images,
87
- fetchMethod: row.fetch_method,
88
- extractorUsed: row.extractor_used,
89
- contentHash: row.content_hash,
90
- fetchedAt: row.fetched_at,
91
- expiresAt: row.expires_at,
92
- };
98
+ return {
99
+ id: row.id,
100
+ url: row.url,
101
+ normalizedUrl: row.normalized_url,
102
+ title: row.title,
103
+ markdown: row.markdown,
104
+ rawHtml: row.raw_html,
105
+ metadata: row.metadata,
106
+ links: row.links,
107
+ images: row.images,
108
+ fetchMethod: row.fetch_method,
109
+ extractorUsed: row.extractor_used,
110
+ contentHash: row.content_hash,
111
+ fetchedAt: row.fetched_at,
112
+ expiresAt: row.expires_at
113
+ };
93
114
  }
94
- export function getCachedContent(url) {
95
- const db = getDatabase();
96
- const normalizedUrl = normalizeUrl(url);
97
- const row = db.prepare(`
115
+ function getCachedContent(url) {
116
+ const db = getDatabase();
117
+ const normalizedUrl = normalizeUrl(url);
118
+ const row = db.prepare(`
98
119
  SELECT * FROM url_cache WHERE url = ? OR normalized_url = ? LIMIT 1
99
120
  `).get(url, normalizedUrl);
100
- return row ? rowToCachedContent(row) : null;
121
+ return row ? rowToCachedContent(row) : null;
101
122
  }
102
- export function isExpired(cached) {
103
- if (!cached.expiresAt)
104
- return false;
105
- return new Date(cached.expiresAt).getTime() < Date.now();
123
+ function getCachedContentByNormalizedUrl(normalizedUrl) {
124
+ const db = getDatabase();
125
+ const row = db.prepare(
126
+ "SELECT * FROM url_cache WHERE normalized_url = ? LIMIT 1"
127
+ ).get(normalizedUrl);
128
+ return row ? rowToCachedContent(row) : null;
106
129
  }
107
- export function searchCache(query) {
108
- const db = getDatabase();
109
- const rows = db.prepare(`
130
+ function getHashForNormalizedUrl(normalizedUrl) {
131
+ const db = getDatabase();
132
+ const row = db.prepare(
133
+ "SELECT content_hash FROM url_cache WHERE normalized_url = ? LIMIT 1"
134
+ ).get(normalizedUrl);
135
+ return row?.content_hash ?? null;
136
+ }
137
+ function getMarkdownForNormalizedUrl(normalizedUrl) {
138
+ const db = getDatabase();
139
+ const row = db.prepare(
140
+ "SELECT markdown FROM url_cache WHERE normalized_url = ? LIMIT 1"
141
+ ).get(normalizedUrl);
142
+ return row ? row.markdown : null;
143
+ }
144
+ function isExpired(cached) {
145
+ if (!cached.expiresAt) return false;
146
+ return new Date(cached.expiresAt).getTime() < Date.now();
147
+ }
148
+ function isCacheUsable(cached, opts = {}) {
149
+ if (!cached.expiresAt) return { usable: true, stale: false };
150
+ const expiresMs = new Date(cached.expiresAt).getTime();
151
+ const now = Date.now();
152
+ if (expiresMs >= now) return { usable: true, stale: false };
153
+ const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1e3;
154
+ if (now - expiresMs <= staleMaxMs) return { usable: true, stale: true };
155
+ return { usable: false, stale: false };
156
+ }
157
+ function searchCache(query) {
158
+ const db = getDatabase();
159
+ const rows = db.prepare(`
110
160
  SELECT url_cache.*
111
161
  FROM url_cache
112
162
  JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid
113
163
  WHERE url_cache_fts MATCH ?
114
164
  ORDER BY rank
115
- `).all(query);
116
- return rows.map(rowToCachedContent);
165
+ `).all(sanitizeFtsQuery(query));
166
+ return rows.map(rowToCachedContent);
117
167
  }
118
- export function cacheSearchResults(query, results, enginesUsed) {
119
- const db = getDatabase();
120
- const config = getConfig();
121
- const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');
122
- const now = new Date();
123
- const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1000);
124
- const stmt = db.prepare(`
168
+ function cacheSearchResults(query, results, enginesUsed) {
169
+ const db = getDatabase();
170
+ const config = getConfig();
171
+ const queryHash = createHash("sha256").update(query.toLowerCase().trim()).digest("hex");
172
+ const now = /* @__PURE__ */ new Date();
173
+ const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1e3);
174
+ const stmt = db.prepare(`
125
175
  INSERT OR REPLACE INTO search_cache (query, query_hash, results, engines_used, searched_at, expires_at)
126
176
  VALUES (@query, @queryHash, @results, @enginesUsed, @searchedAt, @expiresAt)
127
177
  `);
128
- stmt.run({
129
- query,
130
- queryHash,
131
- results: JSON.stringify(results),
132
- enginesUsed: JSON.stringify(enginesUsed),
133
- searchedAt: toIsoSeconds(now),
134
- expiresAt: toIsoSeconds(expiresAt),
135
- });
178
+ stmt.run({
179
+ query,
180
+ queryHash,
181
+ results: JSON.stringify(results),
182
+ enginesUsed: JSON.stringify(enginesUsed),
183
+ searchedAt: toIsoSeconds(now),
184
+ expiresAt: toIsoSeconds(expiresAt)
185
+ });
136
186
  }
137
- export function getCachedSearchResults(query) {
138
- const db = getDatabase();
139
- const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');
140
- const row = db.prepare(`
141
- SELECT * FROM search_cache WHERE query_hash = ? AND (expires_at IS NULL OR expires_at > datetime('now'))
142
- `).get(queryHash);
143
- if (!row)
144
- return null;
145
- return {
187
+ function getCachedSearchResults(query, opts = {}) {
188
+ const db = getDatabase();
189
+ const queryHash = createHash("sha256").update(query.toLowerCase().trim()).digest("hex");
190
+ const row = db.prepare(
191
+ "SELECT query, results, engines_used, searched_at, expires_at FROM search_cache WHERE query_hash = ? LIMIT 1"
192
+ ).get(queryHash);
193
+ if (!row) return null;
194
+ if (row.expires_at) {
195
+ const expiresMs = new Date(row.expires_at).getTime();
196
+ const now = Date.now();
197
+ if (expiresMs < now) {
198
+ const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1e3;
199
+ if (now - expiresMs > staleMaxMs) return null;
200
+ return {
146
201
  query: row.query,
147
202
  results: JSON.parse(row.results),
148
203
  engines_used: JSON.parse(row.engines_used),
149
204
  searched_at: row.searched_at,
150
- };
151
- }
152
- export function searchCacheFiltered(options) {
153
- const db = getDatabase();
154
- const conditions = [];
155
- const params = [];
156
- let fromClause = 'url_cache';
157
- if (options.query) {
158
- fromClause = 'url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid';
159
- conditions.push('url_cache_fts MATCH ?');
160
- params.push(options.query);
205
+ stale: true
206
+ };
161
207
  }
162
- if (options.urlPattern) {
163
- conditions.push('url_cache.normalized_url GLOB ?');
164
- params.push(options.urlPattern);
165
- }
166
- if (options.since) {
167
- conditions.push('url_cache.fetched_at > datetime(?)');
168
- params.push(options.since);
169
- }
170
- const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
171
- const orderClause = options.query ? 'ORDER BY rank' : 'ORDER BY url_cache.fetched_at DESC';
172
- const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT 100`;
173
- const rows = db.prepare(sql).all(...params);
174
- return rows.map(rowToCachedContent);
208
+ }
209
+ return {
210
+ query: row.query,
211
+ results: JSON.parse(row.results),
212
+ engines_used: JSON.parse(row.engines_used),
213
+ searched_at: row.searched_at
214
+ };
175
215
  }
176
- export function clearCacheEntries(options) {
177
- const db = getDatabase();
178
- const conditions = [];
179
- const params = [];
180
- if (options.query) {
181
- conditions.push('id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)');
182
- params.push(options.query);
183
- }
184
- if (options.urlPattern) {
185
- conditions.push('normalized_url GLOB ?');
186
- params.push(options.urlPattern);
187
- }
188
- if (options.since) {
189
- conditions.push('fetched_at > datetime(?)');
190
- params.push(options.since);
191
- }
192
- const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
193
- const sql = `DELETE FROM url_cache ${whereClause}`;
194
- const result = db.prepare(sql).run(...params);
195
- return result.changes;
216
+ function searchCacheFiltered(options) {
217
+ const db = getDatabase();
218
+ const conditions = [];
219
+ const params = [];
220
+ let fromClause = "url_cache";
221
+ if (options.query) {
222
+ fromClause = "url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid";
223
+ conditions.push("url_cache_fts MATCH ?");
224
+ params.push(sanitizeFtsQuery(options.query));
225
+ }
226
+ if (options.urlPattern) {
227
+ conditions.push("url_cache.normalized_url GLOB ?");
228
+ params.push(options.urlPattern);
229
+ }
230
+ if (options.since) {
231
+ conditions.push("url_cache.fetched_at > datetime(?)");
232
+ params.push(options.since);
233
+ }
234
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
235
+ const orderClause = options.query ? "ORDER BY rank" : "ORDER BY url_cache.fetched_at DESC";
236
+ const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT 100`;
237
+ const rows = db.prepare(sql).all(...params);
238
+ return rows.map(rowToCachedContent);
196
239
  }
197
- export function getCacheStats() {
198
- const db = getDatabase();
199
- const row = db.prepare(`
240
+ function ftsSearchRanked(query, limit) {
241
+ if (!query.trim() || limit <= 0) return [];
242
+ const db = getDatabase();
243
+ const rows = db.prepare(`
244
+ SELECT url_cache.normalized_url AS url, url_cache_fts.rank AS rank
245
+ FROM url_cache
246
+ JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid
247
+ WHERE url_cache_fts MATCH ?
248
+ ORDER BY url_cache_fts.rank
249
+ LIMIT ?
250
+ `).all(sanitizeFtsQuery(query), limit);
251
+ return rows.map((r) => ({ url: r.url, score: -r.rank }));
252
+ }
253
+ function clearCacheEntries(options) {
254
+ const db = getDatabase();
255
+ const conditions = [];
256
+ const params = [];
257
+ if (options.query) {
258
+ conditions.push(
259
+ "id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)"
260
+ );
261
+ params.push(sanitizeFtsQuery(options.query));
262
+ }
263
+ if (options.urlPattern) {
264
+ conditions.push("normalized_url GLOB ?");
265
+ params.push(options.urlPattern);
266
+ }
267
+ if (options.since) {
268
+ conditions.push("fetched_at > datetime(?)");
269
+ params.push(options.since);
270
+ }
271
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
272
+ const sql = `DELETE FROM url_cache ${whereClause}`;
273
+ const result = db.prepare(sql).run(...params);
274
+ return result.changes;
275
+ }
276
+ function countCachedUrlsForDomain(domain) {
277
+ const db = getDatabase();
278
+ const normalized = domain.toLowerCase().replace(/^www\./, "");
279
+ const stmt = db.prepare(`
280
+ SELECT COUNT(*) AS n FROM url_cache
281
+ WHERE url LIKE 'http://' || ? || '/%'
282
+ OR url LIKE 'https://' || ? || '/%'
283
+ OR url LIKE 'http://www.' || ? || '/%'
284
+ OR url LIKE 'https://www.' || ? || '/%'
285
+ OR url = 'http://' || ?
286
+ OR url = 'https://' || ?
287
+ OR url = 'http://www.' || ?
288
+ OR url = 'https://www.' || ?
289
+ `);
290
+ const row = stmt.get(
291
+ normalized,
292
+ normalized,
293
+ normalized,
294
+ normalized,
295
+ normalized,
296
+ normalized,
297
+ normalized,
298
+ normalized
299
+ );
300
+ return row.n;
301
+ }
302
+ function getCacheStats() {
303
+ const db = getDatabase();
304
+ const row = db.prepare(`
200
305
  SELECT
201
306
  COUNT(*) as total_urls,
202
307
  COALESCE(SUM(LENGTH(markdown) + LENGTH(COALESCE(raw_html, ''))), 0) as total_bytes,
@@ -204,11 +309,100 @@ export function getCacheStats() {
204
309
  MAX(fetched_at) as newest
205
310
  FROM url_cache
206
311
  `).get();
312
+ return {
313
+ total_urls: row.total_urls,
314
+ total_size_mb: Math.round(row.total_bytes / (1024 * 1024) * 1e6) / 1e6,
315
+ oldest: row.oldest ?? "",
316
+ newest: row.newest ?? ""
317
+ };
318
+ }
319
+ function updateCacheEmbedding(url, embedding, model, dims) {
320
+ try {
321
+ const db = getDatabase();
322
+ let normalized;
323
+ try {
324
+ normalized = normalizeUrl(url);
325
+ } catch {
326
+ normalized = url;
327
+ }
328
+ const result = db.prepare(`
329
+ UPDATE url_cache
330
+ SET embedding = ?, embedding_model = ?, embedding_dims = ?, updated_at = datetime('now')
331
+ WHERE normalized_url = ?
332
+ `).run(embedding, model, dims, normalized);
333
+ return result.changes > 0;
334
+ } catch {
335
+ return false;
336
+ }
337
+ }
338
+ function getEmbeddingForUrl(url, modelId) {
339
+ try {
340
+ const db = getDatabase();
341
+ let normalized;
342
+ try {
343
+ normalized = normalizeUrl(url);
344
+ } catch {
345
+ normalized = url;
346
+ }
347
+ const row = db.prepare(`
348
+ SELECT embedding, embedding_model, embedding_dims
349
+ FROM url_cache
350
+ WHERE (url = ? OR normalized_url = ?) AND embedding IS NOT NULL
351
+ LIMIT 1
352
+ `).get(url, normalized);
353
+ if (!row) return null;
354
+ if (modelId !== void 0 && row.embedding_model !== modelId) return null;
207
355
  return {
208
- total_urls: row.total_urls,
209
- total_size_mb: Math.round((row.total_bytes / (1024 * 1024)) * 1e6) / 1e6,
210
- oldest: row.oldest ?? '',
211
- newest: row.newest ?? '',
356
+ embedding: row.embedding,
357
+ model: row.embedding_model,
358
+ dims: row.embedding_dims
212
359
  };
360
+ } catch {
361
+ return null;
362
+ }
363
+ }
364
+ function getAllEmbeddings(modelId) {
365
+ try {
366
+ const db = getDatabase();
367
+ const rows = modelId !== void 0 ? db.prepare(`
368
+ SELECT normalized_url, embedding, embedding_model, embedding_dims
369
+ FROM url_cache
370
+ WHERE embedding IS NOT NULL AND embedding_model = ?
371
+ `).all(modelId) : db.prepare(`
372
+ SELECT normalized_url, embedding, embedding_model, embedding_dims
373
+ FROM url_cache
374
+ WHERE embedding IS NOT NULL
375
+ `).all();
376
+ return rows.map((r) => ({
377
+ normalizedUrl: r.normalized_url,
378
+ embedding: r.embedding,
379
+ model: r.embedding_model,
380
+ dims: r.embedding_dims
381
+ }));
382
+ } catch {
383
+ return [];
384
+ }
213
385
  }
386
+ export {
387
+ cacheContent,
388
+ cacheSearchResults,
389
+ clearCacheEntries,
390
+ countCachedUrlsForDomain,
391
+ ftsSearchRanked,
392
+ getAllEmbeddings,
393
+ getCacheStats,
394
+ getCachedContent,
395
+ getCachedContentByNormalizedUrl,
396
+ getCachedSearchResults,
397
+ getEmbeddingForUrl,
398
+ getHashForNormalizedUrl,
399
+ getMarkdownForNormalizedUrl,
400
+ isCacheUsable,
401
+ isExpired,
402
+ normalizeUrl,
403
+ sanitizeFtsQuery,
404
+ searchCache,
405
+ searchCacheFiltered,
406
+ updateCacheEmbedding
407
+ };
214
408
  //# sourceMappingURL=store.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cache/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,aAAa;IACb,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE5B,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEtE,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAClD,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAE3B,IAAI,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAE/B,gDAAgD;IAChD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,kDAAkD;IAClD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAU;IAC9B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAsB,EAAE,UAA4B;IAC/E,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAE1E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;GAWvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,aAAa;QACb,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;QACzC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,aAAa,EAAE,UAAU,CAAC,SAAS;QACnC,WAAW,EAAE,WAAW;QACxB,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC;QAC5B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAmBD,SAAS,kBAAkB,CAAC,GAAU;IACpC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,aAAa,EAAE,GAAG,CAAC,cAAc;QACjC,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,YAA4C;QAC7D,aAAa,EAAE,GAAG,CAAC,cAAgD;QACnE,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAsB,CAAC;IAEhD,OAAO,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAqB;IAC7C,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;GAMvB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAY,CAAC;IAEzB,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AASD,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAA2B,EAC3B,WAAqB;IAErB,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAGvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,KAAK;QACL,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACxC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC;QAC7B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa;IAClD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAExF,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,SAAS,CAA8F,CAAC;IAE/G,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC;QAC1C,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAInC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,WAAW,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,GAAG,oEAAoE,CAAC;QAClF,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,oCAAoC,CAAC;IAE3F,MAAM,GAAG,GAAG,2BAA2B,UAAU,IAAI,WAAW,IAAI,WAAW,YAAY,CAAC;IAC5F,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAY,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAIjC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CACb,iIAAiI,CAClI,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,GAAG,GAAG,yBAAyB,WAAW,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;GAOtB,CAAC,CAAC,GAAG,EAA+F,CAAC;IAEtG,OAAO;QACL,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACxE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;KACzB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"sources":["../../src/cache/store.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { getDatabase } from './db.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\nimport type { RawFetchResult, ExtractionResult, CachedContent, SearchResultItem, CacheStats } from '../types.js';\n\nconst log = createLogger('cache');\n\n/**\n * Sanitize a user query for sqlite FTS5 MATCH.\n *\n * Why: bare tokens with `.` / `-` / `/` / `:` / digits-with-dot\n * (e.g. \"5.4\", \"x-y\", \"https://foo\") raise `fts5: syntax error near \".\"`.\n * Quoting tokens that aren't pure word-chars lets FTS5 treat them as phrases.\n * Already-quoted phrases and explicit operators (AND/OR/NOT/parens) pass through.\n */\nexport function sanitizeFtsQuery(q: string): string {\n const trimmed = q.trim();\n if (!trimmed) return '';\n if (/^\".*\"$/.test(trimmed)) return trimmed;\n const tokens = trimmed.match(/\"[^\"]*\"|\\S+/g) ?? [];\n const RESERVED = new Set(['AND', 'OR', 'NOT', '(', ')']);\n return tokens.map(tok => {\n if (tok.startsWith('\"') && tok.endsWith('\"')) return tok;\n if (RESERVED.has(tok)) return tok;\n if (/^\\w+\\*?$/.test(tok)) return tok;\n return `\"${tok.replace(/\"/g, '\"\"')}\"`;\n }).join(' ');\n}\n\nconst TRACKING_PARAMS = new Set([\n 'utm_source',\n 'utm_medium',\n 'utm_campaign',\n 'utm_content',\n 'utm_term',\n 'utm_id',\n 'fbclid',\n 'gclid',\n 'msclkid',\n 'mc_cid',\n 'mc_eid',\n]);\n\nexport function normalizeUrl(url: string): string {\n const parsed = new URL(url);\n\n parsed.protocol = parsed.protocol.toLowerCase();\n parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\\./, '');\n\n for (const key of [...parsed.searchParams.keys()]) {\n if (TRACKING_PARAMS.has(key) || key.startsWith('utm_')) {\n parsed.searchParams.delete(key);\n }\n }\n\n parsed.searchParams.sort();\n\n let result = parsed.toString();\n\n // Strip trailing slash from path (but not root)\n if (parsed.pathname !== '/' && result.endsWith('/')) {\n result = result.slice(0, -1);\n }\n // Remove trailing slash from origin-only URLs too\n if (parsed.pathname === '/' && !parsed.search && !parsed.hash) {\n result = result.replace(/\\/$/, '');\n }\n\n return result;\n}\n\nfunction toIsoSeconds(date: Date): string {\n return date.toISOString().replace('T', ' ').replace(/\\.\\d+Z$/, '');\n}\n\nexport function cacheContent(result: RawFetchResult, extraction: ExtractionResult): void {\n try {\n const db = getDatabase();\n const config = getConfig();\n\n const normalizedUrl = normalizeUrl(result.finalUrl || result.url);\n const contentHash = createHash('sha256').update(extraction.markdown).digest('hex');\n\n const now = new Date();\n const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1000);\n\n const stmt = db.prepare(`\n INSERT OR REPLACE INTO url_cache (\n url, normalized_url, title, markdown, raw_html,\n metadata, links, images, fetch_method, extractor_used,\n content_hash, fetched_at, expires_at\n )\n VALUES (\n @url, @normalizedUrl, @title, @markdown, @rawHtml,\n @metadata, @links, @images, @fetchMethod, @extractorUsed,\n @contentHash, @fetchedAt, @expiresAt\n )\n `);\n\n stmt.run({\n url: result.url,\n normalizedUrl,\n title: extraction.title,\n markdown: extraction.markdown,\n rawHtml: result.html,\n metadata: JSON.stringify(extraction.metadata),\n links: JSON.stringify(extraction.links),\n images: JSON.stringify(extraction.images),\n fetchMethod: result.method,\n extractorUsed: extraction.extractor,\n contentHash: contentHash,\n fetchedAt: toIsoSeconds(now),\n expiresAt: toIsoSeconds(expiresAt),\n });\n } catch (err) {\n log.warn('cacheContent failed', {\n url: result.url,\n finalUrl: result.finalUrl,\n error: err instanceof Error ? err.message : String(err),\n });\n }\n}\n\ninterface DbRow {\n id: number;\n url: string;\n normalized_url: string;\n title: string;\n markdown: string;\n raw_html: string;\n metadata: string;\n links: string;\n images: string;\n fetch_method: string;\n extractor_used: string;\n content_hash: string;\n fetched_at: string;\n expires_at: string | null;\n}\n\nfunction rowToCachedContent(row: DbRow): CachedContent {\n return {\n id: row.id,\n url: row.url,\n normalizedUrl: row.normalized_url,\n title: row.title,\n markdown: row.markdown,\n rawHtml: row.raw_html,\n metadata: row.metadata,\n links: row.links,\n images: row.images,\n fetchMethod: row.fetch_method as CachedContent['fetchMethod'],\n extractorUsed: row.extractor_used as CachedContent['extractorUsed'],\n contentHash: row.content_hash,\n fetchedAt: row.fetched_at,\n expiresAt: row.expires_at,\n };\n}\n\nexport function getCachedContent(url: string): CachedContent | null {\n const db = getDatabase();\n const normalizedUrl = normalizeUrl(url);\n\n const row = db.prepare(`\n SELECT * FROM url_cache WHERE url = ? OR normalized_url = ? LIMIT 1\n `).get(url, normalizedUrl) as DbRow | undefined;\n\n return row ? rowToCachedContent(row) : null;\n}\n\nexport function getCachedContentByNormalizedUrl(normalizedUrl: string): CachedContent | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT * FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as DbRow | undefined;\n return row ? rowToCachedContent(row) : null;\n}\n\nexport function getHashForNormalizedUrl(normalizedUrl: string): string | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT content_hash FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as { content_hash: string } | undefined;\n return row?.content_hash ?? null;\n}\n\nexport function getMarkdownForNormalizedUrl(normalizedUrl: string): string | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT markdown FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as { markdown: string } | undefined;\n return row ? row.markdown : null;\n}\n\nexport function isExpired(cached: CachedContent): boolean {\n if (!cached.expiresAt) return false;\n return new Date(cached.expiresAt).getTime() < Date.now();\n}\n\nexport interface CacheLookupOptions {\n staleMaxSeconds?: number;\n}\n\nexport function isCacheUsable(\n cached: CachedContent,\n opts: CacheLookupOptions = {},\n): { usable: boolean; stale: boolean } {\n if (!cached.expiresAt) return { usable: true, stale: false };\n const expiresMs = new Date(cached.expiresAt).getTime();\n const now = Date.now();\n if (expiresMs >= now) return { usable: true, stale: false };\n const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1000;\n if (now - expiresMs <= staleMaxMs) return { usable: true, stale: true };\n return { usable: false, stale: false };\n}\n\nexport function searchCache(query: string): CachedContent[] {\n const db = getDatabase();\n\n const rows = db.prepare(`\n SELECT url_cache.*\n FROM url_cache\n JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid\n WHERE url_cache_fts MATCH ?\n ORDER BY rank\n `).all(sanitizeFtsQuery(query)) as DbRow[];\n\n return rows.map(rowToCachedContent);\n}\n\nexport interface CachedSearchResult {\n query: string;\n results: SearchResultItem[];\n engines_used: string[];\n searched_at: string;\n stale?: boolean;\n}\n\nexport function cacheSearchResults(\n query: string,\n results: SearchResultItem[],\n enginesUsed: string[],\n): void {\n const db = getDatabase();\n const config = getConfig();\n\n const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');\n const now = new Date();\n const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1000);\n\n const stmt = db.prepare(`\n INSERT OR REPLACE INTO search_cache (query, query_hash, results, engines_used, searched_at, expires_at)\n VALUES (@query, @queryHash, @results, @enginesUsed, @searchedAt, @expiresAt)\n `);\n\n stmt.run({\n query,\n queryHash,\n results: JSON.stringify(results),\n enginesUsed: JSON.stringify(enginesUsed),\n searchedAt: toIsoSeconds(now),\n expiresAt: toIsoSeconds(expiresAt),\n });\n}\n\nexport function getCachedSearchResults(\n query: string,\n opts: CacheLookupOptions = {},\n): CachedSearchResult | null {\n const db = getDatabase();\n const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');\n\n const row = db.prepare(\n 'SELECT query, results, engines_used, searched_at, expires_at FROM search_cache WHERE query_hash = ? LIMIT 1',\n ).get(queryHash) as\n | { query: string; results: string; engines_used: string; searched_at: string; expires_at: string | null }\n | undefined;\n\n if (!row) return null;\n\n if (row.expires_at) {\n const expiresMs = new Date(row.expires_at).getTime();\n const now = Date.now();\n if (expiresMs < now) {\n const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1000;\n if (now - expiresMs > staleMaxMs) return null;\n return {\n query: row.query,\n results: JSON.parse(row.results) as SearchResultItem[],\n engines_used: JSON.parse(row.engines_used) as string[],\n searched_at: row.searched_at,\n stale: true,\n };\n }\n }\n\n return {\n query: row.query,\n results: JSON.parse(row.results) as SearchResultItem[],\n engines_used: JSON.parse(row.engines_used) as string[],\n searched_at: row.searched_at,\n };\n}\n\nexport function searchCacheFiltered(options: {\n query?: string;\n urlPattern?: string;\n since?: string;\n}): CachedContent[] {\n const db = getDatabase();\n const conditions: string[] = [];\n const params: unknown[] = [];\n let fromClause = 'url_cache';\n\n if (options.query) {\n fromClause = 'url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid';\n conditions.push('url_cache_fts MATCH ?');\n params.push(sanitizeFtsQuery(options.query));\n }\n\n if (options.urlPattern) {\n conditions.push('url_cache.normalized_url GLOB ?');\n params.push(options.urlPattern);\n }\n\n if (options.since) {\n conditions.push('url_cache.fetched_at > datetime(?)');\n params.push(options.since);\n }\n\n const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n const orderClause = options.query ? 'ORDER BY rank' : 'ORDER BY url_cache.fetched_at DESC';\n\n const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT 100`;\n const rows = db.prepare(sql).all(...params) as DbRow[];\n return rows.map(rowToCachedContent);\n}\n\n/**\n * BM25-ranked FTS5 search across cached pages. Returns normalized URLs\n * paired with their rank score. `rank` from FTS5 is negative (lower is\n * better in sqlite ordering), so we flip the sign to surface a \"higher is\n * better\" score for consumers (e.g. RRF input).\n */\nexport function ftsSearchRanked(query: string, limit: number): Array<{ url: string; score: number }> {\n if (!query.trim() || limit <= 0) return [];\n const db = getDatabase();\n const rows = db.prepare(`\n SELECT url_cache.normalized_url AS url, url_cache_fts.rank AS rank\n FROM url_cache\n JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid\n WHERE url_cache_fts MATCH ?\n ORDER BY url_cache_fts.rank\n LIMIT ?\n `).all(sanitizeFtsQuery(query), limit) as Array<{ url: string; rank: number }>;\n return rows.map(r => ({ url: r.url, score: -r.rank }));\n}\n\nexport function clearCacheEntries(options: {\n query?: string;\n urlPattern?: string;\n since?: string;\n}): number {\n const db = getDatabase();\n const conditions: string[] = [];\n const params: unknown[] = [];\n\n if (options.query) {\n conditions.push(\n 'id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)',\n );\n params.push(sanitizeFtsQuery(options.query));\n }\n\n if (options.urlPattern) {\n conditions.push('normalized_url GLOB ?');\n params.push(options.urlPattern);\n }\n\n if (options.since) {\n conditions.push('fetched_at > datetime(?)');\n params.push(options.since);\n }\n\n const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n const sql = `DELETE FROM url_cache ${whereClause}`;\n const result = db.prepare(sql).run(...params);\n return result.changes;\n}\n\n// Counts cached URLs for an exact host (apex scoping — `blog.example.com`\n// and `example.com` are NOT collapsed). Leading `www.` is stripped to align\n// with normalizeUrl.\nexport function countCachedUrlsForDomain(domain: string): number {\n const db = getDatabase();\n const normalized = domain.toLowerCase().replace(/^www\\./, '');\n const stmt = db.prepare(`\n SELECT COUNT(*) AS n FROM url_cache\n WHERE url LIKE 'http://' || ? || '/%'\n OR url LIKE 'https://' || ? || '/%'\n OR url LIKE 'http://www.' || ? || '/%'\n OR url LIKE 'https://www.' || ? || '/%'\n OR url = 'http://' || ?\n OR url = 'https://' || ?\n OR url = 'http://www.' || ?\n OR url = 'https://www.' || ?\n `);\n const row = stmt.get(\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n ) as { n: number };\n return row.n;\n}\n\nexport function getCacheStats(): CacheStats {\n const db = getDatabase();\n const row = db.prepare(`\n SELECT\n COUNT(*) as total_urls,\n COALESCE(SUM(LENGTH(markdown) + LENGTH(COALESCE(raw_html, ''))), 0) as total_bytes,\n MIN(fetched_at) as oldest,\n MAX(fetched_at) as newest\n FROM url_cache\n `).get() as { total_urls: number; total_bytes: number; oldest: string | null; newest: string | null };\n\n return {\n total_urls: row.total_urls,\n total_size_mb: Math.round((row.total_bytes / (1024 * 1024)) * 1e6) / 1e6,\n oldest: row.oldest ?? '',\n newest: row.newest ?? '',\n };\n}\n\n// --- Embedding store functions (Slice 22) ---\n\nexport function updateCacheEmbedding(\n url: string,\n embedding: Buffer,\n model: string,\n dims: number,\n): boolean {\n try {\n const db = getDatabase();\n let normalized: string;\n try {\n normalized = normalizeUrl(url);\n } catch {\n normalized = url;\n }\n\n const result = db.prepare(`\n UPDATE url_cache\n SET embedding = ?, embedding_model = ?, embedding_dims = ?, updated_at = datetime('now')\n WHERE normalized_url = ?\n `).run(embedding, model, dims, normalized);\n\n return result.changes > 0;\n } catch {\n return false;\n }\n}\n\nexport interface EmbeddingData {\n embedding: Buffer;\n model: string;\n dims: number;\n}\n\nexport function getEmbeddingForUrl(url: string, modelId?: string): EmbeddingData | null {\n try {\n const db = getDatabase();\n let normalized: string;\n try {\n normalized = normalizeUrl(url);\n } catch {\n normalized = url;\n }\n\n const row = db.prepare(`\n SELECT embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE (url = ? OR normalized_url = ?) AND embedding IS NOT NULL\n LIMIT 1\n `).get(url, normalized) as { embedding: Buffer; embedding_model: string; embedding_dims: number } | undefined;\n\n if (!row) return null;\n // Filter by modelId when caller wants only embeddings from the current\n // model; mismatched entries return null so they are treated as cache miss.\n if (modelId !== undefined && row.embedding_model !== modelId) return null;\n\n return {\n embedding: row.embedding,\n model: row.embedding_model,\n dims: row.embedding_dims,\n };\n } catch {\n return null;\n }\n}\n\nexport interface StoredEmbedding {\n normalizedUrl: string;\n embedding: Buffer;\n model: string;\n dims: number;\n}\n\nexport function getAllEmbeddings(modelId?: string): StoredEmbedding[] {\n try {\n const db = getDatabase();\n // Filter by modelId when provided so stale entries from a previous model\n // (different dim / vector space) are skipped — the in-memory vector index\n // requires matching dimensionality across all entries.\n const rows = modelId !== undefined\n ? db.prepare(`\n SELECT normalized_url, embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE embedding IS NOT NULL AND embedding_model = ?\n `).all(modelId) as Array<{\n normalized_url: string;\n embedding: Buffer;\n embedding_model: string;\n embedding_dims: number;\n }>\n : db.prepare(`\n SELECT normalized_url, embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE embedding IS NOT NULL\n `).all() as Array<{\n normalized_url: string;\n embedding: Buffer;\n embedding_model: string;\n embedding_dims: number;\n }>;\n\n return rows.map(r => ({\n normalizedUrl: r.normalized_url,\n embedding: r.embedding,\n model: r.embedding_model,\n dims: r.embedding_dims,\n }));\n } catch {\n return [];\n }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,mBAAmB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAUzB,SAAS,iBAAiB,GAAmB;AAClD,QAAM,UAAU,EAAE,KAAK;AACvB,MAAI,CAAC,QAAS,QAAO;AACrB,MAAI,SAAS,KAAK,OAAO,EAAG,QAAO;AACnC,QAAM,SAAS,QAAQ,MAAM,cAAc,KAAK,CAAC;AACjD,QAAM,WAAW,oBAAI,IAAI,CAAC,OAAO,MAAM,OAAO,KAAK,GAAG,CAAC;AACvD,SAAO,OAAO,IAAI,SAAO;AACvB,QAAI,IAAI,WAAW,GAAG,KAAK,IAAI,SAAS,GAAG,EAAG,QAAO;AACrD,QAAI,SAAS,IAAI,GAAG,EAAG,QAAO;AAC9B,QAAI,WAAW,KAAK,GAAG,EAAG,QAAO;AACjC,WAAO,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC;AAAA,EACpC,CAAC,EAAE,KAAK,GAAG;AACb;AAEA,MAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,SAAS,aAAa,KAAqB;AAChD,QAAM,SAAS,IAAI,IAAI,GAAG;AAE1B,SAAO,WAAW,OAAO,SAAS,YAAY;AAC9C,SAAO,WAAW,OAAO,SAAS,YAAY,EAAE,QAAQ,UAAU,EAAE;AAEpE,aAAW,OAAO,CAAC,GAAG,OAAO,aAAa,KAAK,CAAC,GAAG;AACjD,QAAI,gBAAgB,IAAI,GAAG,KAAK,IAAI,WAAW,MAAM,GAAG;AACtD,aAAO,aAAa,OAAO,GAAG;AAAA,IAChC;AAAA,EACF;AAEA,SAAO,aAAa,KAAK;AAEzB,MAAI,SAAS,OAAO,SAAS;AAG7B,MAAI,OAAO,aAAa,OAAO,OAAO,SAAS,GAAG,GAAG;AACnD,aAAS,OAAO,MAAM,GAAG,EAAE;AAAA,EAC7B;AAEA,MAAI,OAAO,aAAa,OAAO,CAAC,OAAO,UAAU,CAAC,OAAO,MAAM;AAC7D,aAAS,OAAO,QAAQ,OAAO,EAAE;AAAA,EACnC;AAEA,SAAO;AACT;AAEA,SAAS,aAAa,MAAoB;AACxC,SAAO,KAAK,YAAY,EAAE,QAAQ,KAAK,GAAG,EAAE,QAAQ,WAAW,EAAE;AACnE;AAEO,SAAS,aAAa,QAAwB,YAAoC;AACvF,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,UAAM,SAAS,UAAU;AAEzB,UAAM,gBAAgB,aAAa,OAAO,YAAY,OAAO,GAAG;AAChE,UAAM,cAAc,WAAW,QAAQ,EAAE,OAAO,WAAW,QAAQ,EAAE,OAAO,KAAK;AAEjF,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM,YAAY,IAAI,KAAK,IAAI,QAAQ,IAAI,OAAO,kBAAkB,GAAI;AAExE,UAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAWvB;AAED,SAAK,IAAI;AAAA,MACP,KAAK,OAAO;AAAA,MACZ;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,UAAU,WAAW;AAAA,MACrB,SAAS,OAAO;AAAA,MAChB,UAAU,KAAK,UAAU,WAAW,QAAQ;AAAA,MAC5C,OAAO,KAAK,UAAU,WAAW,KAAK;AAAA,MACtC,QAAQ,KAAK,UAAU,WAAW,MAAM;AAAA,MACxC,aAAa,OAAO;AAAA,MACpB,eAAe,WAAW;AAAA,MAC1B;AAAA,MACA,WAAW,aAAa,GAAG;AAAA,MAC3B,WAAW,aAAa,SAAS;AAAA,IACnC,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,QAAI,KAAK,uBAAuB;AAAA,MAC9B,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,MACjB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AAAA,EACH;AACF;AAmBA,SAAS,mBAAmB,KAA2B;AACrD,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,KAAK,IAAI;AAAA,IACT,eAAe,IAAI;AAAA,IACnB,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,OAAO,IAAI;AAAA,IACX,QAAQ,IAAI;AAAA,IACZ,aAAa,IAAI;AAAA,IACjB,eAAe,IAAI;AAAA,IACnB,aAAa,IAAI;AAAA,IACjB,WAAW,IAAI;AAAA,IACf,WAAW,IAAI;AAAA,EACjB;AACF;AAEO,SAAS,iBAAiB,KAAmC;AAClE,QAAM,KAAK,YAAY;AACvB,QAAM,gBAAgB,aAAa,GAAG;AAEtC,QAAM,MAAM,GAAG,QAAQ;AAAA;AAAA,GAEtB,EAAE,IAAI,KAAK,aAAa;AAEzB,SAAO,MAAM,mBAAmB,GAAG,IAAI;AACzC;AAEO,SAAS,gCAAgC,eAA6C;AAC3F,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,MAAM,mBAAmB,GAAG,IAAI;AACzC;AAEO,SAAS,wBAAwB,eAAsC;AAC5E,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,KAAK,gBAAgB;AAC9B;AAEO,SAAS,4BAA4B,eAAsC;AAChF,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,MAAM,IAAI,WAAW;AAC9B;AAEO,SAAS,UAAU,QAAgC;AACxD,MAAI,CAAC,OAAO,UAAW,QAAO;AAC9B,SAAO,IAAI,KAAK,OAAO,SAAS,EAAE,QAAQ,IAAI,KAAK,IAAI;AACzD;AAMO,SAAS,cACd,QACA,OAA2B,CAAC,GACS;AACrC,MAAI,CAAC,OAAO,UAAW,QAAO,EAAE,QAAQ,MAAM,OAAO,MAAM;AAC3D,QAAM,YAAY,IAAI,KAAK,OAAO,SAAS,EAAE,QAAQ;AACrD,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,aAAa,IAAK,QAAO,EAAE,QAAQ,MAAM,OAAO,MAAM;AAC1D,QAAM,cAAc,KAAK,mBAAmB,KAAK;AACjD,MAAI,MAAM,aAAa,WAAY,QAAO,EAAE,QAAQ,MAAM,OAAO,KAAK;AACtE,SAAO,EAAE,QAAQ,OAAO,OAAO,MAAM;AACvC;AAEO,SAAS,YAAY,OAAgC;AAC1D,QAAM,KAAK,YAAY;AAEvB,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAMvB,EAAE,IAAI,iBAAiB,KAAK,CAAC;AAE9B,SAAO,KAAK,IAAI,kBAAkB;AACpC;AAUO,SAAS,mBACd,OACA,SACA,aACM;AACN,QAAM,KAAK,YAAY;AACvB,QAAM,SAAS,UAAU;AAEzB,QAAM,YAAY,WAAW,QAAQ,EAAE,OAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK;AACtF,QAAM,MAAM,oBAAI,KAAK;AACrB,QAAM,YAAY,IAAI,KAAK,IAAI,QAAQ,IAAI,OAAO,iBAAiB,GAAI;AAEvE,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA,GAGvB;AAED,OAAK,IAAI;AAAA,IACP;AAAA,IACA;AAAA,IACA,SAAS,KAAK,UAAU,OAAO;AAAA,IAC/B,aAAa,KAAK,UAAU,WAAW;AAAA,IACvC,YAAY,aAAa,GAAG;AAAA,IAC5B,WAAW,aAAa,SAAS;AAAA,EACnC,CAAC;AACH;AAEO,SAAS,uBACd,OACA,OAA2B,CAAC,GACD;AAC3B,QAAM,KAAK,YAAY;AACvB,QAAM,YAAY,WAAW,QAAQ,EAAE,OAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK;AAEtF,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,SAAS;AAIf,MAAI,CAAC,IAAK,QAAO;AAEjB,MAAI,IAAI,YAAY;AAClB,UAAM,YAAY,IAAI,KAAK,IAAI,UAAU,EAAE,QAAQ;AACnD,UAAM,MAAM,KAAK,IAAI;AACrB,QAAI,YAAY,KAAK;AACnB,YAAM,cAAc,KAAK,mBAAmB,KAAK;AACjD,UAAI,MAAM,YAAY,WAAY,QAAO;AACzC,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,SAAS,KAAK,MAAM,IAAI,OAAO;AAAA,QAC/B,cAAc,KAAK,MAAM,IAAI,YAAY;AAAA,QACzC,aAAa,IAAI;AAAA,QACjB,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,KAAK,MAAM,IAAI,OAAO;AAAA,IAC/B,cAAc,KAAK,MAAM,IAAI,YAAY;AAAA,IACzC,aAAa,IAAI;AAAA,EACnB;AACF;AAEO,SAAS,oBAAoB,SAIhB;AAClB,QAAM,KAAK,YAAY;AACvB,QAAM,aAAuB,CAAC;AAC9B,QAAM,SAAoB,CAAC;AAC3B,MAAI,aAAa;AAEjB,MAAI,QAAQ,OAAO;AACjB,iBAAa;AACb,eAAW,KAAK,uBAAuB;AACvC,WAAO,KAAK,iBAAiB,QAAQ,KAAK,CAAC;AAAA,EAC7C;AAEA,MAAI,QAAQ,YAAY;AACtB,eAAW,KAAK,iCAAiC;AACjD,WAAO,KAAK,QAAQ,UAAU;AAAA,EAChC;AAEA,MAAI,QAAQ,OAAO;AACjB,eAAW,KAAK,oCAAoC;AACpD,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AAEA,QAAM,cAAc,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,CAAC,KAAK;AAClF,QAAM,cAAc,QAAQ,QAAQ,kBAAkB;AAEtD,QAAM,MAAM,2BAA2B,UAAU,IAAI,WAAW,IAAI,WAAW;AAC/E,QAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAC1C,SAAO,KAAK,IAAI,kBAAkB;AACpC;AAQO,SAAS,gBAAgB,OAAe,OAAsD;AACnG,MAAI,CAAC,MAAM,KAAK,KAAK,SAAS,EAAG,QAAO,CAAC;AACzC,QAAM,KAAK,YAAY;AACvB,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOvB,EAAE,IAAI,iBAAiB,KAAK,GAAG,KAAK;AACrC,SAAO,KAAK,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,OAAO,CAAC,EAAE,KAAK,EAAE;AACvD;AAEO,SAAS,kBAAkB,SAIvB;AACT,QAAM,KAAK,YAAY;AACvB,QAAM,aAAuB,CAAC;AAC9B,QAAM,SAAoB,CAAC;AAE3B,MAAI,QAAQ,OAAO;AACjB,eAAW;AAAA,MACT;AAAA,IACF;AACA,WAAO,KAAK,iBAAiB,QAAQ,KAAK,CAAC;AAAA,EAC7C;AAEA,MAAI,QAAQ,YAAY;AACtB,eAAW,KAAK,uBAAuB;AACvC,WAAO,KAAK,QAAQ,UAAU;AAAA,EAChC;AAEA,MAAI,QAAQ,OAAO;AACjB,eAAW,KAAK,0BAA0B;AAC1C,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AAEA,QAAM,cAAc,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,CAAC,KAAK;AAClF,QAAM,MAAM,yBAAyB,WAAW;AAChD,QAAM,SAAS,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAC5C,SAAO,OAAO;AAChB;AAKO,SAAS,yBAAyB,QAAwB;AAC/D,QAAM,KAAK,YAAY;AACvB,QAAM,aAAa,OAAO,YAAY,EAAE,QAAQ,UAAU,EAAE;AAC5D,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAUvB;AACD,QAAM,MAAM,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,IAAI;AACb;AAEO,SAAS,gBAA4B;AAC1C,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOtB,EAAE,IAAI;AAEP,SAAO;AAAA,IACL,YAAY,IAAI;AAAA,IAChB,eAAe,KAAK,MAAO,IAAI,eAAe,OAAO,QAAS,GAAG,IAAI;AAAA,IACrE,QAAQ,IAAI,UAAU;AAAA,IACtB,QAAQ,IAAI,UAAU;AAAA,EACxB;AACF;AAIO,SAAS,qBACd,KACA,WACA,OACA,MACS;AACT,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,QAAI;AACJ,QAAI;AACF,mBAAa,aAAa,GAAG;AAAA,IAC/B,QAAQ;AACN,mBAAa;AAAA,IACf;AAEA,UAAM,SAAS,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIzB,EAAE,IAAI,WAAW,OAAO,MAAM,UAAU;AAEzC,WAAO,OAAO,UAAU;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,mBAAmB,KAAa,SAAwC;AACtF,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,QAAI;AACJ,QAAI;AACF,mBAAa,aAAa,GAAG;AAAA,IAC/B,QAAQ;AACN,mBAAa;AAAA,IACf;AAEA,UAAM,MAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAKtB,EAAE,IAAI,KAAK,UAAU;AAEtB,QAAI,CAAC,IAAK,QAAO;AAGjB,QAAI,YAAY,UAAa,IAAI,oBAAoB,QAAS,QAAO;AAErE,WAAO;AAAA,MACL,WAAW,IAAI;AAAA,MACf,OAAO,IAAI;AAAA,MACX,MAAM,IAAI;AAAA,IACZ;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AASO,SAAS,iBAAiB,SAAqC;AACpE,MAAI;AACF,UAAM,KAAK,YAAY;AAIvB,UAAM,OAAO,YAAY,SACrB,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,SAIV,EAAE,IAAI,OAAO,IAMd,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,SAIV,EAAE,IAAI;AAOX,WAAO,KAAK,IAAI,QAAM;AAAA,MACpB,eAAe,EAAE;AAAA,MACjB,WAAW,EAAE;AAAA,MACb,OAAO,EAAE;AAAA,MACT,MAAM,EAAE;AAAA,IACV,EAAE;AAAA,EACJ,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;","names":[]}