@staticn0va/wigolo 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +146 -227
- package/SKILL.md +382 -0
- package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
- package/assets/blocks/claude-code/wigolo-command.md +40 -0
- package/assets/blocks/cursor/wigolo.mdc +46 -0
- package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
- package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
- package/assets/skills/wigolo/SKILL.md +50 -0
- package/assets/skills/wigolo/rules/cache-first.md +30 -0
- package/assets/skills/wigolo/rules/synthesis.md +43 -0
- package/assets/skills/wigolo-agent/SKILL.md +73 -0
- package/assets/skills/wigolo-crawl/SKILL.md +60 -0
- package/assets/skills/wigolo-extract/SKILL.md +59 -0
- package/assets/skills/wigolo-fetch/SKILL.md +65 -0
- package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
- package/assets/skills/wigolo-research/SKILL.md +77 -0
- package/assets/skills/wigolo-search/SKILL.md +78 -0
- package/dist/agent/executor.d.ts +33 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +233 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +238 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +13 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +271 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/agent/relevance.d.ts +15 -0
- package/dist/agent/relevance.d.ts.map +1 -0
- package/dist/agent/relevance.js +60 -0
- package/dist/agent/relevance.js.map +1 -0
- package/dist/cache/backfill-embeddings.d.ts +23 -0
- package/dist/cache/backfill-embeddings.d.ts.map +1 -0
- package/dist/cache/backfill-embeddings.js +105 -0
- package/dist/cache/backfill-embeddings.js.map +1 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/db.d.ts +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +94 -22
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +82 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/migrations/runner.d.ts +29 -0
- package/dist/cache/migrations/runner.d.ts.map +1 -0
- package/dist/cache/migrations/runner.js +147 -0
- package/dist/cache/migrations/runner.js.map +1 -0
- package/dist/cache/sqlite-vec-store.d.ts +42 -0
- package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
- package/dist/cache/sqlite-vec-store.js +176 -0
- package/dist/cache/sqlite-vec-store.js.map +1 -0
- package/dist/cache/store.d.ts +47 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +364 -168
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/agents/antigravity.d.ts +20 -0
- package/dist/cli/agents/antigravity.d.ts.map +1 -0
- package/dist/cli/agents/antigravity.js +49 -0
- package/dist/cli/agents/antigravity.js.map +1 -0
- package/dist/cli/agents/claude-code.d.ts +25 -0
- package/dist/cli/agents/claude-code.d.ts.map +1 -0
- package/dist/cli/agents/claude-code.js +111 -0
- package/dist/cli/agents/claude-code.js.map +1 -0
- package/dist/cli/agents/cursor.d.ts +21 -0
- package/dist/cli/agents/cursor.d.ts.map +1 -0
- package/dist/cli/agents/cursor.js +58 -0
- package/dist/cli/agents/cursor.js.map +1 -0
- package/dist/cli/agents/gemini-cli.d.ts +21 -0
- package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
- package/dist/cli/agents/gemini-cli.js +55 -0
- package/dist/cli/agents/gemini-cli.js.map +1 -0
- package/dist/cli/agents/registry.d.ts +21 -0
- package/dist/cli/agents/registry.d.ts.map +1 -0
- package/dist/cli/agents/registry.js +27 -0
- package/dist/cli/agents/registry.js.map +1 -0
- package/dist/cli/agents/utils.d.ts +26 -0
- package/dist/cli/agents/utils.d.ts.map +1 -0
- package/dist/cli/agents/utils.js +136 -0
- package/dist/cli/agents/utils.js.map +1 -0
- package/dist/cli/agents/vscode.d.ts +21 -0
- package/dist/cli/agents/vscode.d.ts.map +1 -0
- package/dist/cli/agents/vscode.js +62 -0
- package/dist/cli/agents/vscode.js.map +1 -0
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +94 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/backfill.d.ts +2 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +58 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +61 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/doctor.d.ts +8 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +344 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +42 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/help.d.ts +6 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +63 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +35 -7
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +201 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +185 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/setup-mcp.d.ts +2 -0
- package/dist/cli/setup-mcp.d.ts.map +1 -0
- package/dist/cli/setup-mcp.js +114 -0
- package/dist/cli/setup-mcp.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/shutdown.d.ts +2 -0
- package/dist/cli/shutdown.d.ts.map +1 -0
- package/dist/cli/shutdown.js +26 -0
- package/dist/cli/shutdown.js.map +1 -0
- package/dist/cli/status.d.ts +2 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +31 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/telemetry.d.ts +10 -0
- package/dist/cli/telemetry.d.ts.map +1 -0
- package/dist/cli/telemetry.js +56 -0
- package/dist/cli/telemetry.js.map +1 -0
- package/dist/cli/tui/agents-types.d.ts +28 -0
- package/dist/cli/tui/agents-types.d.ts.map +1 -0
- package/dist/cli/tui/agents-types.js +1 -0
- package/dist/cli/tui/agents-types.js.map +1 -0
- package/dist/cli/tui/agents.d.ts +11 -0
- package/dist/cli/tui/agents.d.ts.map +1 -0
- package/dist/cli/tui/agents.js +93 -0
- package/dist/cli/tui/agents.js.map +1 -0
- package/dist/cli/tui/banner.d.ts +3 -0
- package/dist/cli/tui/banner.d.ts.map +1 -0
- package/dist/cli/tui/banner.js +30 -0
- package/dist/cli/tui/banner.js.map +1 -0
- package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
- package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/AgentSelect.js +116 -0
- package/dist/cli/tui/components/AgentSelect.js.map +1 -0
- package/dist/cli/tui/components/Banner.d.ts +6 -0
- package/dist/cli/tui/components/Banner.d.ts.map +1 -0
- package/dist/cli/tui/components/Banner.js +25 -0
- package/dist/cli/tui/components/Banner.js.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.js +19 -0
- package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
- package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
- package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
- package/dist/cli/tui/components/InstallProgress.js +67 -0
- package/dist/cli/tui/components/InstallProgress.js.map +1 -0
- package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
- package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
- package/dist/cli/tui/components/SkillInstall.js +94 -0
- package/dist/cli/tui/components/SkillInstall.js.map +1 -0
- package/dist/cli/tui/components/Summary.d.ts +22 -0
- package/dist/cli/tui/components/Summary.d.ts.map +1 -0
- package/dist/cli/tui/components/Summary.js +135 -0
- package/dist/cli/tui/components/Summary.js.map +1 -0
- package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
- package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/components/SystemCheck.js +71 -0
- package/dist/cli/tui/components/SystemCheck.js.map +1 -0
- package/dist/cli/tui/components/Verification.d.ts +8 -0
- package/dist/cli/tui/components/Verification.d.ts.map +1 -0
- package/dist/cli/tui/components/Verification.js +63 -0
- package/dist/cli/tui/components/Verification.js.map +1 -0
- package/dist/cli/tui/config-writer-cli.d.ts +12 -0
- package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-cli.js +39 -0
- package/dist/cli/tui/config-writer-cli.js.map +1 -0
- package/dist/cli/tui/config-writer-json.d.ts +16 -0
- package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-json.js +86 -0
- package/dist/cli/tui/config-writer-json.js.map +1 -0
- package/dist/cli/tui/config-writer-toml.d.ts +16 -0
- package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-toml.js +83 -0
- package/dist/cli/tui/config-writer-toml.js.map +1 -0
- package/dist/cli/tui/config-writer.d.ts +25 -0
- package/dist/cli/tui/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/config-writer.js +101 -0
- package/dist/cli/tui/config-writer.js.map +1 -0
- package/dist/cli/tui/detect-helpers.d.ts +6 -0
- package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
- package/dist/cli/tui/detect-helpers.js +45 -0
- package/dist/cli/tui/detect-helpers.js.map +1 -0
- package/dist/cli/tui/extras-prompt.d.ts +7 -0
- package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
- package/dist/cli/tui/extras-prompt.js +42 -0
- package/dist/cli/tui/extras-prompt.js.map +1 -0
- package/dist/cli/tui/flags-types.d.ts +19 -0
- package/dist/cli/tui/flags-types.d.ts.map +1 -0
- package/dist/cli/tui/flags-types.js +23 -0
- package/dist/cli/tui/flags-types.js.map +1 -0
- package/dist/cli/tui/flags.d.ts +5 -0
- package/dist/cli/tui/flags.d.ts.map +1 -0
- package/dist/cli/tui/flags.js +132 -0
- package/dist/cli/tui/flags.js.map +1 -0
- package/dist/cli/tui/format.d.ts +14 -0
- package/dist/cli/tui/format.d.ts.map +1 -0
- package/dist/cli/tui/format.js +37 -0
- package/dist/cli/tui/format.js.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
- package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
- package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
- package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useInstall.js +90 -0
- package/dist/cli/tui/hooks/useInstall.js.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
- package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
- package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
- package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useVerify.js +71 -0
- package/dist/cli/tui/hooks/useVerify.js.map +1 -0
- package/dist/cli/tui/ink-init.d.ts +2 -0
- package/dist/cli/tui/ink-init.d.ts.map +1 -0
- package/dist/cli/tui/ink-init.js +198 -0
- package/dist/cli/tui/ink-init.js.map +1 -0
- package/dist/cli/tui/reporter-auto.d.ts +7 -0
- package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
- package/dist/cli/tui/reporter-auto.js +15 -0
- package/dist/cli/tui/reporter-auto.js.map +1 -0
- package/dist/cli/tui/reporter.d.ts +26 -0
- package/dist/cli/tui/reporter.d.ts.map +1 -0
- package/dist/cli/tui/reporter.js +32 -0
- package/dist/cli/tui/reporter.js.map +1 -0
- package/dist/cli/tui/run-command.d.ts +14 -0
- package/dist/cli/tui/run-command.d.ts.map +1 -0
- package/dist/cli/tui/run-command.js +72 -0
- package/dist/cli/tui/run-command.js.map +1 -0
- package/dist/cli/tui/select-agents.d.ts +6 -0
- package/dist/cli/tui/select-agents.d.ts.map +1 -0
- package/dist/cli/tui/select-agents.js +32 -0
- package/dist/cli/tui/select-agents.js.map +1 -0
- package/dist/cli/tui/status-agents.d.ts +11 -0
- package/dist/cli/tui/status-agents.d.ts.map +1 -0
- package/dist/cli/tui/status-agents.js +53 -0
- package/dist/cli/tui/status-agents.js.map +1 -0
- package/dist/cli/tui/status-cache.d.ts +6 -0
- package/dist/cli/tui/status-cache.d.ts.map +1 -0
- package/dist/cli/tui/status-cache.js +39 -0
- package/dist/cli/tui/status-cache.js.map +1 -0
- package/dist/cli/tui/status-format.d.ts +14 -0
- package/dist/cli/tui/status-format.d.ts.map +1 -0
- package/dist/cli/tui/status-format.js +41 -0
- package/dist/cli/tui/status-format.js.map +1 -0
- package/dist/cli/tui/status-python.d.ts +6 -0
- package/dist/cli/tui/status-python.d.ts.map +1 -0
- package/dist/cli/tui/status-python.js +30 -0
- package/dist/cli/tui/status-python.js.map +1 -0
- package/dist/cli/tui/system-check.d.ts +24 -0
- package/dist/cli/tui/system-check.d.ts.map +1 -0
- package/dist/cli/tui/system-check.js +103 -0
- package/dist/cli/tui/system-check.js.map +1 -0
- package/dist/cli/tui/tui-reporter.d.ts +19 -0
- package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
- package/dist/cli/tui/tui-reporter.js +95 -0
- package/dist/cli/tui/tui-reporter.js.map +1 -0
- package/dist/cli/tui/utils/config-writer.d.ts +3 -0
- package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/utils/config-writer.js +22 -0
- package/dist/cli/tui/utils/config-writer.js.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.js +11 -0
- package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
- package/dist/cli/tui/verify-suggestions.d.ts +5 -0
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
- package/dist/cli/tui/verify-suggestions.js +20 -0
- package/dist/cli/tui/verify-suggestions.js.map +1 -0
- package/dist/cli/tui/verify.d.ts +14 -0
- package/dist/cli/tui/verify.d.ts.map +1 -0
- package/dist/cli/tui/verify.js +101 -0
- package/dist/cli/tui/verify.js.map +1 -0
- package/dist/cli/tui/version.d.ts +2 -0
- package/dist/cli/tui/version.d.ts.map +1 -0
- package/dist/cli/tui/version.js +14 -0
- package/dist/cli/tui/version.js.map +1 -0
- package/dist/cli/uninstall.d.ts +2 -0
- package/dist/cli/uninstall.d.ts.map +1 -0
- package/dist/cli/uninstall.js +57 -0
- package/dist/cli/uninstall.js.map +1 -0
- package/dist/cli/warmup.d.ts +10 -2
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +226 -93
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +28 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +106 -56
- package/dist/config.js.map +1 -1
- package/dist/crawl/crawler.d.ts +6 -0
- package/dist/crawl/crawler.d.ts.map +1 -1
- package/dist/crawl/crawler.js +210 -209
- package/dist/crawl/crawler.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +124 -81
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/crawl/etag-incremental.d.ts +43 -0
- package/dist/crawl/etag-incremental.d.ts.map +1 -0
- package/dist/crawl/etag-incremental.js +94 -0
- package/dist/crawl/etag-incremental.js.map +1 -0
- package/dist/crawl/index-to-vec.d.ts +10 -0
- package/dist/crawl/index-to-vec.d.ts.map +1 -0
- package/dist/crawl/index-to-vec.js +44 -0
- package/dist/crawl/index-to-vec.js.map +1 -0
- package/dist/crawl/mapper.js +136 -164
- package/dist/crawl/mapper.js.map +1 -1
- package/dist/crawl/rate-limiter.js +63 -66
- package/dist/crawl/rate-limiter.js.map +1 -1
- package/dist/crawl/robots.js +58 -57
- package/dist/crawl/robots.js.map +1 -1
- package/dist/crawl/sitemap-first.d.ts +12 -0
- package/dist/crawl/sitemap-first.d.ts.map +1 -0
- package/dist/crawl/sitemap-first.js +47 -0
- package/dist/crawl/sitemap-first.js.map +1 -0
- package/dist/crawl/sitemap.js +33 -32
- package/dist/crawl/sitemap.js.map +1 -1
- package/dist/crawl/url-utils.d.ts +1 -0
- package/dist/crawl/url-utils.d.ts.map +1 -1
- package/dist/crawl/url-utils.js +49 -37
- package/dist/crawl/url-utils.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +33 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +275 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +93 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +233 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/fastembed-provider.d.ts +19 -0
- package/dist/embedding/fastembed-provider.d.ts.map +1 -0
- package/dist/embedding/fastembed-provider.js +51 -0
- package/dist/embedding/fastembed-provider.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +234 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +52 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +27 -23
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/extract.d.ts.map +1 -1
- package/dist/extraction/extract.js +76 -76
- package/dist/extraction/extract.js.map +1 -1
- package/dist/extraction/jsonld.js +50 -54
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +30 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +130 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown-sanitize.d.ts +2 -0
- package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
- package/dist/extraction/markdown-sanitize.js +151 -0
- package/dist/extraction/markdown-sanitize.js.map +1 -0
- package/dist/extraction/markdown.d.ts +11 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +195 -91
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts +8 -0
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +57 -91
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +28 -29
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +135 -72
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +81 -91
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +87 -95
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +46 -54
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +71 -80
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +173 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/extraction/structured.d.ts +4 -0
- package/dist/extraction/structured.d.ts.map +1 -0
- package/dist/extraction/structured.js +163 -0
- package/dist/extraction/structured.js.map +1 -0
- package/dist/extraction/v1/classifier.d.ts +3 -0
- package/dist/extraction/v1/classifier.d.ts.map +1 -0
- package/dist/extraction/v1/classifier.js +110 -0
- package/dist/extraction/v1/classifier.js.map +1 -0
- package/dist/extraction/v1/extract-provider.d.ts +16 -0
- package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
- package/dist/extraction/v1/extract-provider.js +43 -0
- package/dist/extraction/v1/extract-provider.js.map +1 -0
- package/dist/extraction/v1/local-llm.d.ts +8 -0
- package/dist/extraction/v1/local-llm.d.ts.map +1 -0
- package/dist/extraction/v1/local-llm.js +34 -0
- package/dist/extraction/v1/local-llm.js.map +1 -0
- package/dist/extraction/v1/news.d.ts +3 -0
- package/dist/extraction/v1/news.d.ts.map +1 -0
- package/dist/extraction/v1/news.js +61 -0
- package/dist/extraction/v1/news.js.map +1 -0
- package/dist/extraction/v1/product.d.ts +3 -0
- package/dist/extraction/v1/product.d.ts.map +1 -0
- package/dist/extraction/v1/product.js +166 -0
- package/dist/extraction/v1/product.js.map +1 -0
- package/dist/extraction/v1/recipe.d.ts +3 -0
- package/dist/extraction/v1/recipe.d.ts.map +1 -0
- package/dist/extraction/v1/recipe.js +136 -0
- package/dist/extraction/v1/recipe.js.map +1 -0
- package/dist/extraction/v1/routed.d.ts +17 -0
- package/dist/extraction/v1/routed.d.ts.map +1 -0
- package/dist/extraction/v1/routed.js +68 -0
- package/dist/extraction/v1/routed.js.map +1 -0
- package/dist/extraction/v1/schemas/Article.d.ts +11 -0
- package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Article.js +23 -0
- package/dist/extraction/v1/schemas/Article.js.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.js +122 -0
- package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
- package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
- package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Paper.js +156 -0
- package/dist/extraction/v1/schemas/Paper.js.map +1 -0
- package/dist/extraction/v1/schemas/Product.d.ts +17 -0
- package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Product.js +149 -0
- package/dist/extraction/v1/schemas/Product.js.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.js +160 -0
- package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
- package/dist/extraction/v1/schemas/index.d.ts +13 -0
- package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/index.js +44 -0
- package/dist/extraction/v1/schemas/index.js.map +1 -0
- package/dist/extraction/v1/site-extractors.d.ts +5 -0
- package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
- package/dist/extraction/v1/site-extractors.js +31 -0
- package/dist/extraction/v1/site-extractors.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +88 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +56 -26
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +303 -127
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +72 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +89 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/content-check.js +39 -46
- package/dist/fetch/content-check.js.map +1 -1
- package/dist/fetch/error-describe.d.ts +7 -0
- package/dist/fetch/error-describe.d.ts.map +1 -0
- package/dist/fetch/error-describe.js +37 -0
- package/dist/fetch/error-describe.js.map +1 -0
- package/dist/fetch/http-client.d.ts +4 -0
- package/dist/fetch/http-client.d.ts.map +1 -1
- package/dist/fetch/http-client.js +147 -128
- package/dist/fetch/http-client.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +174 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/fetch/playwright-tier.d.ts +19 -0
- package/dist/fetch/playwright-tier.d.ts.map +1 -0
- package/dist/fetch/playwright-tier.js +76 -0
- package/dist/fetch/playwright-tier.js.map +1 -0
- package/dist/fetch/router.d.ts +49 -3
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +187 -81
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +102 -17
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +31 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +245 -0
- package/dist/instructions.js.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.js +41 -0
- package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
- package/dist/integrations/cloud/llm/cache.d.ts +5 -0
- package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/cache.js +49 -0
- package/dist/integrations/cloud/llm/cache.js.map +1 -0
- package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
- package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/gemini.js +37 -0
- package/dist/integrations/cloud/llm/gemini.js.map +1 -0
- package/dist/integrations/cloud/llm/groq.d.ts +3 -0
- package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/groq.js +74 -0
- package/dist/integrations/cloud/llm/groq.js.map +1 -0
- package/dist/integrations/cloud/llm/hash.d.ts +3 -0
- package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/hash.js +26 -0
- package/dist/integrations/cloud/llm/hash.js.map +1 -0
- package/dist/integrations/cloud/llm/model-select.d.ts +5 -0
- package/dist/integrations/cloud/llm/model-select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/model-select.js +32 -0
- package/dist/integrations/cloud/llm/model-select.js.map +1 -0
- package/dist/integrations/cloud/llm/openai.d.ts +3 -0
- package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/openai.js +43 -0
- package/dist/integrations/cloud/llm/openai.js.map +1 -0
- package/dist/integrations/cloud/llm/run.d.ts +27 -0
- package/dist/integrations/cloud/llm/run.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/run.js +99 -0
- package/dist/integrations/cloud/llm/run.js.map +1 -0
- package/dist/integrations/cloud/llm/select.d.ts +5 -0
- package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/select.js +30 -0
- package/dist/integrations/cloud/llm/select.js.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts +19 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.js +103 -0
- package/dist/integrations/cloud/llm/text-adapters.js.map +1 -0
- package/dist/integrations/cloud/llm/types.d.ts +24 -0
- package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/types.js +1 -0
- package/dist/integrations/cloud/llm/types.js.map +1 -0
- package/dist/integrations/cloud/llm/validate.d.ts +6 -0
- package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/validate.js +63 -0
- package/dist/integrations/cloud/llm/validate.js.map +1 -0
- package/dist/logger.d.ts +4 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +71 -30
- package/dist/logger.js.map +1 -1
- package/dist/pdf-parse.d.js +1 -0
- package/dist/pdf-parse.d.js.map +1 -0
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +157 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +71 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +79 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/providers/embed-provider.d.ts +11 -0
- package/dist/providers/embed-provider.d.ts.map +1 -0
- package/dist/providers/embed-provider.js +24 -0
- package/dist/providers/embed-provider.js.map +1 -0
- package/dist/providers/extract-provider.d.ts +23 -0
- package/dist/providers/extract-provider.d.ts.map +1 -0
- package/dist/providers/extract-provider.js +25 -0
- package/dist/providers/extract-provider.js.map +1 -0
- package/dist/providers/rerank-provider.d.ts +17 -0
- package/dist/providers/rerank-provider.d.ts.map +1 -0
- package/dist/providers/rerank-provider.js +41 -0
- package/dist/providers/rerank-provider.js.map +1 -0
- package/dist/providers/search-provider.d.ts +25 -0
- package/dist/providers/search-provider.d.ts.map +1 -0
- package/dist/providers/search-provider.js +44 -0
- package/dist/providers/search-provider.js.map +1 -0
- package/dist/providers/vector-store.d.ts +27 -0
- package/dist/providers/vector-store.d.ts.map +1 -0
- package/dist/providers/vector-store.js +27 -0
- package/dist/providers/vector-store.js.map +1 -0
- package/dist/python-env.d.ts +9 -0
- package/dist/python-env.d.ts.map +1 -0
- package/dist/python-env.js +13 -0
- package/dist/python-env.js.map +1 -0
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +62 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +43 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +44 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +47 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +67 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +74 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +65 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +74 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +1 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +13 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +283 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +86 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +184 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/research/branch-exploration.d.ts +14 -0
- package/dist/research/branch-exploration.d.ts.map +1 -0
- package/dist/research/branch-exploration.js +100 -0
- package/dist/research/branch-exploration.js.map +1 -0
- package/dist/research/brief.d.ts +6 -0
- package/dist/research/brief.d.ts.map +1 -0
- package/dist/research/brief.js +246 -0
- package/dist/research/brief.js.map +1 -0
- package/dist/research/citation-graph.d.ts +9 -0
- package/dist/research/citation-graph.d.ts.map +1 -0
- package/dist/research/citation-graph.js +114 -0
- package/dist/research/citation-graph.js.map +1 -0
- package/dist/research/decompose.d.ts +14 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +439 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +269 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesis-local.d.ts +19 -0
- package/dist/research/synthesis-local.d.ts.map +1 -0
- package/dist/research/synthesis-local.js +62 -0
- package/dist/research/synthesis-local.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +137 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +33 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +244 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/search/dedup.d.ts +1 -0
- package/dist/search/dedup.d.ts.map +1 -1
- package/dist/search/dedup.js +40 -32
- package/dist/search/dedup.js.map +1 -1
- package/dist/search/engines/arxiv.d.ts +7 -0
- package/dist/search/engines/arxiv.d.ts.map +1 -0
- package/dist/search/engines/arxiv.js +70 -0
- package/dist/search/engines/arxiv.js.map +1 -0
- package/dist/search/engines/bing-news.d.ts +7 -0
- package/dist/search/engines/bing-news.d.ts.map +1 -0
- package/dist/search/engines/bing-news.js +97 -0
- package/dist/search/engines/bing-news.js.map +1 -0
- package/dist/search/engines/bing.d.ts +1 -0
- package/dist/search/engines/bing.d.ts.map +1 -1
- package/dist/search/engines/bing.js +100 -44
- package/dist/search/engines/bing.js.map +1 -1
- package/dist/search/engines/devdocs.d.ts +6 -0
- package/dist/search/engines/devdocs.d.ts.map +1 -0
- package/dist/search/engines/devdocs.js +56 -0
- package/dist/search/engines/devdocs.js.map +1 -0
- package/dist/search/engines/duckduckgo.d.ts.map +1 -1
- package/dist/search/engines/duckduckgo.js +56 -44
- package/dist/search/engines/duckduckgo.js.map +1 -1
- package/dist/search/engines/github-code.d.ts +7 -0
- package/dist/search/engines/github-code.d.ts.map +1 -0
- package/dist/search/engines/github-code.js +55 -0
- package/dist/search/engines/github-code.js.map +1 -0
- package/dist/search/engines/hn-algolia.d.ts +7 -0
- package/dist/search/engines/hn-algolia.d.ts.map +1 -0
- package/dist/search/engines/hn-algolia.js +76 -0
- package/dist/search/engines/hn-algolia.js.map +1 -0
- package/dist/search/engines/lobsters.d.ts +7 -0
- package/dist/search/engines/lobsters.d.ts.map +1 -0
- package/dist/search/engines/lobsters.js +83 -0
- package/dist/search/engines/lobsters.js.map +1 -0
- package/dist/search/engines/mdn.d.ts +7 -0
- package/dist/search/engines/mdn.d.ts.map +1 -0
- package/dist/search/engines/mdn.js +48 -0
- package/dist/search/engines/mdn.js.map +1 -0
- package/dist/search/engines/semantic-scholar.d.ts +7 -0
- package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
- package/dist/search/engines/semantic-scholar.js +69 -0
- package/dist/search/engines/semantic-scholar.js.map +1 -0
- package/dist/search/engines/stackoverflow.d.ts +7 -0
- package/dist/search/engines/stackoverflow.d.ts.map +1 -0
- package/dist/search/engines/stackoverflow.js +73 -0
- package/dist/search/engines/stackoverflow.js.map +1 -0
- package/dist/search/engines/startpage.d.ts.map +1 -1
- package/dist/search/engines/startpage.js +65 -46
- package/dist/search/engines/startpage.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +220 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/filters.d.ts.map +1 -1
- package/dist/search/filters.js +58 -54
- package/dist/search/filters.js.map +1 -1
- package/dist/search/find-similar/crawl-rank.d.ts +9 -0
- package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
- package/dist/search/find-similar/crawl-rank.js +272 -0
- package/dist/search/find-similar/crawl-rank.js.map +1 -0
- package/dist/search/find-similar/mode.d.ts +4 -0
- package/dist/search/find-similar/mode.d.ts.map +1 -0
- package/dist/search/find-similar/mode.js +12 -0
- package/dist/search/find-similar/mode.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +509 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/highlights.d.ts +19 -0
- package/dist/search/highlights.d.ts.map +1 -0
- package/dist/search/highlights.js +167 -0
- package/dist/search/highlights.js.map +1 -0
- package/dist/search/language-filter.d.ts +29 -0
- package/dist/search/language-filter.d.ts.map +1 -0
- package/dist/search/language-filter.js +126 -0
- package/dist/search/language-filter.js.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.js +501 -0
- package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
- package/dist/search/legacy/searxng-provider.d.ts +7 -0
- package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
- package/dist/search/legacy/searxng-provider.js +11 -0
- package/dist/search/legacy/searxng-provider.js.map +1 -0
- package/dist/search/multi-query.d.ts +25 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +228 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/query.js +32 -34
- package/dist/search/query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -1
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +44 -35
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/authority-boost.d.ts +3 -0
- package/dist/search/reranker/authority-boost.d.ts.map +1 -0
- package/dist/search/reranker/authority-boost.js +179 -0
- package/dist/search/reranker/authority-boost.js.map +1 -0
- package/dist/search/reranker/consensus-boost.d.ts +3 -0
- package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
- package/dist/search/reranker/consensus-boost.js +27 -0
- package/dist/search/reranker/consensus-boost.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +13 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +23 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts +13 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.js +94 -0
- package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +39 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +69 -79
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +39 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +6 -0
- package/dist/search/truncate.d.ts.map +1 -0
- package/dist/search/truncate.js +26 -0
- package/dist/search/truncate.js.map +1 -0
- package/dist/search/url-unwrap.d.ts +3 -0
- package/dist/search/url-unwrap.d.ts.map +1 -0
- package/dist/search/url-unwrap.js +43 -0
- package/dist/search/url-unwrap.js.map +1 -0
- package/dist/search/v1/context-rank.d.ts +13 -0
- package/dist/search/v1/context-rank.d.ts.map +1 -0
- package/dist/search/v1/context-rank.js +74 -0
- package/dist/search/v1/context-rank.js.map +1 -0
- package/dist/search/v1/engine-base.d.ts +27 -0
- package/dist/search/v1/engine-base.d.ts.map +1 -0
- package/dist/search/v1/engine-base.js +110 -0
- package/dist/search/v1/engine-base.js.map +1 -0
- package/dist/search/v1/intent-router.d.ts +22 -0
- package/dist/search/v1/intent-router.d.ts.map +1 -0
- package/dist/search/v1/intent-router.js +138 -0
- package/dist/search/v1/intent-router.js.map +1 -0
- package/dist/search/v1/orchestrator.d.ts +24 -0
- package/dist/search/v1/orchestrator.d.ts.map +1 -0
- package/dist/search/v1/orchestrator.js +163 -0
- package/dist/search/v1/orchestrator.js.map +1 -0
- package/dist/search/v1/recency-boost.d.ts +9 -0
- package/dist/search/v1/recency-boost.d.ts.map +1 -0
- package/dist/search/v1/recency-boost.js +37 -0
- package/dist/search/v1/recency-boost.js.map +1 -0
- package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
- package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
- package/dist/search/v1/recent-cache-dedup.js +85 -0
- package/dist/search/v1/recent-cache-dedup.js.map +1 -0
- package/dist/search/v1/rss/feed-config.d.ts +21 -0
- package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-config.js +90 -0
- package/dist/search/v1/rss/feed-config.js.map +1 -0
- package/dist/search/v1/rss/feed-parser.d.ts +14 -0
- package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-parser.js +104 -0
- package/dist/search/v1/rss/feed-parser.js.map +1 -0
- package/dist/search/v1/rss/feed-poller.d.ts +22 -0
- package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-poller.js +102 -0
- package/dist/search/v1/rss/feed-poller.js.map +1 -0
- package/dist/search/v1/rss/feed-store.d.ts +30 -0
- package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-store.js +134 -0
- package/dist/search/v1/rss/feed-store.js.map +1 -0
- package/dist/search/v1/rss/rss-engine.d.ts +6 -0
- package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
- package/dist/search/v1/rss/rss-engine.js +28 -0
- package/dist/search/v1/rss/rss-engine.js.map +1 -0
- package/dist/search/v1/v1-provider.d.ts +7 -0
- package/dist/search/v1/v1-provider.d.ts.map +1 -0
- package/dist/search/v1/v1-provider.js +68 -0
- package/dist/search/v1/v1-provider.js.map +1 -0
- package/dist/search/v1/verticals/code.d.ts +4 -0
- package/dist/search/v1/verticals/code.d.ts.map +1 -0
- package/dist/search/v1/verticals/code.js +20 -0
- package/dist/search/v1/verticals/code.js.map +1 -0
- package/dist/search/v1/verticals/docs.d.ts +4 -0
- package/dist/search/v1/verticals/docs.d.ts.map +1 -0
- package/dist/search/v1/verticals/docs.js +20 -0
- package/dist/search/v1/verticals/docs.js.map +1 -0
- package/dist/search/v1/verticals/general.d.ts +4 -0
- package/dist/search/v1/verticals/general.d.ts.map +1 -0
- package/dist/search/v1/verticals/general.js +22 -0
- package/dist/search/v1/verticals/general.js.map +1 -0
- package/dist/search/v1/verticals/news.d.ts +10 -0
- package/dist/search/v1/verticals/news.d.ts.map +1 -0
- package/dist/search/v1/verticals/news.js +52 -0
- package/dist/search/v1/verticals/news.js.map +1 -0
- package/dist/search/v1/verticals/papers.d.ts +4 -0
- package/dist/search/v1/verticals/papers.d.ts.map +1 -0
- package/dist/search/v1/verticals/papers.js +23 -0
- package/dist/search/v1/verticals/papers.js.map +1 -0
- package/dist/search/validator.js +31 -31
- package/dist/search/validator.js.map +1 -1
- package/dist/searxng/bootstrap.d.ts +30 -0
- package/dist/searxng/bootstrap.d.ts.map +1 -1
- package/dist/searxng/bootstrap.js +223 -85
- package/dist/searxng/bootstrap.js.map +1 -1
- package/dist/searxng/docker.d.ts.map +1 -1
- package/dist/searxng/docker.js +69 -60
- package/dist/searxng/docker.js.map +1 -1
- package/dist/searxng/process.d.ts +13 -1
- package/dist/searxng/process.d.ts.map +1 -1
- package/dist/searxng/process.js +231 -164
- package/dist/searxng/process.js.map +1 -1
- package/dist/server/backend-status.d.ts +13 -0
- package/dist/server/backend-status.d.ts.map +1 -0
- package/dist/server/backend-status.js +40 -0
- package/dist/server/backend-status.js.map +1 -0
- package/dist/server/tool-schemas.d.ts +549 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +464 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server/warmup-on-start.d.ts +9 -0
- package/dist/server/warmup-on-start.d.ts.map +1 -0
- package/dist/server/warmup-on-start.js +55 -0
- package/dist/server/warmup-on-start.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +454 -297
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +128 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +177 -44
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +171 -88
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts +2 -2
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +175 -59
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts +2 -2
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +174 -68
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +127 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +107 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +10 -2
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -158
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +350 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +6 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +34 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +78 -8
- package/dist/extraction/trafilatura.d.ts +0 -6
- package/dist/extraction/trafilatura.d.ts.map +0 -1
- package/dist/extraction/trafilatura.js +0 -105
- package/dist/extraction/trafilatura.js.map +0 -1
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -63
- package/dist/search/flashrank.js.map +0 -1
package/dist/cache/store.js
CHANGED
|
@@ -1,202 +1,309 @@
|
|
|
1
|
-
import { createHash } from
|
|
2
|
-
import { getDatabase } from
|
|
3
|
-
import { getConfig } from
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
'
|
|
14
|
-
|
|
15
|
-
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { getDatabase } from "./db.js";
|
|
3
|
+
import { getConfig } from "../config.js";
|
|
4
|
+
import { createLogger } from "../logger.js";
|
|
5
|
+
const log = createLogger("cache");
|
|
6
|
+
function sanitizeFtsQuery(q) {
|
|
7
|
+
const trimmed = q.trim();
|
|
8
|
+
if (!trimmed) return "";
|
|
9
|
+
if (/^".*"$/.test(trimmed)) return trimmed;
|
|
10
|
+
const tokens = trimmed.match(/"[^"]*"|\S+/g) ?? [];
|
|
11
|
+
const RESERVED = /* @__PURE__ */ new Set(["AND", "OR", "NOT", "(", ")"]);
|
|
12
|
+
return tokens.map((tok) => {
|
|
13
|
+
if (tok.startsWith('"') && tok.endsWith('"')) return tok;
|
|
14
|
+
if (RESERVED.has(tok)) return tok;
|
|
15
|
+
if (/^\w+\*?$/.test(tok)) return tok;
|
|
16
|
+
return `"${tok.replace(/"/g, '""')}"`;
|
|
17
|
+
}).join(" ");
|
|
18
|
+
}
|
|
19
|
+
const TRACKING_PARAMS = /* @__PURE__ */ new Set([
|
|
20
|
+
"utm_source",
|
|
21
|
+
"utm_medium",
|
|
22
|
+
"utm_campaign",
|
|
23
|
+
"utm_content",
|
|
24
|
+
"utm_term",
|
|
25
|
+
"utm_id",
|
|
26
|
+
"fbclid",
|
|
27
|
+
"gclid",
|
|
28
|
+
"msclkid",
|
|
29
|
+
"mc_cid",
|
|
30
|
+
"mc_eid"
|
|
16
31
|
]);
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
parsed.searchParams.sort();
|
|
27
|
-
let result = parsed.toString();
|
|
28
|
-
// Strip trailing slash from path (but not root)
|
|
29
|
-
if (parsed.pathname !== '/' && result.endsWith('/')) {
|
|
30
|
-
result = result.slice(0, -1);
|
|
32
|
+
function normalizeUrl(url) {
|
|
33
|
+
const parsed = new URL(url);
|
|
34
|
+
parsed.protocol = parsed.protocol.toLowerCase();
|
|
35
|
+
parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
|
|
36
|
+
for (const key of [...parsed.searchParams.keys()]) {
|
|
37
|
+
if (TRACKING_PARAMS.has(key) || key.startsWith("utm_")) {
|
|
38
|
+
parsed.searchParams.delete(key);
|
|
31
39
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
40
|
+
}
|
|
41
|
+
parsed.searchParams.sort();
|
|
42
|
+
let result = parsed.toString();
|
|
43
|
+
if (parsed.pathname !== "/" && result.endsWith("/")) {
|
|
44
|
+
result = result.slice(0, -1);
|
|
45
|
+
}
|
|
46
|
+
if (parsed.pathname === "/" && !parsed.search && !parsed.hash) {
|
|
47
|
+
result = result.replace(/\/$/, "");
|
|
48
|
+
}
|
|
49
|
+
return result;
|
|
37
50
|
}
|
|
38
51
|
function toIsoSeconds(date) {
|
|
39
|
-
|
|
52
|
+
return date.toISOString().replace("T", " ").replace(/\.\d+Z$/, "");
|
|
40
53
|
}
|
|
41
|
-
|
|
54
|
+
function cacheContent(result, extraction) {
|
|
55
|
+
try {
|
|
42
56
|
const db = getDatabase();
|
|
43
57
|
const config = getConfig();
|
|
44
58
|
const normalizedUrl = normalizeUrl(result.finalUrl || result.url);
|
|
45
|
-
const contentHash = createHash(
|
|
46
|
-
const now = new Date();
|
|
47
|
-
const expiresAt = new Date(now.getTime() + config.cacheTtlContent *
|
|
59
|
+
const contentHash = createHash("sha256").update(extraction.markdown).digest("hex");
|
|
60
|
+
const now = /* @__PURE__ */ new Date();
|
|
61
|
+
const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1e3);
|
|
48
62
|
const stmt = db.prepare(`
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
INSERT OR REPLACE INTO url_cache (
|
|
64
|
+
url, normalized_url, title, markdown, raw_html,
|
|
65
|
+
metadata, links, images, fetch_method, extractor_used,
|
|
66
|
+
content_hash, fetched_at, expires_at
|
|
67
|
+
)
|
|
68
|
+
VALUES (
|
|
69
|
+
@url, @normalizedUrl, @title, @markdown, @rawHtml,
|
|
70
|
+
@metadata, @links, @images, @fetchMethod, @extractorUsed,
|
|
71
|
+
@contentHash, @fetchedAt, @expiresAt
|
|
72
|
+
)
|
|
73
|
+
`);
|
|
60
74
|
stmt.run({
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
75
|
+
url: result.url,
|
|
76
|
+
normalizedUrl,
|
|
77
|
+
title: extraction.title,
|
|
78
|
+
markdown: extraction.markdown,
|
|
79
|
+
rawHtml: result.html,
|
|
80
|
+
metadata: JSON.stringify(extraction.metadata),
|
|
81
|
+
links: JSON.stringify(extraction.links),
|
|
82
|
+
images: JSON.stringify(extraction.images),
|
|
83
|
+
fetchMethod: result.method,
|
|
84
|
+
extractorUsed: extraction.extractor,
|
|
85
|
+
contentHash,
|
|
86
|
+
fetchedAt: toIsoSeconds(now),
|
|
87
|
+
expiresAt: toIsoSeconds(expiresAt)
|
|
74
88
|
});
|
|
89
|
+
} catch (err) {
|
|
90
|
+
log.warn("cacheContent failed", {
|
|
91
|
+
url: result.url,
|
|
92
|
+
finalUrl: result.finalUrl,
|
|
93
|
+
error: err instanceof Error ? err.message : String(err)
|
|
94
|
+
});
|
|
95
|
+
}
|
|
75
96
|
}
|
|
76
97
|
function rowToCachedContent(row) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
98
|
+
return {
|
|
99
|
+
id: row.id,
|
|
100
|
+
url: row.url,
|
|
101
|
+
normalizedUrl: row.normalized_url,
|
|
102
|
+
title: row.title,
|
|
103
|
+
markdown: row.markdown,
|
|
104
|
+
rawHtml: row.raw_html,
|
|
105
|
+
metadata: row.metadata,
|
|
106
|
+
links: row.links,
|
|
107
|
+
images: row.images,
|
|
108
|
+
fetchMethod: row.fetch_method,
|
|
109
|
+
extractorUsed: row.extractor_used,
|
|
110
|
+
contentHash: row.content_hash,
|
|
111
|
+
fetchedAt: row.fetched_at,
|
|
112
|
+
expiresAt: row.expires_at
|
|
113
|
+
};
|
|
93
114
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
115
|
+
function getCachedContent(url) {
|
|
116
|
+
const db = getDatabase();
|
|
117
|
+
const normalizedUrl = normalizeUrl(url);
|
|
118
|
+
const row = db.prepare(`
|
|
98
119
|
SELECT * FROM url_cache WHERE url = ? OR normalized_url = ? LIMIT 1
|
|
99
120
|
`).get(url, normalizedUrl);
|
|
100
|
-
|
|
121
|
+
return row ? rowToCachedContent(row) : null;
|
|
101
122
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
123
|
+
function getCachedContentByNormalizedUrl(normalizedUrl) {
|
|
124
|
+
const db = getDatabase();
|
|
125
|
+
const row = db.prepare(
|
|
126
|
+
"SELECT * FROM url_cache WHERE normalized_url = ? LIMIT 1"
|
|
127
|
+
).get(normalizedUrl);
|
|
128
|
+
return row ? rowToCachedContent(row) : null;
|
|
106
129
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
130
|
+
function getHashForNormalizedUrl(normalizedUrl) {
|
|
131
|
+
const db = getDatabase();
|
|
132
|
+
const row = db.prepare(
|
|
133
|
+
"SELECT content_hash FROM url_cache WHERE normalized_url = ? LIMIT 1"
|
|
134
|
+
).get(normalizedUrl);
|
|
135
|
+
return row?.content_hash ?? null;
|
|
136
|
+
}
|
|
137
|
+
function getMarkdownForNormalizedUrl(normalizedUrl) {
|
|
138
|
+
const db = getDatabase();
|
|
139
|
+
const row = db.prepare(
|
|
140
|
+
"SELECT markdown FROM url_cache WHERE normalized_url = ? LIMIT 1"
|
|
141
|
+
).get(normalizedUrl);
|
|
142
|
+
return row ? row.markdown : null;
|
|
143
|
+
}
|
|
144
|
+
function isExpired(cached) {
|
|
145
|
+
if (!cached.expiresAt) return false;
|
|
146
|
+
return new Date(cached.expiresAt).getTime() < Date.now();
|
|
147
|
+
}
|
|
148
|
+
function isCacheUsable(cached, opts = {}) {
|
|
149
|
+
if (!cached.expiresAt) return { usable: true, stale: false };
|
|
150
|
+
const expiresMs = new Date(cached.expiresAt).getTime();
|
|
151
|
+
const now = Date.now();
|
|
152
|
+
if (expiresMs >= now) return { usable: true, stale: false };
|
|
153
|
+
const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1e3;
|
|
154
|
+
if (now - expiresMs <= staleMaxMs) return { usable: true, stale: true };
|
|
155
|
+
return { usable: false, stale: false };
|
|
156
|
+
}
|
|
157
|
+
function searchCache(query) {
|
|
158
|
+
const db = getDatabase();
|
|
159
|
+
const rows = db.prepare(`
|
|
110
160
|
SELECT url_cache.*
|
|
111
161
|
FROM url_cache
|
|
112
162
|
JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid
|
|
113
163
|
WHERE url_cache_fts MATCH ?
|
|
114
164
|
ORDER BY rank
|
|
115
|
-
`).all(query);
|
|
116
|
-
|
|
165
|
+
`).all(sanitizeFtsQuery(query));
|
|
166
|
+
return rows.map(rowToCachedContent);
|
|
117
167
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
168
|
+
function cacheSearchResults(query, results, enginesUsed) {
|
|
169
|
+
const db = getDatabase();
|
|
170
|
+
const config = getConfig();
|
|
171
|
+
const queryHash = createHash("sha256").update(query.toLowerCase().trim()).digest("hex");
|
|
172
|
+
const now = /* @__PURE__ */ new Date();
|
|
173
|
+
const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1e3);
|
|
174
|
+
const stmt = db.prepare(`
|
|
125
175
|
INSERT OR REPLACE INTO search_cache (query, query_hash, results, engines_used, searched_at, expires_at)
|
|
126
176
|
VALUES (@query, @queryHash, @results, @enginesUsed, @searchedAt, @expiresAt)
|
|
127
177
|
`);
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
178
|
+
stmt.run({
|
|
179
|
+
query,
|
|
180
|
+
queryHash,
|
|
181
|
+
results: JSON.stringify(results),
|
|
182
|
+
enginesUsed: JSON.stringify(enginesUsed),
|
|
183
|
+
searchedAt: toIsoSeconds(now),
|
|
184
|
+
expiresAt: toIsoSeconds(expiresAt)
|
|
185
|
+
});
|
|
136
186
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
SELECT
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
187
|
+
function getCachedSearchResults(query, opts = {}) {
|
|
188
|
+
const db = getDatabase();
|
|
189
|
+
const queryHash = createHash("sha256").update(query.toLowerCase().trim()).digest("hex");
|
|
190
|
+
const row = db.prepare(
|
|
191
|
+
"SELECT query, results, engines_used, searched_at, expires_at FROM search_cache WHERE query_hash = ? LIMIT 1"
|
|
192
|
+
).get(queryHash);
|
|
193
|
+
if (!row) return null;
|
|
194
|
+
if (row.expires_at) {
|
|
195
|
+
const expiresMs = new Date(row.expires_at).getTime();
|
|
196
|
+
const now = Date.now();
|
|
197
|
+
if (expiresMs < now) {
|
|
198
|
+
const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1e3;
|
|
199
|
+
if (now - expiresMs > staleMaxMs) return null;
|
|
200
|
+
return {
|
|
146
201
|
query: row.query,
|
|
147
202
|
results: JSON.parse(row.results),
|
|
148
203
|
engines_used: JSON.parse(row.engines_used),
|
|
149
204
|
searched_at: row.searched_at,
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
export function searchCacheFiltered(options) {
|
|
153
|
-
const db = getDatabase();
|
|
154
|
-
const conditions = [];
|
|
155
|
-
const params = [];
|
|
156
|
-
let fromClause = 'url_cache';
|
|
157
|
-
if (options.query) {
|
|
158
|
-
fromClause = 'url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid';
|
|
159
|
-
conditions.push('url_cache_fts MATCH ?');
|
|
160
|
-
params.push(options.query);
|
|
205
|
+
stale: true
|
|
206
|
+
};
|
|
161
207
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
}
|
|
170
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
171
|
-
const orderClause = options.query ? 'ORDER BY rank' : 'ORDER BY url_cache.fetched_at DESC';
|
|
172
|
-
const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT 100`;
|
|
173
|
-
const rows = db.prepare(sql).all(...params);
|
|
174
|
-
return rows.map(rowToCachedContent);
|
|
208
|
+
}
|
|
209
|
+
return {
|
|
210
|
+
query: row.query,
|
|
211
|
+
results: JSON.parse(row.results),
|
|
212
|
+
engines_used: JSON.parse(row.engines_used),
|
|
213
|
+
searched_at: row.searched_at
|
|
214
|
+
};
|
|
175
215
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
216
|
+
const DEFAULT_FILTERED_LIMIT = 100;
|
|
217
|
+
function searchCacheFiltered(options) {
|
|
218
|
+
const db = getDatabase();
|
|
219
|
+
const conditions = [];
|
|
220
|
+
const params = [];
|
|
221
|
+
let fromClause = "url_cache";
|
|
222
|
+
if (options.query) {
|
|
223
|
+
fromClause = "url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid";
|
|
224
|
+
conditions.push("url_cache_fts MATCH ?");
|
|
225
|
+
params.push(sanitizeFtsQuery(options.query));
|
|
226
|
+
}
|
|
227
|
+
if (options.urlPattern) {
|
|
228
|
+
conditions.push("url_cache.normalized_url GLOB ?");
|
|
229
|
+
params.push(options.urlPattern);
|
|
230
|
+
}
|
|
231
|
+
if (options.since) {
|
|
232
|
+
conditions.push("url_cache.fetched_at > datetime(?)");
|
|
233
|
+
params.push(options.since);
|
|
234
|
+
}
|
|
235
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
236
|
+
const orderClause = options.query ? "ORDER BY rank" : "ORDER BY url_cache.fetched_at DESC";
|
|
237
|
+
const limit = Math.max(1, Math.floor(options.limit ?? DEFAULT_FILTERED_LIMIT));
|
|
238
|
+
const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT ?`;
|
|
239
|
+
const rows = db.prepare(sql).all(...params, limit);
|
|
240
|
+
return rows.map(rowToCachedContent);
|
|
196
241
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
242
|
+
function ftsSearchRanked(query, limit) {
|
|
243
|
+
if (!query.trim() || limit <= 0) return [];
|
|
244
|
+
const db = getDatabase();
|
|
245
|
+
const rows = db.prepare(`
|
|
246
|
+
SELECT url_cache.normalized_url AS url, url_cache_fts.rank AS rank
|
|
247
|
+
FROM url_cache
|
|
248
|
+
JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid
|
|
249
|
+
WHERE url_cache_fts MATCH ?
|
|
250
|
+
ORDER BY url_cache_fts.rank
|
|
251
|
+
LIMIT ?
|
|
252
|
+
`).all(sanitizeFtsQuery(query), limit);
|
|
253
|
+
return rows.map((r) => ({ url: r.url, score: -r.rank }));
|
|
254
|
+
}
|
|
255
|
+
function clearCacheEntries(options) {
|
|
256
|
+
const db = getDatabase();
|
|
257
|
+
const conditions = [];
|
|
258
|
+
const params = [];
|
|
259
|
+
if (options.query) {
|
|
260
|
+
conditions.push(
|
|
261
|
+
"id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)"
|
|
262
|
+
);
|
|
263
|
+
params.push(sanitizeFtsQuery(options.query));
|
|
264
|
+
}
|
|
265
|
+
if (options.urlPattern) {
|
|
266
|
+
conditions.push("normalized_url GLOB ?");
|
|
267
|
+
params.push(options.urlPattern);
|
|
268
|
+
}
|
|
269
|
+
if (options.since) {
|
|
270
|
+
conditions.push("fetched_at > datetime(?)");
|
|
271
|
+
params.push(options.since);
|
|
272
|
+
}
|
|
273
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
274
|
+
const sql = `DELETE FROM url_cache ${whereClause}`;
|
|
275
|
+
const result = db.prepare(sql).run(...params);
|
|
276
|
+
return result.changes;
|
|
277
|
+
}
|
|
278
|
+
function countCachedUrlsForDomain(domain) {
|
|
279
|
+
const db = getDatabase();
|
|
280
|
+
const normalized = domain.toLowerCase().replace(/^www\./, "");
|
|
281
|
+
const stmt = db.prepare(`
|
|
282
|
+
SELECT COUNT(*) AS n FROM url_cache
|
|
283
|
+
WHERE url LIKE 'http://' || ? || '/%'
|
|
284
|
+
OR url LIKE 'https://' || ? || '/%'
|
|
285
|
+
OR url LIKE 'http://www.' || ? || '/%'
|
|
286
|
+
OR url LIKE 'https://www.' || ? || '/%'
|
|
287
|
+
OR url = 'http://' || ?
|
|
288
|
+
OR url = 'https://' || ?
|
|
289
|
+
OR url = 'http://www.' || ?
|
|
290
|
+
OR url = 'https://www.' || ?
|
|
291
|
+
`);
|
|
292
|
+
const row = stmt.get(
|
|
293
|
+
normalized,
|
|
294
|
+
normalized,
|
|
295
|
+
normalized,
|
|
296
|
+
normalized,
|
|
297
|
+
normalized,
|
|
298
|
+
normalized,
|
|
299
|
+
normalized,
|
|
300
|
+
normalized
|
|
301
|
+
);
|
|
302
|
+
return row.n;
|
|
303
|
+
}
|
|
304
|
+
function getCacheStats() {
|
|
305
|
+
const db = getDatabase();
|
|
306
|
+
const row = db.prepare(`
|
|
200
307
|
SELECT
|
|
201
308
|
COUNT(*) as total_urls,
|
|
202
309
|
COALESCE(SUM(LENGTH(markdown) + LENGTH(COALESCE(raw_html, ''))), 0) as total_bytes,
|
|
@@ -204,11 +311,100 @@ export function getCacheStats() {
|
|
|
204
311
|
MAX(fetched_at) as newest
|
|
205
312
|
FROM url_cache
|
|
206
313
|
`).get();
|
|
314
|
+
return {
|
|
315
|
+
total_urls: row.total_urls,
|
|
316
|
+
total_size_mb: Math.round(row.total_bytes / (1024 * 1024) * 1e6) / 1e6,
|
|
317
|
+
oldest: row.oldest ?? "",
|
|
318
|
+
newest: row.newest ?? ""
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
function updateCacheEmbedding(url, embedding, model, dims) {
|
|
322
|
+
try {
|
|
323
|
+
const db = getDatabase();
|
|
324
|
+
let normalized;
|
|
325
|
+
try {
|
|
326
|
+
normalized = normalizeUrl(url);
|
|
327
|
+
} catch {
|
|
328
|
+
normalized = url;
|
|
329
|
+
}
|
|
330
|
+
const result = db.prepare(`
|
|
331
|
+
UPDATE url_cache
|
|
332
|
+
SET embedding = ?, embedding_model = ?, embedding_dims = ?, updated_at = datetime('now')
|
|
333
|
+
WHERE normalized_url = ?
|
|
334
|
+
`).run(embedding, model, dims, normalized);
|
|
335
|
+
return result.changes > 0;
|
|
336
|
+
} catch {
|
|
337
|
+
return false;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function getEmbeddingForUrl(url, modelId) {
|
|
341
|
+
try {
|
|
342
|
+
const db = getDatabase();
|
|
343
|
+
let normalized;
|
|
344
|
+
try {
|
|
345
|
+
normalized = normalizeUrl(url);
|
|
346
|
+
} catch {
|
|
347
|
+
normalized = url;
|
|
348
|
+
}
|
|
349
|
+
const row = db.prepare(`
|
|
350
|
+
SELECT embedding, embedding_model, embedding_dims
|
|
351
|
+
FROM url_cache
|
|
352
|
+
WHERE (url = ? OR normalized_url = ?) AND embedding IS NOT NULL
|
|
353
|
+
LIMIT 1
|
|
354
|
+
`).get(url, normalized);
|
|
355
|
+
if (!row) return null;
|
|
356
|
+
if (modelId !== void 0 && row.embedding_model !== modelId) return null;
|
|
207
357
|
return {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
newest: row.newest ?? '',
|
|
358
|
+
embedding: row.embedding,
|
|
359
|
+
model: row.embedding_model,
|
|
360
|
+
dims: row.embedding_dims
|
|
212
361
|
};
|
|
362
|
+
} catch {
|
|
363
|
+
return null;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
function getAllEmbeddings(modelId) {
|
|
367
|
+
try {
|
|
368
|
+
const db = getDatabase();
|
|
369
|
+
const rows = modelId !== void 0 ? db.prepare(`
|
|
370
|
+
SELECT normalized_url, embedding, embedding_model, embedding_dims
|
|
371
|
+
FROM url_cache
|
|
372
|
+
WHERE embedding IS NOT NULL AND embedding_model = ?
|
|
373
|
+
`).all(modelId) : db.prepare(`
|
|
374
|
+
SELECT normalized_url, embedding, embedding_model, embedding_dims
|
|
375
|
+
FROM url_cache
|
|
376
|
+
WHERE embedding IS NOT NULL
|
|
377
|
+
`).all();
|
|
378
|
+
return rows.map((r) => ({
|
|
379
|
+
normalizedUrl: r.normalized_url,
|
|
380
|
+
embedding: r.embedding,
|
|
381
|
+
model: r.embedding_model,
|
|
382
|
+
dims: r.embedding_dims
|
|
383
|
+
}));
|
|
384
|
+
} catch {
|
|
385
|
+
return [];
|
|
386
|
+
}
|
|
213
387
|
}
|
|
388
|
+
export {
|
|
389
|
+
cacheContent,
|
|
390
|
+
cacheSearchResults,
|
|
391
|
+
clearCacheEntries,
|
|
392
|
+
countCachedUrlsForDomain,
|
|
393
|
+
ftsSearchRanked,
|
|
394
|
+
getAllEmbeddings,
|
|
395
|
+
getCacheStats,
|
|
396
|
+
getCachedContent,
|
|
397
|
+
getCachedContentByNormalizedUrl,
|
|
398
|
+
getCachedSearchResults,
|
|
399
|
+
getEmbeddingForUrl,
|
|
400
|
+
getHashForNormalizedUrl,
|
|
401
|
+
getMarkdownForNormalizedUrl,
|
|
402
|
+
isCacheUsable,
|
|
403
|
+
isExpired,
|
|
404
|
+
normalizeUrl,
|
|
405
|
+
sanitizeFtsQuery,
|
|
406
|
+
searchCache,
|
|
407
|
+
searchCacheFiltered,
|
|
408
|
+
updateCacheEmbedding
|
|
409
|
+
};
|
|
214
410
|
//# sourceMappingURL=store.js.map
|
package/dist/cache/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cache/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,aAAa;IACb,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE5B,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEtE,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAClD,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAE3B,IAAI,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAE/B,gDAAgD;IAChD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,kDAAkD;IAClD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAU;IAC9B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAsB,EAAE,UAA4B;IAC/E,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAE1E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;GAWvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,aAAa;QACb,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;QACzC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,aAAa,EAAE,UAAU,CAAC,SAAS;QACnC,WAAW,EAAE,WAAW;QACxB,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC;QAC5B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAmBD,SAAS,kBAAkB,CAAC,GAAU;IACpC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,aAAa,EAAE,GAAG,CAAC,cAAc;QACjC,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,YAA4C;QAC7D,aAAa,EAAE,GAAG,CAAC,cAAgD;QACnE,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAsB,CAAC;IAEhD,OAAO,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAqB;IAC7C,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;GAMvB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAY,CAAC;IAEzB,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AASD,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAA2B,EAC3B,WAAqB;IAErB,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAGvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,KAAK;QACL,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACxC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC;QAC7B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa;IAClD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAExF,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,SAAS,CAA8F,CAAC;IAE/G,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC;QAC1C,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAInC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,WAAW,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,GAAG,oEAAoE,CAAC;QAClF,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,oCAAoC,CAAC;IAE3F,MAAM,GAAG,GAAG,2BAA2B,UAAU,IAAI,WAAW,IAAI,WAAW,YAAY,CAAC;IAC5F,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAY,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAIjC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CACb,iIAAiI,CAClI,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,GAAG,GAAG,yBAAyB,WAAW,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;GAOtB,CAAC,CAAC,GAAG,EAA+F,CAAC;IAEtG,OAAO;QACL,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACxE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;KACzB,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"sources":["../../src/cache/store.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { getDatabase } from './db.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\nimport type { RawFetchResult, ExtractionResult, CachedContent, SearchResultItem, CacheStats } from '../types.js';\n\nconst log = createLogger('cache');\n\n/**\n * Sanitize a user query for sqlite FTS5 MATCH.\n *\n * Why: bare tokens with `.` / `-` / `/` / `:` / digits-with-dot\n * (e.g. \"5.4\", \"x-y\", \"https://foo\") raise `fts5: syntax error near \".\"`.\n * Quoting tokens that aren't pure word-chars lets FTS5 treat them as phrases.\n * Already-quoted phrases and explicit operators (AND/OR/NOT/parens) pass through.\n */\nexport function sanitizeFtsQuery(q: string): string {\n const trimmed = q.trim();\n if (!trimmed) return '';\n if (/^\".*\"$/.test(trimmed)) return trimmed;\n const tokens = trimmed.match(/\"[^\"]*\"|\\S+/g) ?? [];\n const RESERVED = new Set(['AND', 'OR', 'NOT', '(', ')']);\n return tokens.map(tok => {\n if (tok.startsWith('\"') && tok.endsWith('\"')) return tok;\n if (RESERVED.has(tok)) return tok;\n if (/^\\w+\\*?$/.test(tok)) return tok;\n return `\"${tok.replace(/\"/g, '\"\"')}\"`;\n }).join(' ');\n}\n\nconst TRACKING_PARAMS = new Set([\n 'utm_source',\n 'utm_medium',\n 'utm_campaign',\n 'utm_content',\n 'utm_term',\n 'utm_id',\n 'fbclid',\n 'gclid',\n 'msclkid',\n 'mc_cid',\n 'mc_eid',\n]);\n\nexport function normalizeUrl(url: string): string {\n const parsed = new URL(url);\n\n parsed.protocol = parsed.protocol.toLowerCase();\n parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\\./, '');\n\n for (const key of [...parsed.searchParams.keys()]) {\n if (TRACKING_PARAMS.has(key) || key.startsWith('utm_')) {\n parsed.searchParams.delete(key);\n }\n }\n\n parsed.searchParams.sort();\n\n let result = parsed.toString();\n\n // Strip trailing slash from path (but not root)\n if (parsed.pathname !== '/' && result.endsWith('/')) {\n result = result.slice(0, -1);\n }\n // Remove trailing slash from origin-only URLs too\n if (parsed.pathname === '/' && !parsed.search && !parsed.hash) {\n result = result.replace(/\\/$/, '');\n }\n\n return result;\n}\n\nfunction toIsoSeconds(date: Date): string {\n return date.toISOString().replace('T', ' ').replace(/\\.\\d+Z$/, '');\n}\n\nexport function cacheContent(result: RawFetchResult, extraction: ExtractionResult): void {\n try {\n const db = getDatabase();\n const config = getConfig();\n\n const normalizedUrl = normalizeUrl(result.finalUrl || result.url);\n const contentHash = createHash('sha256').update(extraction.markdown).digest('hex');\n\n const now = new Date();\n const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1000);\n\n const stmt = db.prepare(`\n INSERT OR REPLACE INTO url_cache (\n url, normalized_url, title, markdown, raw_html,\n metadata, links, images, fetch_method, extractor_used,\n content_hash, fetched_at, expires_at\n )\n VALUES (\n @url, @normalizedUrl, @title, @markdown, @rawHtml,\n @metadata, @links, @images, @fetchMethod, @extractorUsed,\n @contentHash, @fetchedAt, @expiresAt\n )\n `);\n\n stmt.run({\n url: result.url,\n normalizedUrl,\n title: extraction.title,\n markdown: extraction.markdown,\n rawHtml: result.html,\n metadata: JSON.stringify(extraction.metadata),\n links: JSON.stringify(extraction.links),\n images: JSON.stringify(extraction.images),\n fetchMethod: result.method,\n extractorUsed: extraction.extractor,\n contentHash: contentHash,\n fetchedAt: toIsoSeconds(now),\n expiresAt: toIsoSeconds(expiresAt),\n });\n } catch (err) {\n log.warn('cacheContent failed', {\n url: result.url,\n finalUrl: result.finalUrl,\n error: err instanceof Error ? err.message : String(err),\n });\n }\n}\n\ninterface DbRow {\n id: number;\n url: string;\n normalized_url: string;\n title: string;\n markdown: string;\n raw_html: string;\n metadata: string;\n links: string;\n images: string;\n fetch_method: string;\n extractor_used: string;\n content_hash: string;\n fetched_at: string;\n expires_at: string | null;\n}\n\nfunction rowToCachedContent(row: DbRow): CachedContent {\n return {\n id: row.id,\n url: row.url,\n normalizedUrl: row.normalized_url,\n title: row.title,\n markdown: row.markdown,\n rawHtml: row.raw_html,\n metadata: row.metadata,\n links: row.links,\n images: row.images,\n fetchMethod: row.fetch_method as CachedContent['fetchMethod'],\n extractorUsed: row.extractor_used as CachedContent['extractorUsed'],\n contentHash: row.content_hash,\n fetchedAt: row.fetched_at,\n expiresAt: row.expires_at,\n };\n}\n\nexport function getCachedContent(url: string): CachedContent | null {\n const db = getDatabase();\n const normalizedUrl = normalizeUrl(url);\n\n const row = db.prepare(`\n SELECT * FROM url_cache WHERE url = ? OR normalized_url = ? LIMIT 1\n `).get(url, normalizedUrl) as DbRow | undefined;\n\n return row ? rowToCachedContent(row) : null;\n}\n\nexport function getCachedContentByNormalizedUrl(normalizedUrl: string): CachedContent | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT * FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as DbRow | undefined;\n return row ? rowToCachedContent(row) : null;\n}\n\nexport function getHashForNormalizedUrl(normalizedUrl: string): string | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT content_hash FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as { content_hash: string } | undefined;\n return row?.content_hash ?? null;\n}\n\nexport function getMarkdownForNormalizedUrl(normalizedUrl: string): string | null {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT markdown FROM url_cache WHERE normalized_url = ? LIMIT 1',\n ).get(normalizedUrl) as { markdown: string } | undefined;\n return row ? row.markdown : null;\n}\n\nexport function isExpired(cached: CachedContent): boolean {\n if (!cached.expiresAt) return false;\n return new Date(cached.expiresAt).getTime() < Date.now();\n}\n\nexport interface CacheLookupOptions {\n staleMaxSeconds?: number;\n}\n\nexport function isCacheUsable(\n cached: CachedContent,\n opts: CacheLookupOptions = {},\n): { usable: boolean; stale: boolean } {\n if (!cached.expiresAt) return { usable: true, stale: false };\n const expiresMs = new Date(cached.expiresAt).getTime();\n const now = Date.now();\n if (expiresMs >= now) return { usable: true, stale: false };\n const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1000;\n if (now - expiresMs <= staleMaxMs) return { usable: true, stale: true };\n return { usable: false, stale: false };\n}\n\nexport function searchCache(query: string): CachedContent[] {\n const db = getDatabase();\n\n const rows = db.prepare(`\n SELECT url_cache.*\n FROM url_cache\n JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid\n WHERE url_cache_fts MATCH ?\n ORDER BY rank\n `).all(sanitizeFtsQuery(query)) as DbRow[];\n\n return rows.map(rowToCachedContent);\n}\n\nexport interface CachedSearchResult {\n query: string;\n results: SearchResultItem[];\n engines_used: string[];\n searched_at: string;\n stale?: boolean;\n}\n\nexport function cacheSearchResults(\n query: string,\n results: SearchResultItem[],\n enginesUsed: string[],\n): void {\n const db = getDatabase();\n const config = getConfig();\n\n const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');\n const now = new Date();\n const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1000);\n\n const stmt = db.prepare(`\n INSERT OR REPLACE INTO search_cache (query, query_hash, results, engines_used, searched_at, expires_at)\n VALUES (@query, @queryHash, @results, @enginesUsed, @searchedAt, @expiresAt)\n `);\n\n stmt.run({\n query,\n queryHash,\n results: JSON.stringify(results),\n enginesUsed: JSON.stringify(enginesUsed),\n searchedAt: toIsoSeconds(now),\n expiresAt: toIsoSeconds(expiresAt),\n });\n}\n\nexport function getCachedSearchResults(\n query: string,\n opts: CacheLookupOptions = {},\n): CachedSearchResult | null {\n const db = getDatabase();\n const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');\n\n const row = db.prepare(\n 'SELECT query, results, engines_used, searched_at, expires_at FROM search_cache WHERE query_hash = ? LIMIT 1',\n ).get(queryHash) as\n | { query: string; results: string; engines_used: string; searched_at: string; expires_at: string | null }\n | undefined;\n\n if (!row) return null;\n\n if (row.expires_at) {\n const expiresMs = new Date(row.expires_at).getTime();\n const now = Date.now();\n if (expiresMs < now) {\n const staleMaxMs = (opts.staleMaxSeconds ?? 0) * 1000;\n if (now - expiresMs > staleMaxMs) return null;\n return {\n query: row.query,\n results: JSON.parse(row.results) as SearchResultItem[],\n engines_used: JSON.parse(row.engines_used) as string[],\n searched_at: row.searched_at,\n stale: true,\n };\n }\n }\n\n return {\n query: row.query,\n results: JSON.parse(row.results) as SearchResultItem[],\n engines_used: JSON.parse(row.engines_used) as string[],\n searched_at: row.searched_at,\n };\n}\n\nconst DEFAULT_FILTERED_LIMIT = 100;\n\nexport function searchCacheFiltered(options: {\n query?: string;\n urlPattern?: string;\n since?: string;\n limit?: number;\n}): CachedContent[] {\n const db = getDatabase();\n const conditions: string[] = [];\n const params: unknown[] = [];\n let fromClause = 'url_cache';\n\n if (options.query) {\n fromClause = 'url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid';\n conditions.push('url_cache_fts MATCH ?');\n params.push(sanitizeFtsQuery(options.query));\n }\n\n if (options.urlPattern) {\n conditions.push('url_cache.normalized_url GLOB ?');\n params.push(options.urlPattern);\n }\n\n if (options.since) {\n conditions.push('url_cache.fetched_at > datetime(?)');\n params.push(options.since);\n }\n\n const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n const orderClause = options.query ? 'ORDER BY rank' : 'ORDER BY url_cache.fetched_at DESC';\n const limit = Math.max(1, Math.floor(options.limit ?? DEFAULT_FILTERED_LIMIT));\n\n const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT ?`;\n const rows = db.prepare(sql).all(...params, limit) as DbRow[];\n return rows.map(rowToCachedContent);\n}\n\n/**\n * BM25-ranked FTS5 search across cached pages. Returns normalized URLs\n * paired with their rank score. `rank` from FTS5 is negative (lower is\n * better in sqlite ordering), so we flip the sign to surface a \"higher is\n * better\" score for consumers (e.g. RRF input).\n */\nexport function ftsSearchRanked(query: string, limit: number): Array<{ url: string; score: number }> {\n if (!query.trim() || limit <= 0) return [];\n const db = getDatabase();\n const rows = db.prepare(`\n SELECT url_cache.normalized_url AS url, url_cache_fts.rank AS rank\n FROM url_cache\n JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid\n WHERE url_cache_fts MATCH ?\n ORDER BY url_cache_fts.rank\n LIMIT ?\n `).all(sanitizeFtsQuery(query), limit) as Array<{ url: string; rank: number }>;\n return rows.map(r => ({ url: r.url, score: -r.rank }));\n}\n\nexport function clearCacheEntries(options: {\n query?: string;\n urlPattern?: string;\n since?: string;\n}): number {\n const db = getDatabase();\n const conditions: string[] = [];\n const params: unknown[] = [];\n\n if (options.query) {\n conditions.push(\n 'id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)',\n );\n params.push(sanitizeFtsQuery(options.query));\n }\n\n if (options.urlPattern) {\n conditions.push('normalized_url GLOB ?');\n params.push(options.urlPattern);\n }\n\n if (options.since) {\n conditions.push('fetched_at > datetime(?)');\n params.push(options.since);\n }\n\n const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n const sql = `DELETE FROM url_cache ${whereClause}`;\n const result = db.prepare(sql).run(...params);\n return result.changes;\n}\n\n// Counts cached URLs for an exact host (apex scoping — `blog.example.com`\n// and `example.com` are NOT collapsed). Leading `www.` is stripped to align\n// with normalizeUrl.\nexport function countCachedUrlsForDomain(domain: string): number {\n const db = getDatabase();\n const normalized = domain.toLowerCase().replace(/^www\\./, '');\n const stmt = db.prepare(`\n SELECT COUNT(*) AS n FROM url_cache\n WHERE url LIKE 'http://' || ? || '/%'\n OR url LIKE 'https://' || ? || '/%'\n OR url LIKE 'http://www.' || ? || '/%'\n OR url LIKE 'https://www.' || ? || '/%'\n OR url = 'http://' || ?\n OR url = 'https://' || ?\n OR url = 'http://www.' || ?\n OR url = 'https://www.' || ?\n `);\n const row = stmt.get(\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n normalized,\n ) as { n: number };\n return row.n;\n}\n\nexport function getCacheStats(): CacheStats {\n const db = getDatabase();\n const row = db.prepare(`\n SELECT\n COUNT(*) as total_urls,\n COALESCE(SUM(LENGTH(markdown) + LENGTH(COALESCE(raw_html, ''))), 0) as total_bytes,\n MIN(fetched_at) as oldest,\n MAX(fetched_at) as newest\n FROM url_cache\n `).get() as { total_urls: number; total_bytes: number; oldest: string | null; newest: string | null };\n\n return {\n total_urls: row.total_urls,\n total_size_mb: Math.round((row.total_bytes / (1024 * 1024)) * 1e6) / 1e6,\n oldest: row.oldest ?? '',\n newest: row.newest ?? '',\n };\n}\n\n// --- Embedding store functions (Slice 22) ---\n\nexport function updateCacheEmbedding(\n url: string,\n embedding: Buffer,\n model: string,\n dims: number,\n): boolean {\n try {\n const db = getDatabase();\n let normalized: string;\n try {\n normalized = normalizeUrl(url);\n } catch {\n normalized = url;\n }\n\n const result = db.prepare(`\n UPDATE url_cache\n SET embedding = ?, embedding_model = ?, embedding_dims = ?, updated_at = datetime('now')\n WHERE normalized_url = ?\n `).run(embedding, model, dims, normalized);\n\n return result.changes > 0;\n } catch {\n return false;\n }\n}\n\nexport interface EmbeddingData {\n embedding: Buffer;\n model: string;\n dims: number;\n}\n\nexport function getEmbeddingForUrl(url: string, modelId?: string): EmbeddingData | null {\n try {\n const db = getDatabase();\n let normalized: string;\n try {\n normalized = normalizeUrl(url);\n } catch {\n normalized = url;\n }\n\n const row = db.prepare(`\n SELECT embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE (url = ? OR normalized_url = ?) AND embedding IS NOT NULL\n LIMIT 1\n `).get(url, normalized) as { embedding: Buffer; embedding_model: string; embedding_dims: number } | undefined;\n\n if (!row) return null;\n // Filter by modelId when caller wants only embeddings from the current\n // model; mismatched entries return null so they are treated as cache miss.\n if (modelId !== undefined && row.embedding_model !== modelId) return null;\n\n return {\n embedding: row.embedding,\n model: row.embedding_model,\n dims: row.embedding_dims,\n };\n } catch {\n return null;\n }\n}\n\nexport interface StoredEmbedding {\n normalizedUrl: string;\n embedding: Buffer;\n model: string;\n dims: number;\n}\n\nexport function getAllEmbeddings(modelId?: string): StoredEmbedding[] {\n try {\n const db = getDatabase();\n // Filter by modelId when provided so stale entries from a previous model\n // (different dim / vector space) are skipped — the in-memory vector index\n // requires matching dimensionality across all entries.\n const rows = modelId !== undefined\n ? db.prepare(`\n SELECT normalized_url, embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE embedding IS NOT NULL AND embedding_model = ?\n `).all(modelId) as Array<{\n normalized_url: string;\n embedding: Buffer;\n embedding_model: string;\n embedding_dims: number;\n }>\n : db.prepare(`\n SELECT normalized_url, embedding, embedding_model, embedding_dims\n FROM url_cache\n WHERE embedding IS NOT NULL\n `).all() as Array<{\n normalized_url: string;\n embedding: Buffer;\n embedding_model: string;\n embedding_dims: number;\n }>;\n\n return rows.map(r => ({\n normalizedUrl: r.normalized_url,\n embedding: r.embedding,\n model: r.embedding_model,\n dims: r.embedding_dims,\n }));\n } catch {\n return [];\n }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,mBAAmB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAUzB,SAAS,iBAAiB,GAAmB;AAClD,QAAM,UAAU,EAAE,KAAK;AACvB,MAAI,CAAC,QAAS,QAAO;AACrB,MAAI,SAAS,KAAK,OAAO,EAAG,QAAO;AACnC,QAAM,SAAS,QAAQ,MAAM,cAAc,KAAK,CAAC;AACjD,QAAM,WAAW,oBAAI,IAAI,CAAC,OAAO,MAAM,OAAO,KAAK,GAAG,CAAC;AACvD,SAAO,OAAO,IAAI,SAAO;AACvB,QAAI,IAAI,WAAW,GAAG,KAAK,IAAI,SAAS,GAAG,EAAG,QAAO;AACrD,QAAI,SAAS,IAAI,GAAG,EAAG,QAAO;AAC9B,QAAI,WAAW,KAAK,GAAG,EAAG,QAAO;AACjC,WAAO,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC;AAAA,EACpC,CAAC,EAAE,KAAK,GAAG;AACb;AAEA,MAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,SAAS,aAAa,KAAqB;AAChD,QAAM,SAAS,IAAI,IAAI,GAAG;AAE1B,SAAO,WAAW,OAAO,SAAS,YAAY;AAC9C,SAAO,WAAW,OAAO,SAAS,YAAY,EAAE,QAAQ,UAAU,EAAE;AAEpE,aAAW,OAAO,CAAC,GAAG,OAAO,aAAa,KAAK,CAAC,GAAG;AACjD,QAAI,gBAAgB,IAAI,GAAG,KAAK,IAAI,WAAW,MAAM,GAAG;AACtD,aAAO,aAAa,OAAO,GAAG;AAAA,IAChC;AAAA,EACF;AAEA,SAAO,aAAa,KAAK;AAEzB,MAAI,SAAS,OAAO,SAAS;AAG7B,MAAI,OAAO,aAAa,OAAO,OAAO,SAAS,GAAG,GAAG;AACnD,aAAS,OAAO,MAAM,GAAG,EAAE;AAAA,EAC7B;AAEA,MAAI,OAAO,aAAa,OAAO,CAAC,OAAO,UAAU,CAAC,OAAO,MAAM;AAC7D,aAAS,OAAO,QAAQ,OAAO,EAAE;AAAA,EACnC;AAEA,SAAO;AACT;AAEA,SAAS,aAAa,MAAoB;AACxC,SAAO,KAAK,YAAY,EAAE,QAAQ,KAAK,GAAG,EAAE,QAAQ,WAAW,EAAE;AACnE;AAEO,SAAS,aAAa,QAAwB,YAAoC;AACvF,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,UAAM,SAAS,UAAU;AAEzB,UAAM,gBAAgB,aAAa,OAAO,YAAY,OAAO,GAAG;AAChE,UAAM,cAAc,WAAW,QAAQ,EAAE,OAAO,WAAW,QAAQ,EAAE,OAAO,KAAK;AAEjF,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM,YAAY,IAAI,KAAK,IAAI,QAAQ,IAAI,OAAO,kBAAkB,GAAI;AAExE,UAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAWvB;AAED,SAAK,IAAI;AAAA,MACP,KAAK,OAAO;AAAA,MACZ;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,UAAU,WAAW;AAAA,MACrB,SAAS,OAAO;AAAA,MAChB,UAAU,KAAK,UAAU,WAAW,QAAQ;AAAA,MAC5C,OAAO,KAAK,UAAU,WAAW,KAAK;AAAA,MACtC,QAAQ,KAAK,UAAU,WAAW,MAAM;AAAA,MACxC,aAAa,OAAO;AAAA,MACpB,eAAe,WAAW;AAAA,MAC1B;AAAA,MACA,WAAW,aAAa,GAAG;AAAA,MAC3B,WAAW,aAAa,SAAS;AAAA,IACnC,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,QAAI,KAAK,uBAAuB;AAAA,MAC9B,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,MACjB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AAAA,EACH;AACF;AAmBA,SAAS,mBAAmB,KAA2B;AACrD,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,KAAK,IAAI;AAAA,IACT,eAAe,IAAI;AAAA,IACnB,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,OAAO,IAAI;AAAA,IACX,QAAQ,IAAI;AAAA,IACZ,aAAa,IAAI;AAAA,IACjB,eAAe,IAAI;AAAA,IACnB,aAAa,IAAI;AAAA,IACjB,WAAW,IAAI;AAAA,IACf,WAAW,IAAI;AAAA,EACjB;AACF;AAEO,SAAS,iBAAiB,KAAmC;AAClE,QAAM,KAAK,YAAY;AACvB,QAAM,gBAAgB,aAAa,GAAG;AAEtC,QAAM,MAAM,GAAG,QAAQ;AAAA;AAAA,GAEtB,EAAE,IAAI,KAAK,aAAa;AAEzB,SAAO,MAAM,mBAAmB,GAAG,IAAI;AACzC;AAEO,SAAS,gCAAgC,eAA6C;AAC3F,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,MAAM,mBAAmB,GAAG,IAAI;AACzC;AAEO,SAAS,wBAAwB,eAAsC;AAC5E,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,KAAK,gBAAgB;AAC9B;AAEO,SAAS,4BAA4B,eAAsC;AAChF,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,aAAa;AACnB,SAAO,MAAM,IAAI,WAAW;AAC9B;AAEO,SAAS,UAAU,QAAgC;AACxD,MAAI,CAAC,OAAO,UAAW,QAAO;AAC9B,SAAO,IAAI,KAAK,OAAO,SAAS,EAAE,QAAQ,IAAI,KAAK,IAAI;AACzD;AAMO,SAAS,cACd,QACA,OAA2B,CAAC,GACS;AACrC,MAAI,CAAC,OAAO,UAAW,QAAO,EAAE,QAAQ,MAAM,OAAO,MAAM;AAC3D,QAAM,YAAY,IAAI,KAAK,OAAO,SAAS,EAAE,QAAQ;AACrD,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,aAAa,IAAK,QAAO,EAAE,QAAQ,MAAM,OAAO,MAAM;AAC1D,QAAM,cAAc,KAAK,mBAAmB,KAAK;AACjD,MAAI,MAAM,aAAa,WAAY,QAAO,EAAE,QAAQ,MAAM,OAAO,KAAK;AACtE,SAAO,EAAE,QAAQ,OAAO,OAAO,MAAM;AACvC;AAEO,SAAS,YAAY,OAAgC;AAC1D,QAAM,KAAK,YAAY;AAEvB,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAMvB,EAAE,IAAI,iBAAiB,KAAK,CAAC;AAE9B,SAAO,KAAK,IAAI,kBAAkB;AACpC;AAUO,SAAS,mBACd,OACA,SACA,aACM;AACN,QAAM,KAAK,YAAY;AACvB,QAAM,SAAS,UAAU;AAEzB,QAAM,YAAY,WAAW,QAAQ,EAAE,OAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK;AACtF,QAAM,MAAM,oBAAI,KAAK;AACrB,QAAM,YAAY,IAAI,KAAK,IAAI,QAAQ,IAAI,OAAO,iBAAiB,GAAI;AAEvE,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA,GAGvB;AAED,OAAK,IAAI;AAAA,IACP;AAAA,IACA;AAAA,IACA,SAAS,KAAK,UAAU,OAAO;AAAA,IAC/B,aAAa,KAAK,UAAU,WAAW;AAAA,IACvC,YAAY,aAAa,GAAG;AAAA,IAC5B,WAAW,aAAa,SAAS;AAAA,EACnC,CAAC;AACH;AAEO,SAAS,uBACd,OACA,OAA2B,CAAC,GACD;AAC3B,QAAM,KAAK,YAAY;AACvB,QAAM,YAAY,WAAW,QAAQ,EAAE,OAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK;AAEtF,QAAM,MAAM,GAAG;AAAA,IACb;AAAA,EACF,EAAE,IAAI,SAAS;AAIf,MAAI,CAAC,IAAK,QAAO;AAEjB,MAAI,IAAI,YAAY;AAClB,UAAM,YAAY,IAAI,KAAK,IAAI,UAAU,EAAE,QAAQ;AACnD,UAAM,MAAM,KAAK,IAAI;AACrB,QAAI,YAAY,KAAK;AACnB,YAAM,cAAc,KAAK,mBAAmB,KAAK;AACjD,UAAI,MAAM,YAAY,WAAY,QAAO;AACzC,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,SAAS,KAAK,MAAM,IAAI,OAAO;AAAA,QAC/B,cAAc,KAAK,MAAM,IAAI,YAAY;AAAA,QACzC,aAAa,IAAI;AAAA,QACjB,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,KAAK,MAAM,IAAI,OAAO;AAAA,IAC/B,cAAc,KAAK,MAAM,IAAI,YAAY;AAAA,IACzC,aAAa,IAAI;AAAA,EACnB;AACF;AAEA,MAAM,yBAAyB;AAExB,SAAS,oBAAoB,SAKhB;AAClB,QAAM,KAAK,YAAY;AACvB,QAAM,aAAuB,CAAC;AAC9B,QAAM,SAAoB,CAAC;AAC3B,MAAI,aAAa;AAEjB,MAAI,QAAQ,OAAO;AACjB,iBAAa;AACb,eAAW,KAAK,uBAAuB;AACvC,WAAO,KAAK,iBAAiB,QAAQ,KAAK,CAAC;AAAA,EAC7C;AAEA,MAAI,QAAQ,YAAY;AACtB,eAAW,KAAK,iCAAiC;AACjD,WAAO,KAAK,QAAQ,UAAU;AAAA,EAChC;AAEA,MAAI,QAAQ,OAAO;AACjB,eAAW,KAAK,oCAAoC;AACpD,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AAEA,QAAM,cAAc,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,CAAC,KAAK;AAClF,QAAM,cAAc,QAAQ,QAAQ,kBAAkB;AACtD,QAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,sBAAsB,CAAC;AAE7E,QAAM,MAAM,2BAA2B,UAAU,IAAI,WAAW,IAAI,WAAW;AAC/E,QAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,QAAQ,KAAK;AACjD,SAAO,KAAK,IAAI,kBAAkB;AACpC;AAQO,SAAS,gBAAgB,OAAe,OAAsD;AACnG,MAAI,CAAC,MAAM,KAAK,KAAK,SAAS,EAAG,QAAO,CAAC;AACzC,QAAM,KAAK,YAAY;AACvB,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOvB,EAAE,IAAI,iBAAiB,KAAK,GAAG,KAAK;AACrC,SAAO,KAAK,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,OAAO,CAAC,EAAE,KAAK,EAAE;AACvD;AAEO,SAAS,kBAAkB,SAIvB;AACT,QAAM,KAAK,YAAY;AACvB,QAAM,aAAuB,CAAC;AAC9B,QAAM,SAAoB,CAAC;AAE3B,MAAI,QAAQ,OAAO;AACjB,eAAW;AAAA,MACT;AAAA,IACF;AACA,WAAO,KAAK,iBAAiB,QAAQ,KAAK,CAAC;AAAA,EAC7C;AAEA,MAAI,QAAQ,YAAY;AACtB,eAAW,KAAK,uBAAuB;AACvC,WAAO,KAAK,QAAQ,UAAU;AAAA,EAChC;AAEA,MAAI,QAAQ,OAAO;AACjB,eAAW,KAAK,0BAA0B;AAC1C,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AAEA,QAAM,cAAc,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,CAAC,KAAK;AAClF,QAAM,MAAM,yBAAyB,WAAW;AAChD,QAAM,SAAS,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAC5C,SAAO,OAAO;AAChB;AAKO,SAAS,yBAAyB,QAAwB;AAC/D,QAAM,KAAK,YAAY;AACvB,QAAM,aAAa,OAAO,YAAY,EAAE,QAAQ,UAAU,EAAE;AAC5D,QAAM,OAAO,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAUvB;AACD,QAAM,MAAM,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,IAAI;AACb;AAEO,SAAS,gBAA4B;AAC1C,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOtB,EAAE,IAAI;AAEP,SAAO;AAAA,IACL,YAAY,IAAI;AAAA,IAChB,eAAe,KAAK,MAAO,IAAI,eAAe,OAAO,QAAS,GAAG,IAAI;AAAA,IACrE,QAAQ,IAAI,UAAU;AAAA,IACtB,QAAQ,IAAI,UAAU;AAAA,EACxB;AACF;AAIO,SAAS,qBACd,KACA,WACA,OACA,MACS;AACT,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,QAAI;AACJ,QAAI;AACF,mBAAa,aAAa,GAAG;AAAA,IAC/B,QAAQ;AACN,mBAAa;AAAA,IACf;AAEA,UAAM,SAAS,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIzB,EAAE,IAAI,WAAW,OAAO,MAAM,UAAU;AAEzC,WAAO,OAAO,UAAU;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,mBAAmB,KAAa,SAAwC;AACtF,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,QAAI;AACJ,QAAI;AACF,mBAAa,aAAa,GAAG;AAAA,IAC/B,QAAQ;AACN,mBAAa;AAAA,IACf;AAEA,UAAM,MAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAKtB,EAAE,IAAI,KAAK,UAAU;AAEtB,QAAI,CAAC,IAAK,QAAO;AAGjB,QAAI,YAAY,UAAa,IAAI,oBAAoB,QAAS,QAAO;AAErE,WAAO;AAAA,MACL,WAAW,IAAI;AAAA,MACf,OAAO,IAAI;AAAA,MACX,MAAM,IAAI;AAAA,IACZ;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AASO,SAAS,iBAAiB,SAAqC;AACpE,MAAI;AACF,UAAM,KAAK,YAAY;AAIvB,UAAM,OAAO,YAAY,SACrB,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,SAIV,EAAE,IAAI,OAAO,IAMd,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,SAIV,EAAE,IAAI;AAOX,WAAO,KAAK,IAAI,QAAM;AAAA,MACpB,eAAe,EAAE;AAAA,MACjB,WAAW,EAAE;AAAA,MACb,OAAO,EAAE;AAAA,MACT,MAAM,EAAE;AAAA,IACV,EAAE;AAAA,EACJ,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;","names":[]}
|