@staticn0va/wigolo 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +146 -227
- package/SKILL.md +382 -0
- package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
- package/assets/blocks/claude-code/wigolo-command.md +40 -0
- package/assets/blocks/cursor/wigolo.mdc +46 -0
- package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
- package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
- package/assets/skills/wigolo/SKILL.md +50 -0
- package/assets/skills/wigolo/rules/cache-first.md +30 -0
- package/assets/skills/wigolo/rules/synthesis.md +43 -0
- package/assets/skills/wigolo-agent/SKILL.md +73 -0
- package/assets/skills/wigolo-crawl/SKILL.md +60 -0
- package/assets/skills/wigolo-extract/SKILL.md +59 -0
- package/assets/skills/wigolo-fetch/SKILL.md +65 -0
- package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
- package/assets/skills/wigolo-research/SKILL.md +77 -0
- package/assets/skills/wigolo-search/SKILL.md +78 -0
- package/dist/agent/executor.d.ts +33 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +233 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +238 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +13 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +271 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/agent/relevance.d.ts +15 -0
- package/dist/agent/relevance.d.ts.map +1 -0
- package/dist/agent/relevance.js +60 -0
- package/dist/agent/relevance.js.map +1 -0
- package/dist/cache/backfill-embeddings.d.ts +23 -0
- package/dist/cache/backfill-embeddings.d.ts.map +1 -0
- package/dist/cache/backfill-embeddings.js +105 -0
- package/dist/cache/backfill-embeddings.js.map +1 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/db.d.ts +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +94 -22
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +82 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/migrations/runner.d.ts +29 -0
- package/dist/cache/migrations/runner.d.ts.map +1 -0
- package/dist/cache/migrations/runner.js +147 -0
- package/dist/cache/migrations/runner.js.map +1 -0
- package/dist/cache/sqlite-vec-store.d.ts +42 -0
- package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
- package/dist/cache/sqlite-vec-store.js +176 -0
- package/dist/cache/sqlite-vec-store.js.map +1 -0
- package/dist/cache/store.d.ts +47 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +364 -168
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/agents/antigravity.d.ts +20 -0
- package/dist/cli/agents/antigravity.d.ts.map +1 -0
- package/dist/cli/agents/antigravity.js +49 -0
- package/dist/cli/agents/antigravity.js.map +1 -0
- package/dist/cli/agents/claude-code.d.ts +25 -0
- package/dist/cli/agents/claude-code.d.ts.map +1 -0
- package/dist/cli/agents/claude-code.js +111 -0
- package/dist/cli/agents/claude-code.js.map +1 -0
- package/dist/cli/agents/cursor.d.ts +21 -0
- package/dist/cli/agents/cursor.d.ts.map +1 -0
- package/dist/cli/agents/cursor.js +58 -0
- package/dist/cli/agents/cursor.js.map +1 -0
- package/dist/cli/agents/gemini-cli.d.ts +21 -0
- package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
- package/dist/cli/agents/gemini-cli.js +55 -0
- package/dist/cli/agents/gemini-cli.js.map +1 -0
- package/dist/cli/agents/registry.d.ts +21 -0
- package/dist/cli/agents/registry.d.ts.map +1 -0
- package/dist/cli/agents/registry.js +27 -0
- package/dist/cli/agents/registry.js.map +1 -0
- package/dist/cli/agents/utils.d.ts +26 -0
- package/dist/cli/agents/utils.d.ts.map +1 -0
- package/dist/cli/agents/utils.js +136 -0
- package/dist/cli/agents/utils.js.map +1 -0
- package/dist/cli/agents/vscode.d.ts +21 -0
- package/dist/cli/agents/vscode.d.ts.map +1 -0
- package/dist/cli/agents/vscode.js +62 -0
- package/dist/cli/agents/vscode.js.map +1 -0
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +94 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/backfill.d.ts +2 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +58 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +61 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/doctor.d.ts +8 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +344 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +42 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/help.d.ts +6 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +63 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +35 -7
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +201 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +185 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/setup-mcp.d.ts +2 -0
- package/dist/cli/setup-mcp.d.ts.map +1 -0
- package/dist/cli/setup-mcp.js +114 -0
- package/dist/cli/setup-mcp.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/shutdown.d.ts +2 -0
- package/dist/cli/shutdown.d.ts.map +1 -0
- package/dist/cli/shutdown.js +26 -0
- package/dist/cli/shutdown.js.map +1 -0
- package/dist/cli/status.d.ts +2 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +31 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/telemetry.d.ts +10 -0
- package/dist/cli/telemetry.d.ts.map +1 -0
- package/dist/cli/telemetry.js +56 -0
- package/dist/cli/telemetry.js.map +1 -0
- package/dist/cli/tui/agents-types.d.ts +28 -0
- package/dist/cli/tui/agents-types.d.ts.map +1 -0
- package/dist/cli/tui/agents-types.js +1 -0
- package/dist/cli/tui/agents-types.js.map +1 -0
- package/dist/cli/tui/agents.d.ts +11 -0
- package/dist/cli/tui/agents.d.ts.map +1 -0
- package/dist/cli/tui/agents.js +93 -0
- package/dist/cli/tui/agents.js.map +1 -0
- package/dist/cli/tui/banner.d.ts +3 -0
- package/dist/cli/tui/banner.d.ts.map +1 -0
- package/dist/cli/tui/banner.js +30 -0
- package/dist/cli/tui/banner.js.map +1 -0
- package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
- package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/AgentSelect.js +116 -0
- package/dist/cli/tui/components/AgentSelect.js.map +1 -0
- package/dist/cli/tui/components/Banner.d.ts +6 -0
- package/dist/cli/tui/components/Banner.d.ts.map +1 -0
- package/dist/cli/tui/components/Banner.js +25 -0
- package/dist/cli/tui/components/Banner.js.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.js +19 -0
- package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
- package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
- package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
- package/dist/cli/tui/components/InstallProgress.js +67 -0
- package/dist/cli/tui/components/InstallProgress.js.map +1 -0
- package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
- package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
- package/dist/cli/tui/components/SkillInstall.js +94 -0
- package/dist/cli/tui/components/SkillInstall.js.map +1 -0
- package/dist/cli/tui/components/Summary.d.ts +22 -0
- package/dist/cli/tui/components/Summary.d.ts.map +1 -0
- package/dist/cli/tui/components/Summary.js +135 -0
- package/dist/cli/tui/components/Summary.js.map +1 -0
- package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
- package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/components/SystemCheck.js +71 -0
- package/dist/cli/tui/components/SystemCheck.js.map +1 -0
- package/dist/cli/tui/components/Verification.d.ts +8 -0
- package/dist/cli/tui/components/Verification.d.ts.map +1 -0
- package/dist/cli/tui/components/Verification.js +63 -0
- package/dist/cli/tui/components/Verification.js.map +1 -0
- package/dist/cli/tui/config-writer-cli.d.ts +12 -0
- package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-cli.js +39 -0
- package/dist/cli/tui/config-writer-cli.js.map +1 -0
- package/dist/cli/tui/config-writer-json.d.ts +16 -0
- package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-json.js +86 -0
- package/dist/cli/tui/config-writer-json.js.map +1 -0
- package/dist/cli/tui/config-writer-toml.d.ts +16 -0
- package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-toml.js +83 -0
- package/dist/cli/tui/config-writer-toml.js.map +1 -0
- package/dist/cli/tui/config-writer.d.ts +25 -0
- package/dist/cli/tui/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/config-writer.js +101 -0
- package/dist/cli/tui/config-writer.js.map +1 -0
- package/dist/cli/tui/detect-helpers.d.ts +6 -0
- package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
- package/dist/cli/tui/detect-helpers.js +45 -0
- package/dist/cli/tui/detect-helpers.js.map +1 -0
- package/dist/cli/tui/extras-prompt.d.ts +7 -0
- package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
- package/dist/cli/tui/extras-prompt.js +42 -0
- package/dist/cli/tui/extras-prompt.js.map +1 -0
- package/dist/cli/tui/flags-types.d.ts +19 -0
- package/dist/cli/tui/flags-types.d.ts.map +1 -0
- package/dist/cli/tui/flags-types.js +23 -0
- package/dist/cli/tui/flags-types.js.map +1 -0
- package/dist/cli/tui/flags.d.ts +5 -0
- package/dist/cli/tui/flags.d.ts.map +1 -0
- package/dist/cli/tui/flags.js +132 -0
- package/dist/cli/tui/flags.js.map +1 -0
- package/dist/cli/tui/format.d.ts +14 -0
- package/dist/cli/tui/format.d.ts.map +1 -0
- package/dist/cli/tui/format.js +37 -0
- package/dist/cli/tui/format.js.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
- package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
- package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
- package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useInstall.js +90 -0
- package/dist/cli/tui/hooks/useInstall.js.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
- package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
- package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
- package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useVerify.js +71 -0
- package/dist/cli/tui/hooks/useVerify.js.map +1 -0
- package/dist/cli/tui/ink-init.d.ts +2 -0
- package/dist/cli/tui/ink-init.d.ts.map +1 -0
- package/dist/cli/tui/ink-init.js +198 -0
- package/dist/cli/tui/ink-init.js.map +1 -0
- package/dist/cli/tui/reporter-auto.d.ts +7 -0
- package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
- package/dist/cli/tui/reporter-auto.js +15 -0
- package/dist/cli/tui/reporter-auto.js.map +1 -0
- package/dist/cli/tui/reporter.d.ts +26 -0
- package/dist/cli/tui/reporter.d.ts.map +1 -0
- package/dist/cli/tui/reporter.js +32 -0
- package/dist/cli/tui/reporter.js.map +1 -0
- package/dist/cli/tui/run-command.d.ts +14 -0
- package/dist/cli/tui/run-command.d.ts.map +1 -0
- package/dist/cli/tui/run-command.js +72 -0
- package/dist/cli/tui/run-command.js.map +1 -0
- package/dist/cli/tui/select-agents.d.ts +6 -0
- package/dist/cli/tui/select-agents.d.ts.map +1 -0
- package/dist/cli/tui/select-agents.js +32 -0
- package/dist/cli/tui/select-agents.js.map +1 -0
- package/dist/cli/tui/status-agents.d.ts +11 -0
- package/dist/cli/tui/status-agents.d.ts.map +1 -0
- package/dist/cli/tui/status-agents.js +53 -0
- package/dist/cli/tui/status-agents.js.map +1 -0
- package/dist/cli/tui/status-cache.d.ts +6 -0
- package/dist/cli/tui/status-cache.d.ts.map +1 -0
- package/dist/cli/tui/status-cache.js +39 -0
- package/dist/cli/tui/status-cache.js.map +1 -0
- package/dist/cli/tui/status-format.d.ts +14 -0
- package/dist/cli/tui/status-format.d.ts.map +1 -0
- package/dist/cli/tui/status-format.js +41 -0
- package/dist/cli/tui/status-format.js.map +1 -0
- package/dist/cli/tui/status-python.d.ts +6 -0
- package/dist/cli/tui/status-python.d.ts.map +1 -0
- package/dist/cli/tui/status-python.js +30 -0
- package/dist/cli/tui/status-python.js.map +1 -0
- package/dist/cli/tui/system-check.d.ts +24 -0
- package/dist/cli/tui/system-check.d.ts.map +1 -0
- package/dist/cli/tui/system-check.js +103 -0
- package/dist/cli/tui/system-check.js.map +1 -0
- package/dist/cli/tui/tui-reporter.d.ts +19 -0
- package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
- package/dist/cli/tui/tui-reporter.js +95 -0
- package/dist/cli/tui/tui-reporter.js.map +1 -0
- package/dist/cli/tui/utils/config-writer.d.ts +3 -0
- package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/utils/config-writer.js +22 -0
- package/dist/cli/tui/utils/config-writer.js.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.js +11 -0
- package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
- package/dist/cli/tui/verify-suggestions.d.ts +5 -0
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
- package/dist/cli/tui/verify-suggestions.js +20 -0
- package/dist/cli/tui/verify-suggestions.js.map +1 -0
- package/dist/cli/tui/verify.d.ts +14 -0
- package/dist/cli/tui/verify.d.ts.map +1 -0
- package/dist/cli/tui/verify.js +101 -0
- package/dist/cli/tui/verify.js.map +1 -0
- package/dist/cli/tui/version.d.ts +2 -0
- package/dist/cli/tui/version.d.ts.map +1 -0
- package/dist/cli/tui/version.js +14 -0
- package/dist/cli/tui/version.js.map +1 -0
- package/dist/cli/uninstall.d.ts +2 -0
- package/dist/cli/uninstall.d.ts.map +1 -0
- package/dist/cli/uninstall.js +57 -0
- package/dist/cli/uninstall.js.map +1 -0
- package/dist/cli/warmup.d.ts +10 -2
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +226 -93
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +28 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +106 -56
- package/dist/config.js.map +1 -1
- package/dist/crawl/crawler.d.ts +6 -0
- package/dist/crawl/crawler.d.ts.map +1 -1
- package/dist/crawl/crawler.js +210 -209
- package/dist/crawl/crawler.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +124 -81
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/crawl/etag-incremental.d.ts +43 -0
- package/dist/crawl/etag-incremental.d.ts.map +1 -0
- package/dist/crawl/etag-incremental.js +94 -0
- package/dist/crawl/etag-incremental.js.map +1 -0
- package/dist/crawl/index-to-vec.d.ts +10 -0
- package/dist/crawl/index-to-vec.d.ts.map +1 -0
- package/dist/crawl/index-to-vec.js +44 -0
- package/dist/crawl/index-to-vec.js.map +1 -0
- package/dist/crawl/mapper.js +136 -164
- package/dist/crawl/mapper.js.map +1 -1
- package/dist/crawl/rate-limiter.js +63 -66
- package/dist/crawl/rate-limiter.js.map +1 -1
- package/dist/crawl/robots.js +58 -57
- package/dist/crawl/robots.js.map +1 -1
- package/dist/crawl/sitemap-first.d.ts +12 -0
- package/dist/crawl/sitemap-first.d.ts.map +1 -0
- package/dist/crawl/sitemap-first.js +47 -0
- package/dist/crawl/sitemap-first.js.map +1 -0
- package/dist/crawl/sitemap.js +33 -32
- package/dist/crawl/sitemap.js.map +1 -1
- package/dist/crawl/url-utils.d.ts +1 -0
- package/dist/crawl/url-utils.d.ts.map +1 -1
- package/dist/crawl/url-utils.js +49 -37
- package/dist/crawl/url-utils.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +33 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +275 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +93 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +233 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/fastembed-provider.d.ts +19 -0
- package/dist/embedding/fastembed-provider.d.ts.map +1 -0
- package/dist/embedding/fastembed-provider.js +51 -0
- package/dist/embedding/fastembed-provider.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +234 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +52 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +27 -23
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/extract.d.ts.map +1 -1
- package/dist/extraction/extract.js +76 -76
- package/dist/extraction/extract.js.map +1 -1
- package/dist/extraction/jsonld.js +50 -54
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +30 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +130 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown-sanitize.d.ts +2 -0
- package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
- package/dist/extraction/markdown-sanitize.js +151 -0
- package/dist/extraction/markdown-sanitize.js.map +1 -0
- package/dist/extraction/markdown.d.ts +11 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +195 -91
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts +8 -0
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +57 -91
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +28 -29
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +135 -72
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +81 -91
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +87 -95
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +46 -54
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +71 -80
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +173 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/extraction/structured.d.ts +4 -0
- package/dist/extraction/structured.d.ts.map +1 -0
- package/dist/extraction/structured.js +163 -0
- package/dist/extraction/structured.js.map +1 -0
- package/dist/extraction/v1/classifier.d.ts +3 -0
- package/dist/extraction/v1/classifier.d.ts.map +1 -0
- package/dist/extraction/v1/classifier.js +110 -0
- package/dist/extraction/v1/classifier.js.map +1 -0
- package/dist/extraction/v1/extract-provider.d.ts +16 -0
- package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
- package/dist/extraction/v1/extract-provider.js +43 -0
- package/dist/extraction/v1/extract-provider.js.map +1 -0
- package/dist/extraction/v1/local-llm.d.ts +8 -0
- package/dist/extraction/v1/local-llm.d.ts.map +1 -0
- package/dist/extraction/v1/local-llm.js +34 -0
- package/dist/extraction/v1/local-llm.js.map +1 -0
- package/dist/extraction/v1/news.d.ts +3 -0
- package/dist/extraction/v1/news.d.ts.map +1 -0
- package/dist/extraction/v1/news.js +61 -0
- package/dist/extraction/v1/news.js.map +1 -0
- package/dist/extraction/v1/product.d.ts +3 -0
- package/dist/extraction/v1/product.d.ts.map +1 -0
- package/dist/extraction/v1/product.js +166 -0
- package/dist/extraction/v1/product.js.map +1 -0
- package/dist/extraction/v1/recipe.d.ts +3 -0
- package/dist/extraction/v1/recipe.d.ts.map +1 -0
- package/dist/extraction/v1/recipe.js +136 -0
- package/dist/extraction/v1/recipe.js.map +1 -0
- package/dist/extraction/v1/routed.d.ts +17 -0
- package/dist/extraction/v1/routed.d.ts.map +1 -0
- package/dist/extraction/v1/routed.js +68 -0
- package/dist/extraction/v1/routed.js.map +1 -0
- package/dist/extraction/v1/schemas/Article.d.ts +11 -0
- package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Article.js +23 -0
- package/dist/extraction/v1/schemas/Article.js.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.js +122 -0
- package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
- package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
- package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Paper.js +156 -0
- package/dist/extraction/v1/schemas/Paper.js.map +1 -0
- package/dist/extraction/v1/schemas/Product.d.ts +17 -0
- package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Product.js +149 -0
- package/dist/extraction/v1/schemas/Product.js.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.js +160 -0
- package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
- package/dist/extraction/v1/schemas/index.d.ts +13 -0
- package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/index.js +44 -0
- package/dist/extraction/v1/schemas/index.js.map +1 -0
- package/dist/extraction/v1/site-extractors.d.ts +5 -0
- package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
- package/dist/extraction/v1/site-extractors.js +31 -0
- package/dist/extraction/v1/site-extractors.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +88 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +56 -26
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +303 -127
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +72 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +89 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/content-check.js +39 -46
- package/dist/fetch/content-check.js.map +1 -1
- package/dist/fetch/error-describe.d.ts +7 -0
- package/dist/fetch/error-describe.d.ts.map +1 -0
- package/dist/fetch/error-describe.js +37 -0
- package/dist/fetch/error-describe.js.map +1 -0
- package/dist/fetch/http-client.d.ts +4 -0
- package/dist/fetch/http-client.d.ts.map +1 -1
- package/dist/fetch/http-client.js +147 -128
- package/dist/fetch/http-client.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +174 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/fetch/playwright-tier.d.ts +19 -0
- package/dist/fetch/playwright-tier.d.ts.map +1 -0
- package/dist/fetch/playwright-tier.js +76 -0
- package/dist/fetch/playwright-tier.js.map +1 -0
- package/dist/fetch/router.d.ts +49 -3
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +187 -81
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +102 -17
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +31 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +245 -0
- package/dist/instructions.js.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.js +41 -0
- package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
- package/dist/integrations/cloud/llm/cache.d.ts +5 -0
- package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/cache.js +49 -0
- package/dist/integrations/cloud/llm/cache.js.map +1 -0
- package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
- package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/gemini.js +37 -0
- package/dist/integrations/cloud/llm/gemini.js.map +1 -0
- package/dist/integrations/cloud/llm/groq.d.ts +3 -0
- package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/groq.js +74 -0
- package/dist/integrations/cloud/llm/groq.js.map +1 -0
- package/dist/integrations/cloud/llm/hash.d.ts +3 -0
- package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/hash.js +26 -0
- package/dist/integrations/cloud/llm/hash.js.map +1 -0
- package/dist/integrations/cloud/llm/model-select.d.ts +5 -0
- package/dist/integrations/cloud/llm/model-select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/model-select.js +32 -0
- package/dist/integrations/cloud/llm/model-select.js.map +1 -0
- package/dist/integrations/cloud/llm/openai.d.ts +3 -0
- package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/openai.js +43 -0
- package/dist/integrations/cloud/llm/openai.js.map +1 -0
- package/dist/integrations/cloud/llm/run.d.ts +27 -0
- package/dist/integrations/cloud/llm/run.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/run.js +99 -0
- package/dist/integrations/cloud/llm/run.js.map +1 -0
- package/dist/integrations/cloud/llm/select.d.ts +5 -0
- package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/select.js +30 -0
- package/dist/integrations/cloud/llm/select.js.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts +19 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.js +103 -0
- package/dist/integrations/cloud/llm/text-adapters.js.map +1 -0
- package/dist/integrations/cloud/llm/types.d.ts +24 -0
- package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/types.js +1 -0
- package/dist/integrations/cloud/llm/types.js.map +1 -0
- package/dist/integrations/cloud/llm/validate.d.ts +6 -0
- package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/validate.js +63 -0
- package/dist/integrations/cloud/llm/validate.js.map +1 -0
- package/dist/logger.d.ts +4 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +71 -30
- package/dist/logger.js.map +1 -1
- package/dist/pdf-parse.d.js +1 -0
- package/dist/pdf-parse.d.js.map +1 -0
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +157 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +71 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +79 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/providers/embed-provider.d.ts +11 -0
- package/dist/providers/embed-provider.d.ts.map +1 -0
- package/dist/providers/embed-provider.js +24 -0
- package/dist/providers/embed-provider.js.map +1 -0
- package/dist/providers/extract-provider.d.ts +23 -0
- package/dist/providers/extract-provider.d.ts.map +1 -0
- package/dist/providers/extract-provider.js +25 -0
- package/dist/providers/extract-provider.js.map +1 -0
- package/dist/providers/rerank-provider.d.ts +17 -0
- package/dist/providers/rerank-provider.d.ts.map +1 -0
- package/dist/providers/rerank-provider.js +41 -0
- package/dist/providers/rerank-provider.js.map +1 -0
- package/dist/providers/search-provider.d.ts +25 -0
- package/dist/providers/search-provider.d.ts.map +1 -0
- package/dist/providers/search-provider.js +44 -0
- package/dist/providers/search-provider.js.map +1 -0
- package/dist/providers/vector-store.d.ts +27 -0
- package/dist/providers/vector-store.d.ts.map +1 -0
- package/dist/providers/vector-store.js +27 -0
- package/dist/providers/vector-store.js.map +1 -0
- package/dist/python-env.d.ts +9 -0
- package/dist/python-env.d.ts.map +1 -0
- package/dist/python-env.js +13 -0
- package/dist/python-env.js.map +1 -0
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +62 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +43 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +44 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +47 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +67 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +74 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +65 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +74 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +1 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +13 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +283 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +86 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +184 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/research/branch-exploration.d.ts +14 -0
- package/dist/research/branch-exploration.d.ts.map +1 -0
- package/dist/research/branch-exploration.js +100 -0
- package/dist/research/branch-exploration.js.map +1 -0
- package/dist/research/brief.d.ts +6 -0
- package/dist/research/brief.d.ts.map +1 -0
- package/dist/research/brief.js +246 -0
- package/dist/research/brief.js.map +1 -0
- package/dist/research/citation-graph.d.ts +9 -0
- package/dist/research/citation-graph.d.ts.map +1 -0
- package/dist/research/citation-graph.js +114 -0
- package/dist/research/citation-graph.js.map +1 -0
- package/dist/research/decompose.d.ts +14 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +439 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +269 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesis-local.d.ts +19 -0
- package/dist/research/synthesis-local.d.ts.map +1 -0
- package/dist/research/synthesis-local.js +62 -0
- package/dist/research/synthesis-local.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +137 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +33 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +244 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/search/dedup.d.ts +1 -0
- package/dist/search/dedup.d.ts.map +1 -1
- package/dist/search/dedup.js +40 -32
- package/dist/search/dedup.js.map +1 -1
- package/dist/search/engines/arxiv.d.ts +7 -0
- package/dist/search/engines/arxiv.d.ts.map +1 -0
- package/dist/search/engines/arxiv.js +70 -0
- package/dist/search/engines/arxiv.js.map +1 -0
- package/dist/search/engines/bing-news.d.ts +7 -0
- package/dist/search/engines/bing-news.d.ts.map +1 -0
- package/dist/search/engines/bing-news.js +97 -0
- package/dist/search/engines/bing-news.js.map +1 -0
- package/dist/search/engines/bing.d.ts +1 -0
- package/dist/search/engines/bing.d.ts.map +1 -1
- package/dist/search/engines/bing.js +100 -44
- package/dist/search/engines/bing.js.map +1 -1
- package/dist/search/engines/devdocs.d.ts +6 -0
- package/dist/search/engines/devdocs.d.ts.map +1 -0
- package/dist/search/engines/devdocs.js +56 -0
- package/dist/search/engines/devdocs.js.map +1 -0
- package/dist/search/engines/duckduckgo.d.ts.map +1 -1
- package/dist/search/engines/duckduckgo.js +56 -44
- package/dist/search/engines/duckduckgo.js.map +1 -1
- package/dist/search/engines/github-code.d.ts +7 -0
- package/dist/search/engines/github-code.d.ts.map +1 -0
- package/dist/search/engines/github-code.js +55 -0
- package/dist/search/engines/github-code.js.map +1 -0
- package/dist/search/engines/hn-algolia.d.ts +7 -0
- package/dist/search/engines/hn-algolia.d.ts.map +1 -0
- package/dist/search/engines/hn-algolia.js +76 -0
- package/dist/search/engines/hn-algolia.js.map +1 -0
- package/dist/search/engines/lobsters.d.ts +7 -0
- package/dist/search/engines/lobsters.d.ts.map +1 -0
- package/dist/search/engines/lobsters.js +83 -0
- package/dist/search/engines/lobsters.js.map +1 -0
- package/dist/search/engines/mdn.d.ts +7 -0
- package/dist/search/engines/mdn.d.ts.map +1 -0
- package/dist/search/engines/mdn.js +48 -0
- package/dist/search/engines/mdn.js.map +1 -0
- package/dist/search/engines/semantic-scholar.d.ts +7 -0
- package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
- package/dist/search/engines/semantic-scholar.js +69 -0
- package/dist/search/engines/semantic-scholar.js.map +1 -0
- package/dist/search/engines/stackoverflow.d.ts +7 -0
- package/dist/search/engines/stackoverflow.d.ts.map +1 -0
- package/dist/search/engines/stackoverflow.js +73 -0
- package/dist/search/engines/stackoverflow.js.map +1 -0
- package/dist/search/engines/startpage.d.ts.map +1 -1
- package/dist/search/engines/startpage.js +65 -46
- package/dist/search/engines/startpage.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +220 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/filters.d.ts.map +1 -1
- package/dist/search/filters.js +58 -54
- package/dist/search/filters.js.map +1 -1
- package/dist/search/find-similar/crawl-rank.d.ts +9 -0
- package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
- package/dist/search/find-similar/crawl-rank.js +272 -0
- package/dist/search/find-similar/crawl-rank.js.map +1 -0
- package/dist/search/find-similar/mode.d.ts +4 -0
- package/dist/search/find-similar/mode.d.ts.map +1 -0
- package/dist/search/find-similar/mode.js +12 -0
- package/dist/search/find-similar/mode.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +509 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/highlights.d.ts +19 -0
- package/dist/search/highlights.d.ts.map +1 -0
- package/dist/search/highlights.js +167 -0
- package/dist/search/highlights.js.map +1 -0
- package/dist/search/language-filter.d.ts +29 -0
- package/dist/search/language-filter.d.ts.map +1 -0
- package/dist/search/language-filter.js +126 -0
- package/dist/search/language-filter.js.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.js +501 -0
- package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
- package/dist/search/legacy/searxng-provider.d.ts +7 -0
- package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
- package/dist/search/legacy/searxng-provider.js +11 -0
- package/dist/search/legacy/searxng-provider.js.map +1 -0
- package/dist/search/multi-query.d.ts +25 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +228 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/query.js +32 -34
- package/dist/search/query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -1
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +44 -35
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/authority-boost.d.ts +3 -0
- package/dist/search/reranker/authority-boost.d.ts.map +1 -0
- package/dist/search/reranker/authority-boost.js +179 -0
- package/dist/search/reranker/authority-boost.js.map +1 -0
- package/dist/search/reranker/consensus-boost.d.ts +3 -0
- package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
- package/dist/search/reranker/consensus-boost.js +27 -0
- package/dist/search/reranker/consensus-boost.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +13 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +23 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts +13 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.js +94 -0
- package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +39 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +69 -79
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +39 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +6 -0
- package/dist/search/truncate.d.ts.map +1 -0
- package/dist/search/truncate.js +26 -0
- package/dist/search/truncate.js.map +1 -0
- package/dist/search/url-unwrap.d.ts +3 -0
- package/dist/search/url-unwrap.d.ts.map +1 -0
- package/dist/search/url-unwrap.js +43 -0
- package/dist/search/url-unwrap.js.map +1 -0
- package/dist/search/v1/context-rank.d.ts +13 -0
- package/dist/search/v1/context-rank.d.ts.map +1 -0
- package/dist/search/v1/context-rank.js +74 -0
- package/dist/search/v1/context-rank.js.map +1 -0
- package/dist/search/v1/engine-base.d.ts +27 -0
- package/dist/search/v1/engine-base.d.ts.map +1 -0
- package/dist/search/v1/engine-base.js +110 -0
- package/dist/search/v1/engine-base.js.map +1 -0
- package/dist/search/v1/intent-router.d.ts +22 -0
- package/dist/search/v1/intent-router.d.ts.map +1 -0
- package/dist/search/v1/intent-router.js +138 -0
- package/dist/search/v1/intent-router.js.map +1 -0
- package/dist/search/v1/orchestrator.d.ts +24 -0
- package/dist/search/v1/orchestrator.d.ts.map +1 -0
- package/dist/search/v1/orchestrator.js +163 -0
- package/dist/search/v1/orchestrator.js.map +1 -0
- package/dist/search/v1/recency-boost.d.ts +9 -0
- package/dist/search/v1/recency-boost.d.ts.map +1 -0
- package/dist/search/v1/recency-boost.js +37 -0
- package/dist/search/v1/recency-boost.js.map +1 -0
- package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
- package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
- package/dist/search/v1/recent-cache-dedup.js +85 -0
- package/dist/search/v1/recent-cache-dedup.js.map +1 -0
- package/dist/search/v1/rss/feed-config.d.ts +21 -0
- package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-config.js +90 -0
- package/dist/search/v1/rss/feed-config.js.map +1 -0
- package/dist/search/v1/rss/feed-parser.d.ts +14 -0
- package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-parser.js +104 -0
- package/dist/search/v1/rss/feed-parser.js.map +1 -0
- package/dist/search/v1/rss/feed-poller.d.ts +22 -0
- package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-poller.js +102 -0
- package/dist/search/v1/rss/feed-poller.js.map +1 -0
- package/dist/search/v1/rss/feed-store.d.ts +30 -0
- package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-store.js +134 -0
- package/dist/search/v1/rss/feed-store.js.map +1 -0
- package/dist/search/v1/rss/rss-engine.d.ts +6 -0
- package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
- package/dist/search/v1/rss/rss-engine.js +28 -0
- package/dist/search/v1/rss/rss-engine.js.map +1 -0
- package/dist/search/v1/v1-provider.d.ts +7 -0
- package/dist/search/v1/v1-provider.d.ts.map +1 -0
- package/dist/search/v1/v1-provider.js +68 -0
- package/dist/search/v1/v1-provider.js.map +1 -0
- package/dist/search/v1/verticals/code.d.ts +4 -0
- package/dist/search/v1/verticals/code.d.ts.map +1 -0
- package/dist/search/v1/verticals/code.js +20 -0
- package/dist/search/v1/verticals/code.js.map +1 -0
- package/dist/search/v1/verticals/docs.d.ts +4 -0
- package/dist/search/v1/verticals/docs.d.ts.map +1 -0
- package/dist/search/v1/verticals/docs.js +20 -0
- package/dist/search/v1/verticals/docs.js.map +1 -0
- package/dist/search/v1/verticals/general.d.ts +4 -0
- package/dist/search/v1/verticals/general.d.ts.map +1 -0
- package/dist/search/v1/verticals/general.js +22 -0
- package/dist/search/v1/verticals/general.js.map +1 -0
- package/dist/search/v1/verticals/news.d.ts +10 -0
- package/dist/search/v1/verticals/news.d.ts.map +1 -0
- package/dist/search/v1/verticals/news.js +52 -0
- package/dist/search/v1/verticals/news.js.map +1 -0
- package/dist/search/v1/verticals/papers.d.ts +4 -0
- package/dist/search/v1/verticals/papers.d.ts.map +1 -0
- package/dist/search/v1/verticals/papers.js +23 -0
- package/dist/search/v1/verticals/papers.js.map +1 -0
- package/dist/search/validator.js +31 -31
- package/dist/search/validator.js.map +1 -1
- package/dist/searxng/bootstrap.d.ts +30 -0
- package/dist/searxng/bootstrap.d.ts.map +1 -1
- package/dist/searxng/bootstrap.js +223 -85
- package/dist/searxng/bootstrap.js.map +1 -1
- package/dist/searxng/docker.d.ts.map +1 -1
- package/dist/searxng/docker.js +69 -60
- package/dist/searxng/docker.js.map +1 -1
- package/dist/searxng/process.d.ts +13 -1
- package/dist/searxng/process.d.ts.map +1 -1
- package/dist/searxng/process.js +231 -164
- package/dist/searxng/process.js.map +1 -1
- package/dist/server/backend-status.d.ts +13 -0
- package/dist/server/backend-status.d.ts.map +1 -0
- package/dist/server/backend-status.js +40 -0
- package/dist/server/backend-status.js.map +1 -0
- package/dist/server/tool-schemas.d.ts +549 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +464 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server/warmup-on-start.d.ts +9 -0
- package/dist/server/warmup-on-start.d.ts.map +1 -0
- package/dist/server/warmup-on-start.js +55 -0
- package/dist/server/warmup-on-start.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +454 -297
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +128 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +177 -44
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +171 -88
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts +2 -2
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +175 -59
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts +2 -2
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +174 -68
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +127 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +107 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +10 -2
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -158
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +350 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +6 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +34 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +78 -8
- package/dist/extraction/trafilatura.d.ts +0 -6
- package/dist/extraction/trafilatura.d.ts.map +0 -1
- package/dist/extraction/trafilatura.js +0 -105
- package/dist/extraction/trafilatura.js.map +0 -1
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -63
- package/dist/search/flashrank.js.map +0 -1
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { normalizeUrl } from "../cache/store.js";
|
|
2
|
+
import { getConfig } from "../config.js";
|
|
3
|
+
import { createLogger } from "../logger.js";
|
|
4
|
+
const log = createLogger("search");
|
|
5
|
+
const RRF_K = 60;
|
|
6
|
+
function normalizeQueries(queries) {
|
|
7
|
+
try {
|
|
8
|
+
const config = getConfig();
|
|
9
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10
|
+
const normalized = [];
|
|
11
|
+
for (const raw of queries) {
|
|
12
|
+
const q = raw.toLowerCase().trim().replace(/\s+/g, " ");
|
|
13
|
+
if (q.length === 0) continue;
|
|
14
|
+
if (seen.has(q)) continue;
|
|
15
|
+
seen.add(q);
|
|
16
|
+
normalized.push(q);
|
|
17
|
+
}
|
|
18
|
+
if (normalized.length > config.multiQueryMax) {
|
|
19
|
+
log.warn("multi-query array exceeds max, truncating", {
|
|
20
|
+
provided: normalized.length,
|
|
21
|
+
max: config.multiQueryMax
|
|
22
|
+
});
|
|
23
|
+
return normalized.slice(0, config.multiQueryMax);
|
|
24
|
+
}
|
|
25
|
+
return normalized;
|
|
26
|
+
} catch (err) {
|
|
27
|
+
log.error("normalizeQueries failed", { error: String(err) });
|
|
28
|
+
return [];
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function fanOutSearch(queries, engines, options) {
|
|
32
|
+
const allResults = [];
|
|
33
|
+
const enginesUsed = /* @__PURE__ */ new Set();
|
|
34
|
+
const errors = [];
|
|
35
|
+
if (queries.length === 0 || engines.length === 0) {
|
|
36
|
+
return { results: [], enginesUsed: [], errors: [] };
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
const config = getConfig();
|
|
40
|
+
const concurrency = config.multiQueryConcurrency;
|
|
41
|
+
const hasFilterAttrition = !!(options.includeDomains?.length || options.excludeDomains?.length);
|
|
42
|
+
const overfetchFactor = hasFilterAttrition ? 3 : 2;
|
|
43
|
+
const engineOptions = {
|
|
44
|
+
maxResults: options.maxResults * overfetchFactor,
|
|
45
|
+
timeRange: options.timeRange,
|
|
46
|
+
language: options.language,
|
|
47
|
+
includeDomains: options.includeDomains,
|
|
48
|
+
excludeDomains: options.excludeDomains,
|
|
49
|
+
fromDate: options.fromDate,
|
|
50
|
+
toDate: options.toDate,
|
|
51
|
+
category: options.category
|
|
52
|
+
};
|
|
53
|
+
const effEngines = queries.length >= 3 && engines.length > 2 ? engines.slice(0, 2) : engines;
|
|
54
|
+
const tasks = [];
|
|
55
|
+
for (const engine of effEngines) {
|
|
56
|
+
for (const query of queries) {
|
|
57
|
+
tasks.push({ engine, query });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
for (let i = 0; i < tasks.length; i += concurrency) {
|
|
61
|
+
const batch = tasks.slice(i, i + concurrency);
|
|
62
|
+
const promises = batch.map(async ({ engine, query }) => {
|
|
63
|
+
try {
|
|
64
|
+
const results = await engine.search(query, engineOptions);
|
|
65
|
+
for (const r of results) {
|
|
66
|
+
allResults.push(r);
|
|
67
|
+
enginesUsed.add(engine.name);
|
|
68
|
+
}
|
|
69
|
+
} catch (err) {
|
|
70
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
71
|
+
log.warn("multi-query engine search failed", {
|
|
72
|
+
engine: engine.name,
|
|
73
|
+
query,
|
|
74
|
+
error: msg
|
|
75
|
+
});
|
|
76
|
+
errors.push(`${engine.name}(${query}): ${msg}`);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
await Promise.allSettled(promises);
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
results: allResults,
|
|
83
|
+
enginesUsed: [...enginesUsed],
|
|
84
|
+
errors
|
|
85
|
+
};
|
|
86
|
+
} catch (err) {
|
|
87
|
+
log.error("fanOutSearch failed", { error: String(err) });
|
|
88
|
+
return {
|
|
89
|
+
results: allResults,
|
|
90
|
+
enginesUsed: [...enginesUsed],
|
|
91
|
+
errors: [...errors, `fanOutSearch: ${String(err)}`]
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function synthesizeIntent(queries) {
|
|
96
|
+
try {
|
|
97
|
+
return queries.map((q) => q.trim()).filter(Boolean).join("; ");
|
|
98
|
+
} catch (err) {
|
|
99
|
+
log.error("synthesizeIntent failed", { error: String(err) });
|
|
100
|
+
return "";
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function mergeWithRRF(rankedLists) {
|
|
104
|
+
try {
|
|
105
|
+
if (rankedLists.length === 0) return [];
|
|
106
|
+
const nonEmpty = rankedLists.filter((l) => l.length > 0);
|
|
107
|
+
if (nonEmpty.length === 0) return [];
|
|
108
|
+
const rrfScores = /* @__PURE__ */ new Map();
|
|
109
|
+
const bestAppearance = /* @__PURE__ */ new Map();
|
|
110
|
+
for (const list of nonEmpty) {
|
|
111
|
+
for (let rank = 0; rank < list.length; rank++) {
|
|
112
|
+
const item = list[rank];
|
|
113
|
+
let normalizedUrlStr;
|
|
114
|
+
try {
|
|
115
|
+
normalizedUrlStr = normalizeUrl(item.url);
|
|
116
|
+
} catch {
|
|
117
|
+
normalizedUrlStr = item.url;
|
|
118
|
+
}
|
|
119
|
+
const rrfContribution = 1 / (RRF_K + rank + 1);
|
|
120
|
+
const current = rrfScores.get(normalizedUrlStr) ?? 0;
|
|
121
|
+
rrfScores.set(normalizedUrlStr, current + rrfContribution);
|
|
122
|
+
const existing = bestAppearance.get(normalizedUrlStr);
|
|
123
|
+
if (!existing || rank < existing.bestRank) {
|
|
124
|
+
bestAppearance.set(normalizedUrlStr, { result: item, bestRank: rank });
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
let maxScore = 0;
|
|
129
|
+
for (const score of rrfScores.values()) {
|
|
130
|
+
if (score > maxScore) maxScore = score;
|
|
131
|
+
}
|
|
132
|
+
const merged = [];
|
|
133
|
+
for (const [normalizedUrlStr, score] of rrfScores.entries()) {
|
|
134
|
+
const appearance = bestAppearance.get(normalizedUrlStr);
|
|
135
|
+
merged.push({
|
|
136
|
+
...appearance.result,
|
|
137
|
+
relevance_score: maxScore > 0 ? score / maxScore : 0
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
merged.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
141
|
+
return merged;
|
|
142
|
+
} catch (err) {
|
|
143
|
+
log.error("mergeWithRRF failed", { error: String(err) });
|
|
144
|
+
return [];
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
const DEEP_SUFFIXES = ["guide", "tutorial", "examples", "best practices"];
|
|
148
|
+
const COMPARE_INTRO = /(compare|comparison of|difference[s]? between|trade[- ]?off[s]? (?:of|between)|vs\.?|versus)/i;
|
|
149
|
+
const STOP_PHRASES = /* @__PURE__ */ new Set([
|
|
150
|
+
"and",
|
|
151
|
+
"or",
|
|
152
|
+
"vs",
|
|
153
|
+
"versus",
|
|
154
|
+
"between",
|
|
155
|
+
"of",
|
|
156
|
+
"the",
|
|
157
|
+
"a",
|
|
158
|
+
"an",
|
|
159
|
+
"for",
|
|
160
|
+
"in",
|
|
161
|
+
"on",
|
|
162
|
+
"to",
|
|
163
|
+
"with",
|
|
164
|
+
"compare",
|
|
165
|
+
"comparison",
|
|
166
|
+
"differences",
|
|
167
|
+
"tradeoff",
|
|
168
|
+
"tradeoffs",
|
|
169
|
+
"trade-off",
|
|
170
|
+
"trade-offs"
|
|
171
|
+
]);
|
|
172
|
+
function splitListEntities(segment) {
|
|
173
|
+
const cleaned = segment.replace(/^\s*(?:between|of|the|a|an)\s+/i, "").trim();
|
|
174
|
+
const parts = cleaned.split(/,\s*(?:and\s+)?|\s+and\s+|\s+vs\.?\s+|\s+versus\s+/i).map((s) => s.trim()).filter((s) => s.length >= 2 && s.length <= 120 && !STOP_PHRASES.has(s.toLowerCase()));
|
|
175
|
+
return [...new Set(parts)];
|
|
176
|
+
}
|
|
177
|
+
function decomposeMultiHop(query) {
|
|
178
|
+
const text = query.trim();
|
|
179
|
+
const compareMatch = text.match(/(?:compare|trade[- ]?off[s]? (?:of|between)|difference[s]? between)\s+(.+?)(?:[?.]|$)/i);
|
|
180
|
+
if (compareMatch) {
|
|
181
|
+
const items = splitListEntities(compareMatch[1]);
|
|
182
|
+
if (items.length >= 2 && items.length <= 6) return items;
|
|
183
|
+
}
|
|
184
|
+
const vsMatch = text.match(/^([\w\s.&+#/-]+?)\s+(?:vs\.?|versus)\s+([\w\s.&+#/-]+?)(?:\s+(?:for|in|when|to)\s+|[?.]|$)/i);
|
|
185
|
+
if (vsMatch) {
|
|
186
|
+
const items = [vsMatch[1].trim(), vsMatch[2].trim()].filter(Boolean);
|
|
187
|
+
if (items.length === 2) return items;
|
|
188
|
+
}
|
|
189
|
+
const commaList = text.match(/of\s+([\w\s.,&+#/-]+?(?:,\s*and\s+|\s+and\s+)[\w\s.&+#/-]+?)(?:\s+(?:for|in|when|to)\s+|[?.]|$)/i);
|
|
190
|
+
if (commaList) {
|
|
191
|
+
const items = splitListEntities(commaList[1]);
|
|
192
|
+
if (items.length >= 2 && items.length <= 6) return items;
|
|
193
|
+
}
|
|
194
|
+
return null;
|
|
195
|
+
}
|
|
196
|
+
function expandQueryHeuristic(query) {
|
|
197
|
+
const max = Math.max(1, parseInt(process.env.WIGOLO_QUERY_EXPAND_VARIANTS || "5", 10));
|
|
198
|
+
const trimmed = query.trim();
|
|
199
|
+
if (COMPARE_INTRO.test(trimmed) || /,.+,.+\band\b/.test(trimmed)) {
|
|
200
|
+
const parts = decomposeMultiHop(trimmed);
|
|
201
|
+
if (parts && parts.length >= 2) {
|
|
202
|
+
const cap = Math.max(max, parts.length);
|
|
203
|
+
const out = [trimmed, ...parts];
|
|
204
|
+
return [...new Set(out)].slice(0, cap);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const variants = [trimmed];
|
|
208
|
+
for (const suffix of DEEP_SUFFIXES) {
|
|
209
|
+
if (variants.length >= max) break;
|
|
210
|
+
const candidate = `${trimmed} ${suffix}`;
|
|
211
|
+
if (!variants.includes(candidate)) variants.push(candidate);
|
|
212
|
+
}
|
|
213
|
+
return variants;
|
|
214
|
+
}
|
|
215
|
+
function expandIfSingle(query) {
|
|
216
|
+
if (Array.isArray(query)) return [...query];
|
|
217
|
+
return expandQueryHeuristic(query);
|
|
218
|
+
}
|
|
219
|
+
export {
|
|
220
|
+
decomposeMultiHop,
|
|
221
|
+
expandIfSingle,
|
|
222
|
+
expandQueryHeuristic,
|
|
223
|
+
fanOutSearch,
|
|
224
|
+
mergeWithRRF,
|
|
225
|
+
normalizeQueries,
|
|
226
|
+
synthesizeIntent
|
|
227
|
+
};
|
|
228
|
+
//# sourceMappingURL=multi-query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/search/multi-query.ts"],"sourcesContent":["import type { RawSearchResult, SearchEngine, SearchEngineOptions } from '../types.js';\nimport type { MergedSearchResult } from './dedup.js';\nimport { normalizeUrl } from '../cache/store.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('search');\n\nconst RRF_K = 60;\n\nexport function normalizeQueries(queries: string[]): string[] {\n try {\n const config = getConfig();\n const seen = new Set<string>();\n const normalized: string[] = [];\n\n for (const raw of queries) {\n const q = raw.toLowerCase().trim().replace(/\\s+/g, ' ');\n if (q.length === 0) continue;\n if (seen.has(q)) continue;\n seen.add(q);\n normalized.push(q);\n }\n\n if (normalized.length > config.multiQueryMax) {\n log.warn('multi-query array exceeds max, truncating', {\n provided: normalized.length,\n max: config.multiQueryMax,\n });\n return normalized.slice(0, config.multiQueryMax);\n }\n\n return normalized;\n } catch (err) {\n log.error('normalizeQueries failed', { error: String(err) });\n return [];\n }\n}\n\nexport interface FanOutOptions {\n maxResults: number;\n timeRange?: string;\n language?: string;\n includeDomains?: string[];\n excludeDomains?: string[];\n fromDate?: string;\n toDate?: string;\n category?: 'general' | 'news' | 'code' | 'docs' | 'papers' | 'images';\n}\n\nexport interface FanOutResult {\n results: RawSearchResult[];\n enginesUsed: string[];\n errors: string[];\n}\n\nexport async function fanOutSearch(\n queries: string[],\n engines: SearchEngine[],\n options: FanOutOptions,\n): Promise<FanOutResult> {\n const allResults: RawSearchResult[] = [];\n const enginesUsed = new Set<string>();\n const errors: string[] = [];\n\n if (queries.length === 0 || engines.length === 0) {\n return { results: [], enginesUsed: [], errors: [] };\n }\n\n try {\n const config = getConfig();\n const concurrency = config.multiQueryConcurrency;\n\n const hasFilterAttrition = !!(options.includeDomains?.length || options.excludeDomains?.length);\n const overfetchFactor = hasFilterAttrition ? 3 : 2;\n\n const engineOptions: SearchEngineOptions = {\n maxResults: options.maxResults * overfetchFactor,\n timeRange: options.timeRange,\n language: options.language,\n includeDomains: options.includeDomains,\n excludeDomains: options.excludeDomains,\n fromDate: options.fromDate,\n toDate: options.toDate,\n category: options.category,\n };\n\n // Q9-followup: when multi-hop decomposition produced ≥3 sub-queries,\n // cap engines per query to keep total fan-out tasks bounded\n // (3 entities × 3 engines × variants → wall blows past 180s).\n const effEngines = queries.length >= 3 && engines.length > 2\n ? engines.slice(0, 2)\n : engines;\n\n const tasks: Array<{ engine: SearchEngine; query: string }> = [];\n for (const engine of effEngines) {\n for (const query of queries) {\n tasks.push({ engine, query });\n }\n }\n\n for (let i = 0; i < tasks.length; i += concurrency) {\n const batch = tasks.slice(i, i + concurrency);\n const promises = batch.map(async ({ engine, query }) => {\n try {\n const results = await engine.search(query, engineOptions);\n for (const r of results) {\n allResults.push(r);\n enginesUsed.add(engine.name);\n }\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n log.warn('multi-query engine search failed', {\n engine: engine.name,\n query,\n error: msg,\n });\n errors.push(`${engine.name}(${query}): ${msg}`);\n }\n });\n\n await Promise.allSettled(promises);\n }\n\n return {\n results: allResults,\n enginesUsed: [...enginesUsed],\n errors,\n };\n } catch (err) {\n log.error('fanOutSearch failed', { error: String(err) });\n return {\n results: allResults,\n enginesUsed: [...enginesUsed],\n errors: [...errors, `fanOutSearch: ${String(err)}`],\n };\n }\n}\n\nexport function synthesizeIntent(queries: string[]): string {\n try {\n return queries.map(q => q.trim()).filter(Boolean).join('; ');\n } catch (err) {\n log.error('synthesizeIntent failed', { error: String(err) });\n return '';\n }\n}\n\nexport function mergeWithRRF(rankedLists: MergedSearchResult[][]): MergedSearchResult[] {\n try {\n if (rankedLists.length === 0) return [];\n\n const nonEmpty = rankedLists.filter(l => l.length > 0);\n if (nonEmpty.length === 0) return [];\n\n const rrfScores = new Map<string, number>();\n const bestAppearance = new Map<string, { result: MergedSearchResult; bestRank: number }>();\n\n for (const list of nonEmpty) {\n for (let rank = 0; rank < list.length; rank++) {\n const item = list[rank];\n let normalizedUrlStr: string;\n try {\n normalizedUrlStr = normalizeUrl(item.url);\n } catch {\n normalizedUrlStr = item.url;\n }\n\n const rrfContribution = 1 / (RRF_K + rank + 1);\n const current = rrfScores.get(normalizedUrlStr) ?? 0;\n rrfScores.set(normalizedUrlStr, current + rrfContribution);\n\n const existing = bestAppearance.get(normalizedUrlStr);\n if (!existing || rank < existing.bestRank) {\n bestAppearance.set(normalizedUrlStr, { result: item, bestRank: rank });\n }\n }\n }\n\n let maxScore = 0;\n for (const score of rrfScores.values()) {\n if (score > maxScore) maxScore = score;\n }\n\n const merged: MergedSearchResult[] = [];\n for (const [normalizedUrlStr, score] of rrfScores.entries()) {\n const appearance = bestAppearance.get(normalizedUrlStr)!;\n merged.push({\n ...appearance.result,\n relevance_score: maxScore > 0 ? score / maxScore : 0,\n });\n }\n\n merged.sort((a, b) => b.relevance_score - a.relevance_score);\n return merged;\n } catch (err) {\n log.error('mergeWithRRF failed', { error: String(err) });\n return [];\n }\n}\n\nconst DEEP_SUFFIXES = ['guide', 'tutorial', 'examples', 'best practices'] as const;\n\nconst COMPARE_INTRO = /(compare|comparison of|difference[s]? between|trade[- ]?off[s]? (?:of|between)|vs\\.?|versus)/i;\nconst STOP_PHRASES = new Set([\n 'and', 'or', 'vs', 'versus', 'between', 'of', 'the', 'a', 'an',\n 'for', 'in', 'on', 'to', 'with', 'compare', 'comparison', 'differences',\n 'tradeoff', 'tradeoffs', 'trade-off', 'trade-offs',\n]);\n\nfunction splitListEntities(segment: string): string[] {\n const cleaned = segment.replace(/^\\s*(?:between|of|the|a|an)\\s+/i, '').trim();\n const parts = cleaned\n .split(/,\\s*(?:and\\s+)?|\\s+and\\s+|\\s+vs\\.?\\s+|\\s+versus\\s+/i)\n .map((s) => s.trim())\n .filter((s) => s.length >= 2 && s.length <= 120 && !STOP_PHRASES.has(s.toLowerCase()));\n return [...new Set(parts)];\n}\n\nexport function decomposeMultiHop(query: string): string[] | null {\n const text = query.trim();\n\n const compareMatch = text.match(/(?:compare|trade[- ]?off[s]? (?:of|between)|difference[s]? between)\\s+(.+?)(?:[?.]|$)/i);\n if (compareMatch) {\n const items = splitListEntities(compareMatch[1]);\n if (items.length >= 2 && items.length <= 6) return items;\n }\n\n const vsMatch = text.match(/^([\\w\\s.&+#/-]+?)\\s+(?:vs\\.?|versus)\\s+([\\w\\s.&+#/-]+?)(?:\\s+(?:for|in|when|to)\\s+|[?.]|$)/i);\n if (vsMatch) {\n const items = [vsMatch[1].trim(), vsMatch[2].trim()].filter(Boolean);\n if (items.length === 2) return items;\n }\n\n const commaList = text.match(/of\\s+([\\w\\s.,&+#/-]+?(?:,\\s*and\\s+|\\s+and\\s+)[\\w\\s.&+#/-]+?)(?:\\s+(?:for|in|when|to)\\s+|[?.]|$)/i);\n if (commaList) {\n const items = splitListEntities(commaList[1]);\n if (items.length >= 2 && items.length <= 6) return items;\n }\n\n return null;\n}\n\nexport function expandQueryHeuristic(query: string): string[] {\n const max = Math.max(1, parseInt(process.env.WIGOLO_QUERY_EXPAND_VARIANTS || '5', 10));\n const trimmed = query.trim();\n\n if (COMPARE_INTRO.test(trimmed) || /,.+,.+\\band\\b/.test(trimmed)) {\n const parts = decomposeMultiHop(trimmed);\n if (parts && parts.length >= 2) {\n const cap = Math.max(max, parts.length);\n const out = [trimmed, ...parts];\n return [...new Set(out)].slice(0, cap);\n }\n }\n\n const variants: string[] = [trimmed];\n for (const suffix of DEEP_SUFFIXES) {\n if (variants.length >= max) break;\n const candidate = `${trimmed} ${suffix}`;\n if (!variants.includes(candidate)) variants.push(candidate);\n }\n return variants;\n}\n\nexport function expandIfSingle(query: string | string[]): string[] {\n if (Array.isArray(query)) return [...query];\n return expandQueryHeuristic(query);\n}\n"],"mappings":"AAEA,SAAS,oBAAoB;AAC7B,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,QAAQ;AAEP,SAAS,iBAAiB,SAA6B;AAC5D,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,UAAM,OAAO,oBAAI,IAAY;AAC7B,UAAM,aAAuB,CAAC;AAE9B,eAAW,OAAO,SAAS;AACzB,YAAM,IAAI,IAAI,YAAY,EAAE,KAAK,EAAE,QAAQ,QAAQ,GAAG;AACtD,UAAI,EAAE,WAAW,EAAG;AACpB,UAAI,KAAK,IAAI,CAAC,EAAG;AACjB,WAAK,IAAI,CAAC;AACV,iBAAW,KAAK,CAAC;AAAA,IACnB;AAEA,QAAI,WAAW,SAAS,OAAO,eAAe;AAC5C,UAAI,KAAK,6CAA6C;AAAA,QACpD,UAAU,WAAW;AAAA,QACrB,KAAK,OAAO;AAAA,MACd,CAAC;AACD,aAAO,WAAW,MAAM,GAAG,OAAO,aAAa;AAAA,IACjD;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC3D,WAAO,CAAC;AAAA,EACV;AACF;AAmBA,eAAsB,aACpB,SACA,SACA,SACuB;AACvB,QAAM,aAAgC,CAAC;AACvC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,SAAmB,CAAC;AAE1B,MAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,GAAG;AAChD,WAAO,EAAE,SAAS,CAAC,GAAG,aAAa,CAAC,GAAG,QAAQ,CAAC,EAAE;AAAA,EACpD;AAEA,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,UAAM,cAAc,OAAO;AAE3B,UAAM,qBAAqB,CAAC,EAAE,QAAQ,gBAAgB,UAAU,QAAQ,gBAAgB;AACxF,UAAM,kBAAkB,qBAAqB,IAAI;AAEjD,UAAM,gBAAqC;AAAA,MACzC,YAAY,QAAQ,aAAa;AAAA,MACjC,WAAW,QAAQ;AAAA,MACnB,UAAU,QAAQ;AAAA,MAClB,gBAAgB,QAAQ;AAAA,MACxB,gBAAgB,QAAQ;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,QAAQ,QAAQ;AAAA,MAChB,UAAU,QAAQ;AAAA,IACpB;AAKA,UAAM,aAAa,QAAQ,UAAU,KAAK,QAAQ,SAAS,IACvD,QAAQ,MAAM,GAAG,CAAC,IAClB;AAEJ,UAAM,QAAwD,CAAC;AAC/D,eAAW,UAAU,YAAY;AAC/B,iBAAW,SAAS,SAAS;AAC3B,cAAM,KAAK,EAAE,QAAQ,MAAM,CAAC;AAAA,MAC9B;AAAA,IACF;AAEA,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,aAAa;AAClD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,WAAW;AAC5C,YAAM,WAAW,MAAM,IAAI,OAAO,EAAE,QAAQ,MAAM,MAAM;AACtD,YAAI;AACF,gBAAM,UAAU,MAAM,OAAO,OAAO,OAAO,aAAa;AACxD,qBAAW,KAAK,SAAS;AACvB,uBAAW,KAAK,CAAC;AACjB,wBAAY,IAAI,OAAO,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,KAAK;AACZ,gBAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAI,KAAK,oCAAoC;AAAA,YAC3C,QAAQ,OAAO;AAAA,YACf;AAAA,YACA,OAAO;AAAA,UACT,CAAC;AACD,iBAAO,KAAK,GAAG,OAAO,IAAI,IAAI,KAAK,MAAM,GAAG,EAAE;AAAA,QAChD;AAAA,MACF,CAAC;AAED,YAAM,QAAQ,WAAW,QAAQ;AAAA,IACnC;AAEA,WAAO;AAAA,MACL,SAAS;AAAA,MACT,aAAa,CAAC,GAAG,WAAW;AAAA,MAC5B;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,uBAAuB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACvD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,aAAa,CAAC,GAAG,WAAW;AAAA,MAC5B,QAAQ,CAAC,GAAG,QAAQ,iBAAiB,OAAO,GAAG,CAAC,EAAE;AAAA,IACpD;AAAA,EACF;AACF;AAEO,SAAS,iBAAiB,SAA2B;AAC1D,MAAI;AACF,WAAO,QAAQ,IAAI,OAAK,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO,EAAE,KAAK,IAAI;AAAA,EAC7D,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC3D,WAAO;AAAA,EACT;AACF;AAEO,SAAS,aAAa,aAA2D;AACtF,MAAI;AACF,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAEtC,UAAM,WAAW,YAAY,OAAO,OAAK,EAAE,SAAS,CAAC;AACrD,QAAI,SAAS,WAAW,EAAG,QAAO,CAAC;AAEnC,UAAM,YAAY,oBAAI,IAAoB;AAC1C,UAAM,iBAAiB,oBAAI,IAA8D;AAEzF,eAAW,QAAQ,UAAU;AAC3B,eAAS,OAAO,GAAG,OAAO,KAAK,QAAQ,QAAQ;AAC7C,cAAM,OAAO,KAAK,IAAI;AACtB,YAAI;AACJ,YAAI;AACF,6BAAmB,aAAa,KAAK,GAAG;AAAA,QAC1C,QAAQ;AACN,6BAAmB,KAAK;AAAA,QAC1B;AAEA,cAAM,kBAAkB,KAAK,QAAQ,OAAO;AAC5C,cAAM,UAAU,UAAU,IAAI,gBAAgB,KAAK;AACnD,kBAAU,IAAI,kBAAkB,UAAU,eAAe;AAEzD,cAAM,WAAW,eAAe,IAAI,gBAAgB;AACpD,YAAI,CAAC,YAAY,OAAO,SAAS,UAAU;AACzC,yBAAe,IAAI,kBAAkB,EAAE,QAAQ,MAAM,UAAU,KAAK,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW;AACf,eAAW,SAAS,UAAU,OAAO,GAAG;AACtC,UAAI,QAAQ,SAAU,YAAW;AAAA,IACnC;AAEA,UAAM,SAA+B,CAAC;AACtC,eAAW,CAAC,kBAAkB,KAAK,KAAK,UAAU,QAAQ,GAAG;AAC3D,YAAM,aAAa,eAAe,IAAI,gBAAgB;AACtD,aAAO,KAAK;AAAA,QACV,GAAG,WAAW;AAAA,QACd,iBAAiB,WAAW,IAAI,QAAQ,WAAW;AAAA,MACrD,CAAC;AAAA,IACH;AAEA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,kBAAkB,EAAE,eAAe;AAC3D,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,uBAAuB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACvD,WAAO,CAAC;AAAA,EACV;AACF;AAEA,MAAM,gBAAgB,CAAC,SAAS,YAAY,YAAY,gBAAgB;AAExE,MAAM,gBAAgB;AACtB,MAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B;AAAA,EAAO;AAAA,EAAM;AAAA,EAAM;AAAA,EAAU;AAAA,EAAW;AAAA,EAAM;AAAA,EAAO;AAAA,EAAK;AAAA,EAC1D;AAAA,EAAO;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAW;AAAA,EAAc;AAAA,EAC1D;AAAA,EAAY;AAAA,EAAa;AAAA,EAAa;AACxC,CAAC;AAED,SAAS,kBAAkB,SAA2B;AACpD,QAAM,UAAU,QAAQ,QAAQ,mCAAmC,EAAE,EAAE,KAAK;AAC5E,QAAM,QAAQ,QACX,MAAM,qDAAqD,EAC3D,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,UAAU,OAAO,CAAC,aAAa,IAAI,EAAE,YAAY,CAAC,CAAC;AACvF,SAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC;AAC3B;AAEO,SAAS,kBAAkB,OAAgC;AAChE,QAAM,OAAO,MAAM,KAAK;AAExB,QAAM,eAAe,KAAK,MAAM,wFAAwF;AACxH,MAAI,cAAc;AAChB,UAAM,QAAQ,kBAAkB,aAAa,CAAC,CAAC;AAC/C,QAAI,MAAM,UAAU,KAAK,MAAM,UAAU,EAAG,QAAO;AAAA,EACrD;AAEA,QAAM,UAAU,KAAK,MAAM,6FAA6F;AACxH,MAAI,SAAS;AACX,UAAM,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AACnE,QAAI,MAAM,WAAW,EAAG,QAAO;AAAA,EACjC;AAEA,QAAM,YAAY,KAAK,MAAM,kGAAkG;AAC/H,MAAI,WAAW;AACb,UAAM,QAAQ,kBAAkB,UAAU,CAAC,CAAC;AAC5C,QAAI,MAAM,UAAU,KAAK,MAAM,UAAU,EAAG,QAAO;AAAA,EACrD;AAEA,SAAO;AACT;AAEO,SAAS,qBAAqB,OAAyB;AAC5D,QAAM,MAAM,KAAK,IAAI,GAAG,SAAS,QAAQ,IAAI,gCAAgC,KAAK,EAAE,CAAC;AACrF,QAAM,UAAU,MAAM,KAAK;AAE3B,MAAI,cAAc,KAAK,OAAO,KAAK,gBAAgB,KAAK,OAAO,GAAG;AAChE,UAAM,QAAQ,kBAAkB,OAAO;AACvC,QAAI,SAAS,MAAM,UAAU,GAAG;AAC9B,YAAM,MAAM,KAAK,IAAI,KAAK,MAAM,MAAM;AACtC,YAAM,MAAM,CAAC,SAAS,GAAG,KAAK;AAC9B,aAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,WAAqB,CAAC,OAAO;AACnC,aAAW,UAAU,eAAe;AAClC,QAAI,SAAS,UAAU,IAAK;AAC5B,UAAM,YAAY,GAAG,OAAO,IAAI,MAAM;AACtC,QAAI,CAAC,SAAS,SAAS,SAAS,EAAG,UAAS,KAAK,SAAS;AAAA,EAC5D;AACA,SAAO;AACT;AAEO,SAAS,eAAe,OAAoC;AACjE,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC,GAAG,KAAK;AAC1C,SAAO,qBAAqB,KAAK;AACnC;","names":[]}
|
package/dist/search/query.js
CHANGED
|
@@ -1,41 +1,39 @@
|
|
|
1
1
|
const MAX_QUERY_LENGTH = 200;
|
|
2
|
-
const CLAUSE_SEPARATORS = [
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
return [p];
|
|
14
|
-
return p.split(sep).map(s => s.trim()).filter(Boolean);
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
parts = parts.flatMap(p => {
|
|
18
|
-
if (p.length <= MAX_QUERY_LENGTH)
|
|
19
|
-
return [p];
|
|
20
|
-
return splitAtWordBoundary(p);
|
|
2
|
+
const CLAUSE_SEPARATORS = [". ", "? ", "! ", "; ", ", and ", ", or ", " - ", " \u2014 "];
|
|
3
|
+
function decomposeQuery(query) {
|
|
4
|
+
if (query.length <= MAX_QUERY_LENGTH) {
|
|
5
|
+
return [query];
|
|
6
|
+
}
|
|
7
|
+
let parts = [query];
|
|
8
|
+
for (const sep of CLAUSE_SEPARATORS) {
|
|
9
|
+
if (parts.every((p) => p.length <= MAX_QUERY_LENGTH)) break;
|
|
10
|
+
parts = parts.flatMap((p) => {
|
|
11
|
+
if (p.length <= MAX_QUERY_LENGTH) return [p];
|
|
12
|
+
return p.split(sep).map((s) => s.trim()).filter(Boolean);
|
|
21
13
|
});
|
|
22
|
-
|
|
14
|
+
}
|
|
15
|
+
parts = parts.flatMap((p) => {
|
|
16
|
+
if (p.length <= MAX_QUERY_LENGTH) return [p];
|
|
17
|
+
return splitAtWordBoundary(p);
|
|
18
|
+
});
|
|
19
|
+
return parts;
|
|
23
20
|
}
|
|
24
21
|
function splitAtWordBoundary(text) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
current = current ? `${current} ${word}` : word;
|
|
35
|
-
}
|
|
22
|
+
const chunks = [];
|
|
23
|
+
const words = text.split(" ");
|
|
24
|
+
let current = "";
|
|
25
|
+
for (const word of words) {
|
|
26
|
+
if (current.length + word.length + 1 > MAX_QUERY_LENGTH && current) {
|
|
27
|
+
chunks.push(current);
|
|
28
|
+
current = word;
|
|
29
|
+
} else {
|
|
30
|
+
current = current ? `${current} ${word}` : word;
|
|
36
31
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
32
|
+
}
|
|
33
|
+
if (current) chunks.push(current);
|
|
34
|
+
return chunks;
|
|
40
35
|
}
|
|
36
|
+
export {
|
|
37
|
+
decomposeQuery
|
|
38
|
+
};
|
|
41
39
|
//# sourceMappingURL=query.js.map
|
package/dist/search/query.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"sources":["../../src/search/query.ts"],"sourcesContent":["const MAX_QUERY_LENGTH = 200;\nconst CLAUSE_SEPARATORS = ['. ', '? ', '! ', '; ', ', and ', ', or ', ' - ', ' — '];\n\nexport function decomposeQuery(query: string): string[] {\n if (query.length <= MAX_QUERY_LENGTH) {\n return [query];\n }\n\n let parts = [query];\n\n for (const sep of CLAUSE_SEPARATORS) {\n if (parts.every(p => p.length <= MAX_QUERY_LENGTH)) break;\n\n parts = parts.flatMap(p => {\n if (p.length <= MAX_QUERY_LENGTH) return [p];\n return p.split(sep).map(s => s.trim()).filter(Boolean);\n });\n }\n\n parts = parts.flatMap(p => {\n if (p.length <= MAX_QUERY_LENGTH) return [p];\n return splitAtWordBoundary(p);\n });\n\n return parts;\n}\n\nfunction splitAtWordBoundary(text: string): string[] {\n const chunks: string[] = [];\n const words = text.split(' ');\n let current = '';\n\n for (const word of words) {\n if (current.length + word.length + 1 > MAX_QUERY_LENGTH && current) {\n chunks.push(current);\n current = word;\n } else {\n current = current ? `${current} ${word}` : word;\n }\n }\n if (current) chunks.push(current);\n\n return chunks;\n}\n"],"mappings":"AAAA,MAAM,mBAAmB;AACzB,MAAM,oBAAoB,CAAC,MAAM,MAAM,MAAM,MAAM,UAAU,SAAS,OAAO,UAAK;AAE3E,SAAS,eAAe,OAAyB;AACtD,MAAI,MAAM,UAAU,kBAAkB;AACpC,WAAO,CAAC,KAAK;AAAA,EACf;AAEA,MAAI,QAAQ,CAAC,KAAK;AAElB,aAAW,OAAO,mBAAmB;AACnC,QAAI,MAAM,MAAM,OAAK,EAAE,UAAU,gBAAgB,EAAG;AAEpD,YAAQ,MAAM,QAAQ,OAAK;AACzB,UAAI,EAAE,UAAU,iBAAkB,QAAO,CAAC,CAAC;AAC3C,aAAO,EAAE,MAAM,GAAG,EAAE,IAAI,OAAK,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AAAA,IACvD,CAAC;AAAA,EACH;AAEA,UAAQ,MAAM,QAAQ,OAAK;AACzB,QAAI,EAAE,UAAU,iBAAkB,QAAO,CAAC,CAAC;AAC3C,WAAO,oBAAoB,CAAC;AAAA,EAC9B,CAAC;AAED,SAAO;AACT;AAEA,SAAS,oBAAoB,MAAwB;AACnD,QAAM,SAAmB,CAAC;AAC1B,QAAM,QAAQ,KAAK,MAAM,GAAG;AAC5B,MAAI,UAAU;AAEd,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,SAAS,KAAK,SAAS,IAAI,oBAAoB,SAAS;AAClE,aAAO,KAAK,OAAO;AACnB,gBAAU;AAAA,IACZ,OAAO;AACL,gBAAU,UAAU,GAAG,OAAO,IAAI,IAAI,KAAK;AAAA,IAC7C;AAAA,EACF;AACA,MAAI,QAAS,QAAO,KAAK,OAAO;AAEhC,SAAO;AACT;","names":[]}
|
package/dist/search/rerank.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
import type { MergedSearchResult } from './dedup.js';
|
|
2
|
-
export declare function rerankResults(query: string, results: MergedSearchResult[]
|
|
2
|
+
export declare function rerankResults(query: string, results: MergedSearchResult[], opts?: {
|
|
3
|
+
skip?: boolean;
|
|
4
|
+
}): Promise<MergedSearchResult[]>;
|
|
3
5
|
//# sourceMappingURL=rerank.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAUrD,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAAE,EAC7B,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAkC/B"}
|
package/dist/search/rerank.js
CHANGED
|
@@ -1,40 +1,49 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
1
|
+
import { getRerankProvider } from "../providers/rerank-provider.js";
|
|
2
|
+
import { applyRecencyBoost } from "./reranker/recency-boost.js";
|
|
3
|
+
import { applyAuthorityBoost } from "./reranker/authority-boost.js";
|
|
4
|
+
import { applyConsensusBoost } from "./reranker/consensus-boost.js";
|
|
5
|
+
import { getConfig } from "../config.js";
|
|
6
|
+
import { createLogger } from "../logger.js";
|
|
7
|
+
const log = createLogger("search");
|
|
8
|
+
async function rerankResults(query, results, opts = {}) {
|
|
9
|
+
if (opts.skip) return results;
|
|
10
|
+
const config = getConfig();
|
|
11
|
+
if (results.length === 0) return results;
|
|
12
|
+
if (config.reranker === "onnx") {
|
|
13
|
+
try {
|
|
14
|
+
const provider = await getRerankProvider();
|
|
15
|
+
const candidates = results.map((r, i) => ({
|
|
16
|
+
id: String(i),
|
|
17
|
+
text: `${r.title}
|
|
18
|
+
${r.snippet}`
|
|
19
|
+
}));
|
|
20
|
+
const ranked = await provider.rerank(query, candidates);
|
|
21
|
+
const reordered = ranked.map((s) => ({
|
|
22
|
+
...results[Number(s.id)],
|
|
23
|
+
relevance_score: s.score
|
|
24
|
+
}));
|
|
25
|
+
const consensusBoosted2 = applyConsensusBoost(reordered);
|
|
26
|
+
const authorityBoosted2 = applyAuthorityBoost(query, consensusBoosted2);
|
|
27
|
+
const boosted2 = applyRecencyBoost(query, authorityBoosted2);
|
|
28
|
+
boosted2.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
29
|
+
return applyThreshold(boosted2, config.relevanceThreshold);
|
|
30
|
+
} catch (err) {
|
|
31
|
+
log.warn("Rerank failed, falling back to passthrough", { error: String(err) });
|
|
28
32
|
}
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
} else if (config.reranker !== "none") {
|
|
34
|
+
log.warn("Unknown reranker configured, passing through", { reranker: config.reranker });
|
|
35
|
+
}
|
|
36
|
+
const consensusBoosted = applyConsensusBoost(results);
|
|
37
|
+
const authorityBoosted = applyAuthorityBoost(query, consensusBoosted);
|
|
38
|
+
const boosted = applyRecencyBoost(query, authorityBoosted);
|
|
39
|
+
boosted.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
40
|
+
return applyThreshold(boosted, config.relevanceThreshold);
|
|
34
41
|
}
|
|
35
42
|
function applyThreshold(results, threshold) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
return results.filter((r) => r.relevance_score >= threshold);
|
|
43
|
+
if (!threshold || threshold <= 0) return results;
|
|
44
|
+
return results.filter((r) => r.relevance_score >= threshold);
|
|
39
45
|
}
|
|
46
|
+
export {
|
|
47
|
+
rerankResults
|
|
48
|
+
};
|
|
40
49
|
//# sourceMappingURL=rerank.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"sources":["../../src/search/rerank.ts"],"sourcesContent":["import type { MergedSearchResult } from './dedup.js';\nimport { getRerankProvider } from '../providers/rerank-provider.js';\nimport { applyRecencyBoost } from './reranker/recency-boost.js';\nimport { applyAuthorityBoost } from './reranker/authority-boost.js';\nimport { applyConsensusBoost } from './reranker/consensus-boost.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('search');\n\nexport async function rerankResults(\n query: string,\n results: MergedSearchResult[],\n opts: { skip?: boolean } = {},\n): Promise<MergedSearchResult[]> {\n if (opts.skip) return results;\n const config = getConfig();\n if (results.length === 0) return results;\n\n if (config.reranker === 'onnx') {\n try {\n const provider = await getRerankProvider();\n const candidates = results.map((r, i) => ({\n id: String(i),\n text: `${r.title}\\n${r.snippet}`,\n }));\n const ranked = await provider.rerank(query, candidates);\n const reordered = ranked.map((s) => ({\n ...results[Number(s.id)],\n relevance_score: s.score,\n }));\n const consensusBoosted = applyConsensusBoost(reordered);\n const authorityBoosted = applyAuthorityBoost(query, consensusBoosted);\n const boosted = applyRecencyBoost(query, authorityBoosted);\n boosted.sort((a, b) => b.relevance_score - a.relevance_score);\n return applyThreshold(boosted, config.relevanceThreshold);\n } catch (err) {\n log.warn('Rerank failed, falling back to passthrough', { error: String(err) });\n }\n } else if (config.reranker !== 'none') {\n log.warn('Unknown reranker configured, passing through', { reranker: config.reranker });\n }\n\n const consensusBoosted = applyConsensusBoost(results);\n const authorityBoosted = applyAuthorityBoost(query, consensusBoosted);\n const boosted = applyRecencyBoost(query, authorityBoosted);\n boosted.sort((a, b) => b.relevance_score - a.relevance_score);\n return applyThreshold(boosted, config.relevanceThreshold);\n}\n\nfunction applyThreshold(\n results: MergedSearchResult[],\n threshold: number,\n): MergedSearchResult[] {\n if (!threshold || threshold <= 0) return results;\n return results.filter((r) => r.relevance_score >= threshold);\n}\n"],"mappings":"AACA,SAAS,yBAAyB;AAClC,SAAS,yBAAyB;AAClC,SAAS,2BAA2B;AACpC,SAAS,2BAA2B;AACpC,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,eAAsB,cACpB,OACA,SACA,OAA2B,CAAC,GACG;AAC/B,MAAI,KAAK,KAAM,QAAO;AACtB,QAAM,SAAS,UAAU;AACzB,MAAI,QAAQ,WAAW,EAAG,QAAO;AAEjC,MAAI,OAAO,aAAa,QAAQ;AAC9B,QAAI;AACF,YAAM,WAAW,MAAM,kBAAkB;AACzC,YAAM,aAAa,QAAQ,IAAI,CAAC,GAAG,OAAO;AAAA,QACxC,IAAI,OAAO,CAAC;AAAA,QACZ,MAAM,GAAG,EAAE,KAAK;AAAA,EAAK,EAAE,OAAO;AAAA,MAChC,EAAE;AACF,YAAM,SAAS,MAAM,SAAS,OAAO,OAAO,UAAU;AACtD,YAAM,YAAY,OAAO,IAAI,CAAC,OAAO;AAAA,QACnC,GAAG,QAAQ,OAAO,EAAE,EAAE,CAAC;AAAA,QACvB,iBAAiB,EAAE;AAAA,MACrB,EAAE;AACF,YAAMA,oBAAmB,oBAAoB,SAAS;AACtD,YAAMC,oBAAmB,oBAAoB,OAAOD,iBAAgB;AACpE,YAAME,WAAU,kBAAkB,OAAOD,iBAAgB;AACzD,MAAAC,SAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,kBAAkB,EAAE,eAAe;AAC5D,aAAO,eAAeA,UAAS,OAAO,kBAAkB;AAAA,IAC1D,SAAS,KAAK;AACZ,UAAI,KAAK,8CAA8C,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC/E;AAAA,EACF,WAAW,OAAO,aAAa,QAAQ;AACrC,QAAI,KAAK,gDAAgD,EAAE,UAAU,OAAO,SAAS,CAAC;AAAA,EACxF;AAEA,QAAM,mBAAmB,oBAAoB,OAAO;AACpD,QAAM,mBAAmB,oBAAoB,OAAO,gBAAgB;AACpE,QAAM,UAAU,kBAAkB,OAAO,gBAAgB;AACzD,UAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,kBAAkB,EAAE,eAAe;AAC5D,SAAO,eAAe,SAAS,OAAO,kBAAkB;AAC1D;AAEA,SAAS,eACP,SACA,WACsB;AACtB,MAAI,CAAC,aAAa,aAAa,EAAG,QAAO;AACzC,SAAO,QAAQ,OAAO,CAAC,MAAM,EAAE,mBAAmB,SAAS;AAC7D;","names":["consensusBoosted","authorityBoosted","boosted"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"authority-boost.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/authority-boost.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAwEtD,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAAE,GAC5B,kBAAkB,EAAE,CA6CtB"}
|