@staticn0va/wigolo 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +146 -227
- package/SKILL.md +382 -0
- package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
- package/assets/blocks/claude-code/wigolo-command.md +40 -0
- package/assets/blocks/cursor/wigolo.mdc +46 -0
- package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
- package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
- package/assets/skills/wigolo/SKILL.md +50 -0
- package/assets/skills/wigolo/rules/cache-first.md +30 -0
- package/assets/skills/wigolo/rules/synthesis.md +43 -0
- package/assets/skills/wigolo-agent/SKILL.md +73 -0
- package/assets/skills/wigolo-crawl/SKILL.md +60 -0
- package/assets/skills/wigolo-extract/SKILL.md +59 -0
- package/assets/skills/wigolo-fetch/SKILL.md +65 -0
- package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
- package/assets/skills/wigolo-research/SKILL.md +77 -0
- package/assets/skills/wigolo-search/SKILL.md +78 -0
- package/dist/agent/executor.d.ts +33 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +233 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +238 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +13 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +271 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/agent/relevance.d.ts +15 -0
- package/dist/agent/relevance.d.ts.map +1 -0
- package/dist/agent/relevance.js +60 -0
- package/dist/agent/relevance.js.map +1 -0
- package/dist/cache/backfill-embeddings.d.ts +23 -0
- package/dist/cache/backfill-embeddings.d.ts.map +1 -0
- package/dist/cache/backfill-embeddings.js +105 -0
- package/dist/cache/backfill-embeddings.js.map +1 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/db.d.ts +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +94 -22
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +82 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/migrations/runner.d.ts +29 -0
- package/dist/cache/migrations/runner.d.ts.map +1 -0
- package/dist/cache/migrations/runner.js +147 -0
- package/dist/cache/migrations/runner.js.map +1 -0
- package/dist/cache/sqlite-vec-store.d.ts +42 -0
- package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
- package/dist/cache/sqlite-vec-store.js +176 -0
- package/dist/cache/sqlite-vec-store.js.map +1 -0
- package/dist/cache/store.d.ts +47 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +364 -168
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/agents/antigravity.d.ts +20 -0
- package/dist/cli/agents/antigravity.d.ts.map +1 -0
- package/dist/cli/agents/antigravity.js +49 -0
- package/dist/cli/agents/antigravity.js.map +1 -0
- package/dist/cli/agents/claude-code.d.ts +25 -0
- package/dist/cli/agents/claude-code.d.ts.map +1 -0
- package/dist/cli/agents/claude-code.js +111 -0
- package/dist/cli/agents/claude-code.js.map +1 -0
- package/dist/cli/agents/cursor.d.ts +21 -0
- package/dist/cli/agents/cursor.d.ts.map +1 -0
- package/dist/cli/agents/cursor.js +58 -0
- package/dist/cli/agents/cursor.js.map +1 -0
- package/dist/cli/agents/gemini-cli.d.ts +21 -0
- package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
- package/dist/cli/agents/gemini-cli.js +55 -0
- package/dist/cli/agents/gemini-cli.js.map +1 -0
- package/dist/cli/agents/registry.d.ts +21 -0
- package/dist/cli/agents/registry.d.ts.map +1 -0
- package/dist/cli/agents/registry.js +27 -0
- package/dist/cli/agents/registry.js.map +1 -0
- package/dist/cli/agents/utils.d.ts +26 -0
- package/dist/cli/agents/utils.d.ts.map +1 -0
- package/dist/cli/agents/utils.js +136 -0
- package/dist/cli/agents/utils.js.map +1 -0
- package/dist/cli/agents/vscode.d.ts +21 -0
- package/dist/cli/agents/vscode.d.ts.map +1 -0
- package/dist/cli/agents/vscode.js +62 -0
- package/dist/cli/agents/vscode.js.map +1 -0
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +94 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/backfill.d.ts +2 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +58 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +61 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/doctor.d.ts +8 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +344 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +42 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/help.d.ts +6 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +63 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +35 -7
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +201 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +185 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/setup-mcp.d.ts +2 -0
- package/dist/cli/setup-mcp.d.ts.map +1 -0
- package/dist/cli/setup-mcp.js +114 -0
- package/dist/cli/setup-mcp.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/shutdown.d.ts +2 -0
- package/dist/cli/shutdown.d.ts.map +1 -0
- package/dist/cli/shutdown.js +26 -0
- package/dist/cli/shutdown.js.map +1 -0
- package/dist/cli/status.d.ts +2 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +31 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/telemetry.d.ts +10 -0
- package/dist/cli/telemetry.d.ts.map +1 -0
- package/dist/cli/telemetry.js +56 -0
- package/dist/cli/telemetry.js.map +1 -0
- package/dist/cli/tui/agents-types.d.ts +28 -0
- package/dist/cli/tui/agents-types.d.ts.map +1 -0
- package/dist/cli/tui/agents-types.js +1 -0
- package/dist/cli/tui/agents-types.js.map +1 -0
- package/dist/cli/tui/agents.d.ts +11 -0
- package/dist/cli/tui/agents.d.ts.map +1 -0
- package/dist/cli/tui/agents.js +93 -0
- package/dist/cli/tui/agents.js.map +1 -0
- package/dist/cli/tui/banner.d.ts +3 -0
- package/dist/cli/tui/banner.d.ts.map +1 -0
- package/dist/cli/tui/banner.js +30 -0
- package/dist/cli/tui/banner.js.map +1 -0
- package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
- package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/AgentSelect.js +116 -0
- package/dist/cli/tui/components/AgentSelect.js.map +1 -0
- package/dist/cli/tui/components/Banner.d.ts +6 -0
- package/dist/cli/tui/components/Banner.d.ts.map +1 -0
- package/dist/cli/tui/components/Banner.js +25 -0
- package/dist/cli/tui/components/Banner.js.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.js +19 -0
- package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
- package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
- package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
- package/dist/cli/tui/components/InstallProgress.js +67 -0
- package/dist/cli/tui/components/InstallProgress.js.map +1 -0
- package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
- package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
- package/dist/cli/tui/components/SkillInstall.js +94 -0
- package/dist/cli/tui/components/SkillInstall.js.map +1 -0
- package/dist/cli/tui/components/Summary.d.ts +22 -0
- package/dist/cli/tui/components/Summary.d.ts.map +1 -0
- package/dist/cli/tui/components/Summary.js +135 -0
- package/dist/cli/tui/components/Summary.js.map +1 -0
- package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
- package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/components/SystemCheck.js +71 -0
- package/dist/cli/tui/components/SystemCheck.js.map +1 -0
- package/dist/cli/tui/components/Verification.d.ts +8 -0
- package/dist/cli/tui/components/Verification.d.ts.map +1 -0
- package/dist/cli/tui/components/Verification.js +63 -0
- package/dist/cli/tui/components/Verification.js.map +1 -0
- package/dist/cli/tui/config-writer-cli.d.ts +12 -0
- package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-cli.js +39 -0
- package/dist/cli/tui/config-writer-cli.js.map +1 -0
- package/dist/cli/tui/config-writer-json.d.ts +16 -0
- package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-json.js +86 -0
- package/dist/cli/tui/config-writer-json.js.map +1 -0
- package/dist/cli/tui/config-writer-toml.d.ts +16 -0
- package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-toml.js +83 -0
- package/dist/cli/tui/config-writer-toml.js.map +1 -0
- package/dist/cli/tui/config-writer.d.ts +25 -0
- package/dist/cli/tui/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/config-writer.js +101 -0
- package/dist/cli/tui/config-writer.js.map +1 -0
- package/dist/cli/tui/detect-helpers.d.ts +6 -0
- package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
- package/dist/cli/tui/detect-helpers.js +45 -0
- package/dist/cli/tui/detect-helpers.js.map +1 -0
- package/dist/cli/tui/extras-prompt.d.ts +7 -0
- package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
- package/dist/cli/tui/extras-prompt.js +42 -0
- package/dist/cli/tui/extras-prompt.js.map +1 -0
- package/dist/cli/tui/flags-types.d.ts +19 -0
- package/dist/cli/tui/flags-types.d.ts.map +1 -0
- package/dist/cli/tui/flags-types.js +23 -0
- package/dist/cli/tui/flags-types.js.map +1 -0
- package/dist/cli/tui/flags.d.ts +5 -0
- package/dist/cli/tui/flags.d.ts.map +1 -0
- package/dist/cli/tui/flags.js +132 -0
- package/dist/cli/tui/flags.js.map +1 -0
- package/dist/cli/tui/format.d.ts +14 -0
- package/dist/cli/tui/format.d.ts.map +1 -0
- package/dist/cli/tui/format.js +37 -0
- package/dist/cli/tui/format.js.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
- package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
- package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
- package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useInstall.js +90 -0
- package/dist/cli/tui/hooks/useInstall.js.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
- package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
- package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
- package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useVerify.js +71 -0
- package/dist/cli/tui/hooks/useVerify.js.map +1 -0
- package/dist/cli/tui/ink-init.d.ts +2 -0
- package/dist/cli/tui/ink-init.d.ts.map +1 -0
- package/dist/cli/tui/ink-init.js +198 -0
- package/dist/cli/tui/ink-init.js.map +1 -0
- package/dist/cli/tui/reporter-auto.d.ts +7 -0
- package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
- package/dist/cli/tui/reporter-auto.js +15 -0
- package/dist/cli/tui/reporter-auto.js.map +1 -0
- package/dist/cli/tui/reporter.d.ts +26 -0
- package/dist/cli/tui/reporter.d.ts.map +1 -0
- package/dist/cli/tui/reporter.js +32 -0
- package/dist/cli/tui/reporter.js.map +1 -0
- package/dist/cli/tui/run-command.d.ts +14 -0
- package/dist/cli/tui/run-command.d.ts.map +1 -0
- package/dist/cli/tui/run-command.js +72 -0
- package/dist/cli/tui/run-command.js.map +1 -0
- package/dist/cli/tui/select-agents.d.ts +6 -0
- package/dist/cli/tui/select-agents.d.ts.map +1 -0
- package/dist/cli/tui/select-agents.js +32 -0
- package/dist/cli/tui/select-agents.js.map +1 -0
- package/dist/cli/tui/status-agents.d.ts +11 -0
- package/dist/cli/tui/status-agents.d.ts.map +1 -0
- package/dist/cli/tui/status-agents.js +53 -0
- package/dist/cli/tui/status-agents.js.map +1 -0
- package/dist/cli/tui/status-cache.d.ts +6 -0
- package/dist/cli/tui/status-cache.d.ts.map +1 -0
- package/dist/cli/tui/status-cache.js +39 -0
- package/dist/cli/tui/status-cache.js.map +1 -0
- package/dist/cli/tui/status-format.d.ts +14 -0
- package/dist/cli/tui/status-format.d.ts.map +1 -0
- package/dist/cli/tui/status-format.js +41 -0
- package/dist/cli/tui/status-format.js.map +1 -0
- package/dist/cli/tui/status-python.d.ts +6 -0
- package/dist/cli/tui/status-python.d.ts.map +1 -0
- package/dist/cli/tui/status-python.js +30 -0
- package/dist/cli/tui/status-python.js.map +1 -0
- package/dist/cli/tui/system-check.d.ts +24 -0
- package/dist/cli/tui/system-check.d.ts.map +1 -0
- package/dist/cli/tui/system-check.js +103 -0
- package/dist/cli/tui/system-check.js.map +1 -0
- package/dist/cli/tui/tui-reporter.d.ts +19 -0
- package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
- package/dist/cli/tui/tui-reporter.js +95 -0
- package/dist/cli/tui/tui-reporter.js.map +1 -0
- package/dist/cli/tui/utils/config-writer.d.ts +3 -0
- package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/utils/config-writer.js +22 -0
- package/dist/cli/tui/utils/config-writer.js.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.js +11 -0
- package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
- package/dist/cli/tui/verify-suggestions.d.ts +5 -0
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
- package/dist/cli/tui/verify-suggestions.js +20 -0
- package/dist/cli/tui/verify-suggestions.js.map +1 -0
- package/dist/cli/tui/verify.d.ts +14 -0
- package/dist/cli/tui/verify.d.ts.map +1 -0
- package/dist/cli/tui/verify.js +101 -0
- package/dist/cli/tui/verify.js.map +1 -0
- package/dist/cli/tui/version.d.ts +2 -0
- package/dist/cli/tui/version.d.ts.map +1 -0
- package/dist/cli/tui/version.js +14 -0
- package/dist/cli/tui/version.js.map +1 -0
- package/dist/cli/uninstall.d.ts +2 -0
- package/dist/cli/uninstall.d.ts.map +1 -0
- package/dist/cli/uninstall.js +57 -0
- package/dist/cli/uninstall.js.map +1 -0
- package/dist/cli/warmup.d.ts +10 -2
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +226 -93
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +28 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +106 -56
- package/dist/config.js.map +1 -1
- package/dist/crawl/crawler.d.ts +6 -0
- package/dist/crawl/crawler.d.ts.map +1 -1
- package/dist/crawl/crawler.js +210 -209
- package/dist/crawl/crawler.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +124 -81
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/crawl/etag-incremental.d.ts +43 -0
- package/dist/crawl/etag-incremental.d.ts.map +1 -0
- package/dist/crawl/etag-incremental.js +94 -0
- package/dist/crawl/etag-incremental.js.map +1 -0
- package/dist/crawl/index-to-vec.d.ts +10 -0
- package/dist/crawl/index-to-vec.d.ts.map +1 -0
- package/dist/crawl/index-to-vec.js +44 -0
- package/dist/crawl/index-to-vec.js.map +1 -0
- package/dist/crawl/mapper.js +136 -164
- package/dist/crawl/mapper.js.map +1 -1
- package/dist/crawl/rate-limiter.js +63 -66
- package/dist/crawl/rate-limiter.js.map +1 -1
- package/dist/crawl/robots.js +58 -57
- package/dist/crawl/robots.js.map +1 -1
- package/dist/crawl/sitemap-first.d.ts +12 -0
- package/dist/crawl/sitemap-first.d.ts.map +1 -0
- package/dist/crawl/sitemap-first.js +47 -0
- package/dist/crawl/sitemap-first.js.map +1 -0
- package/dist/crawl/sitemap.js +33 -32
- package/dist/crawl/sitemap.js.map +1 -1
- package/dist/crawl/url-utils.d.ts +1 -0
- package/dist/crawl/url-utils.d.ts.map +1 -1
- package/dist/crawl/url-utils.js +49 -37
- package/dist/crawl/url-utils.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +33 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +275 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +93 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +233 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/fastembed-provider.d.ts +19 -0
- package/dist/embedding/fastembed-provider.d.ts.map +1 -0
- package/dist/embedding/fastembed-provider.js +51 -0
- package/dist/embedding/fastembed-provider.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +234 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +52 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +27 -23
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/extract.d.ts.map +1 -1
- package/dist/extraction/extract.js +76 -76
- package/dist/extraction/extract.js.map +1 -1
- package/dist/extraction/jsonld.js +50 -54
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +30 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +130 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown-sanitize.d.ts +2 -0
- package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
- package/dist/extraction/markdown-sanitize.js +151 -0
- package/dist/extraction/markdown-sanitize.js.map +1 -0
- package/dist/extraction/markdown.d.ts +11 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +195 -91
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts +8 -0
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +57 -91
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +28 -29
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +135 -72
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +81 -91
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +87 -95
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +46 -54
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +71 -80
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +173 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/extraction/structured.d.ts +4 -0
- package/dist/extraction/structured.d.ts.map +1 -0
- package/dist/extraction/structured.js +163 -0
- package/dist/extraction/structured.js.map +1 -0
- package/dist/extraction/v1/classifier.d.ts +3 -0
- package/dist/extraction/v1/classifier.d.ts.map +1 -0
- package/dist/extraction/v1/classifier.js +110 -0
- package/dist/extraction/v1/classifier.js.map +1 -0
- package/dist/extraction/v1/extract-provider.d.ts +16 -0
- package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
- package/dist/extraction/v1/extract-provider.js +43 -0
- package/dist/extraction/v1/extract-provider.js.map +1 -0
- package/dist/extraction/v1/local-llm.d.ts +8 -0
- package/dist/extraction/v1/local-llm.d.ts.map +1 -0
- package/dist/extraction/v1/local-llm.js +34 -0
- package/dist/extraction/v1/local-llm.js.map +1 -0
- package/dist/extraction/v1/news.d.ts +3 -0
- package/dist/extraction/v1/news.d.ts.map +1 -0
- package/dist/extraction/v1/news.js +61 -0
- package/dist/extraction/v1/news.js.map +1 -0
- package/dist/extraction/v1/product.d.ts +3 -0
- package/dist/extraction/v1/product.d.ts.map +1 -0
- package/dist/extraction/v1/product.js +166 -0
- package/dist/extraction/v1/product.js.map +1 -0
- package/dist/extraction/v1/recipe.d.ts +3 -0
- package/dist/extraction/v1/recipe.d.ts.map +1 -0
- package/dist/extraction/v1/recipe.js +136 -0
- package/dist/extraction/v1/recipe.js.map +1 -0
- package/dist/extraction/v1/routed.d.ts +17 -0
- package/dist/extraction/v1/routed.d.ts.map +1 -0
- package/dist/extraction/v1/routed.js +68 -0
- package/dist/extraction/v1/routed.js.map +1 -0
- package/dist/extraction/v1/schemas/Article.d.ts +11 -0
- package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Article.js +23 -0
- package/dist/extraction/v1/schemas/Article.js.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.js +122 -0
- package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
- package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
- package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Paper.js +156 -0
- package/dist/extraction/v1/schemas/Paper.js.map +1 -0
- package/dist/extraction/v1/schemas/Product.d.ts +17 -0
- package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Product.js +149 -0
- package/dist/extraction/v1/schemas/Product.js.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.js +160 -0
- package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
- package/dist/extraction/v1/schemas/index.d.ts +13 -0
- package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/index.js +44 -0
- package/dist/extraction/v1/schemas/index.js.map +1 -0
- package/dist/extraction/v1/site-extractors.d.ts +5 -0
- package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
- package/dist/extraction/v1/site-extractors.js +31 -0
- package/dist/extraction/v1/site-extractors.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +88 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +56 -26
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +303 -127
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +72 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +89 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/content-check.js +39 -46
- package/dist/fetch/content-check.js.map +1 -1
- package/dist/fetch/error-describe.d.ts +7 -0
- package/dist/fetch/error-describe.d.ts.map +1 -0
- package/dist/fetch/error-describe.js +37 -0
- package/dist/fetch/error-describe.js.map +1 -0
- package/dist/fetch/http-client.d.ts +4 -0
- package/dist/fetch/http-client.d.ts.map +1 -1
- package/dist/fetch/http-client.js +147 -128
- package/dist/fetch/http-client.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +174 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/fetch/playwright-tier.d.ts +19 -0
- package/dist/fetch/playwright-tier.d.ts.map +1 -0
- package/dist/fetch/playwright-tier.js +76 -0
- package/dist/fetch/playwright-tier.js.map +1 -0
- package/dist/fetch/router.d.ts +49 -3
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +187 -81
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +102 -17
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +31 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +245 -0
- package/dist/instructions.js.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.js +41 -0
- package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
- package/dist/integrations/cloud/llm/cache.d.ts +5 -0
- package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/cache.js +49 -0
- package/dist/integrations/cloud/llm/cache.js.map +1 -0
- package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
- package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/gemini.js +37 -0
- package/dist/integrations/cloud/llm/gemini.js.map +1 -0
- package/dist/integrations/cloud/llm/groq.d.ts +3 -0
- package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/groq.js +74 -0
- package/dist/integrations/cloud/llm/groq.js.map +1 -0
- package/dist/integrations/cloud/llm/hash.d.ts +3 -0
- package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/hash.js +26 -0
- package/dist/integrations/cloud/llm/hash.js.map +1 -0
- package/dist/integrations/cloud/llm/model-select.d.ts +5 -0
- package/dist/integrations/cloud/llm/model-select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/model-select.js +32 -0
- package/dist/integrations/cloud/llm/model-select.js.map +1 -0
- package/dist/integrations/cloud/llm/openai.d.ts +3 -0
- package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/openai.js +43 -0
- package/dist/integrations/cloud/llm/openai.js.map +1 -0
- package/dist/integrations/cloud/llm/run.d.ts +27 -0
- package/dist/integrations/cloud/llm/run.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/run.js +99 -0
- package/dist/integrations/cloud/llm/run.js.map +1 -0
- package/dist/integrations/cloud/llm/select.d.ts +5 -0
- package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/select.js +30 -0
- package/dist/integrations/cloud/llm/select.js.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts +19 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.js +103 -0
- package/dist/integrations/cloud/llm/text-adapters.js.map +1 -0
- package/dist/integrations/cloud/llm/types.d.ts +24 -0
- package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/types.js +1 -0
- package/dist/integrations/cloud/llm/types.js.map +1 -0
- package/dist/integrations/cloud/llm/validate.d.ts +6 -0
- package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/validate.js +63 -0
- package/dist/integrations/cloud/llm/validate.js.map +1 -0
- package/dist/logger.d.ts +4 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +71 -30
- package/dist/logger.js.map +1 -1
- package/dist/pdf-parse.d.js +1 -0
- package/dist/pdf-parse.d.js.map +1 -0
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +157 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +71 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +79 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/providers/embed-provider.d.ts +11 -0
- package/dist/providers/embed-provider.d.ts.map +1 -0
- package/dist/providers/embed-provider.js +24 -0
- package/dist/providers/embed-provider.js.map +1 -0
- package/dist/providers/extract-provider.d.ts +23 -0
- package/dist/providers/extract-provider.d.ts.map +1 -0
- package/dist/providers/extract-provider.js +25 -0
- package/dist/providers/extract-provider.js.map +1 -0
- package/dist/providers/rerank-provider.d.ts +17 -0
- package/dist/providers/rerank-provider.d.ts.map +1 -0
- package/dist/providers/rerank-provider.js +41 -0
- package/dist/providers/rerank-provider.js.map +1 -0
- package/dist/providers/search-provider.d.ts +25 -0
- package/dist/providers/search-provider.d.ts.map +1 -0
- package/dist/providers/search-provider.js +44 -0
- package/dist/providers/search-provider.js.map +1 -0
- package/dist/providers/vector-store.d.ts +27 -0
- package/dist/providers/vector-store.d.ts.map +1 -0
- package/dist/providers/vector-store.js +27 -0
- package/dist/providers/vector-store.js.map +1 -0
- package/dist/python-env.d.ts +9 -0
- package/dist/python-env.d.ts.map +1 -0
- package/dist/python-env.js +13 -0
- package/dist/python-env.js.map +1 -0
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +62 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +43 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +44 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +47 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +67 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +74 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +65 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +74 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +1 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +13 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +283 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +86 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +184 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/research/branch-exploration.d.ts +14 -0
- package/dist/research/branch-exploration.d.ts.map +1 -0
- package/dist/research/branch-exploration.js +100 -0
- package/dist/research/branch-exploration.js.map +1 -0
- package/dist/research/brief.d.ts +6 -0
- package/dist/research/brief.d.ts.map +1 -0
- package/dist/research/brief.js +246 -0
- package/dist/research/brief.js.map +1 -0
- package/dist/research/citation-graph.d.ts +9 -0
- package/dist/research/citation-graph.d.ts.map +1 -0
- package/dist/research/citation-graph.js +114 -0
- package/dist/research/citation-graph.js.map +1 -0
- package/dist/research/decompose.d.ts +14 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +439 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +269 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesis-local.d.ts +19 -0
- package/dist/research/synthesis-local.d.ts.map +1 -0
- package/dist/research/synthesis-local.js +62 -0
- package/dist/research/synthesis-local.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +137 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +33 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +244 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/search/dedup.d.ts +1 -0
- package/dist/search/dedup.d.ts.map +1 -1
- package/dist/search/dedup.js +40 -32
- package/dist/search/dedup.js.map +1 -1
- package/dist/search/engines/arxiv.d.ts +7 -0
- package/dist/search/engines/arxiv.d.ts.map +1 -0
- package/dist/search/engines/arxiv.js +70 -0
- package/dist/search/engines/arxiv.js.map +1 -0
- package/dist/search/engines/bing-news.d.ts +7 -0
- package/dist/search/engines/bing-news.d.ts.map +1 -0
- package/dist/search/engines/bing-news.js +97 -0
- package/dist/search/engines/bing-news.js.map +1 -0
- package/dist/search/engines/bing.d.ts +1 -0
- package/dist/search/engines/bing.d.ts.map +1 -1
- package/dist/search/engines/bing.js +100 -44
- package/dist/search/engines/bing.js.map +1 -1
- package/dist/search/engines/devdocs.d.ts +6 -0
- package/dist/search/engines/devdocs.d.ts.map +1 -0
- package/dist/search/engines/devdocs.js +56 -0
- package/dist/search/engines/devdocs.js.map +1 -0
- package/dist/search/engines/duckduckgo.d.ts.map +1 -1
- package/dist/search/engines/duckduckgo.js +56 -44
- package/dist/search/engines/duckduckgo.js.map +1 -1
- package/dist/search/engines/github-code.d.ts +7 -0
- package/dist/search/engines/github-code.d.ts.map +1 -0
- package/dist/search/engines/github-code.js +55 -0
- package/dist/search/engines/github-code.js.map +1 -0
- package/dist/search/engines/hn-algolia.d.ts +7 -0
- package/dist/search/engines/hn-algolia.d.ts.map +1 -0
- package/dist/search/engines/hn-algolia.js +76 -0
- package/dist/search/engines/hn-algolia.js.map +1 -0
- package/dist/search/engines/lobsters.d.ts +7 -0
- package/dist/search/engines/lobsters.d.ts.map +1 -0
- package/dist/search/engines/lobsters.js +83 -0
- package/dist/search/engines/lobsters.js.map +1 -0
- package/dist/search/engines/mdn.d.ts +7 -0
- package/dist/search/engines/mdn.d.ts.map +1 -0
- package/dist/search/engines/mdn.js +48 -0
- package/dist/search/engines/mdn.js.map +1 -0
- package/dist/search/engines/semantic-scholar.d.ts +7 -0
- package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
- package/dist/search/engines/semantic-scholar.js +69 -0
- package/dist/search/engines/semantic-scholar.js.map +1 -0
- package/dist/search/engines/stackoverflow.d.ts +7 -0
- package/dist/search/engines/stackoverflow.d.ts.map +1 -0
- package/dist/search/engines/stackoverflow.js +73 -0
- package/dist/search/engines/stackoverflow.js.map +1 -0
- package/dist/search/engines/startpage.d.ts.map +1 -1
- package/dist/search/engines/startpage.js +65 -46
- package/dist/search/engines/startpage.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +220 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/filters.d.ts.map +1 -1
- package/dist/search/filters.js +58 -54
- package/dist/search/filters.js.map +1 -1
- package/dist/search/find-similar/crawl-rank.d.ts +9 -0
- package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
- package/dist/search/find-similar/crawl-rank.js +272 -0
- package/dist/search/find-similar/crawl-rank.js.map +1 -0
- package/dist/search/find-similar/mode.d.ts +4 -0
- package/dist/search/find-similar/mode.d.ts.map +1 -0
- package/dist/search/find-similar/mode.js +12 -0
- package/dist/search/find-similar/mode.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +509 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/highlights.d.ts +19 -0
- package/dist/search/highlights.d.ts.map +1 -0
- package/dist/search/highlights.js +167 -0
- package/dist/search/highlights.js.map +1 -0
- package/dist/search/language-filter.d.ts +29 -0
- package/dist/search/language-filter.d.ts.map +1 -0
- package/dist/search/language-filter.js +126 -0
- package/dist/search/language-filter.js.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.js +501 -0
- package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
- package/dist/search/legacy/searxng-provider.d.ts +7 -0
- package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
- package/dist/search/legacy/searxng-provider.js +11 -0
- package/dist/search/legacy/searxng-provider.js.map +1 -0
- package/dist/search/multi-query.d.ts +25 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +228 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/query.js +32 -34
- package/dist/search/query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -1
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +44 -35
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/authority-boost.d.ts +3 -0
- package/dist/search/reranker/authority-boost.d.ts.map +1 -0
- package/dist/search/reranker/authority-boost.js +179 -0
- package/dist/search/reranker/authority-boost.js.map +1 -0
- package/dist/search/reranker/consensus-boost.d.ts +3 -0
- package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
- package/dist/search/reranker/consensus-boost.js +27 -0
- package/dist/search/reranker/consensus-boost.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +13 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +23 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts +13 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.js +94 -0
- package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +39 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +69 -79
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +39 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +6 -0
- package/dist/search/truncate.d.ts.map +1 -0
- package/dist/search/truncate.js +26 -0
- package/dist/search/truncate.js.map +1 -0
- package/dist/search/url-unwrap.d.ts +3 -0
- package/dist/search/url-unwrap.d.ts.map +1 -0
- package/dist/search/url-unwrap.js +43 -0
- package/dist/search/url-unwrap.js.map +1 -0
- package/dist/search/v1/context-rank.d.ts +13 -0
- package/dist/search/v1/context-rank.d.ts.map +1 -0
- package/dist/search/v1/context-rank.js +74 -0
- package/dist/search/v1/context-rank.js.map +1 -0
- package/dist/search/v1/engine-base.d.ts +27 -0
- package/dist/search/v1/engine-base.d.ts.map +1 -0
- package/dist/search/v1/engine-base.js +110 -0
- package/dist/search/v1/engine-base.js.map +1 -0
- package/dist/search/v1/intent-router.d.ts +22 -0
- package/dist/search/v1/intent-router.d.ts.map +1 -0
- package/dist/search/v1/intent-router.js +138 -0
- package/dist/search/v1/intent-router.js.map +1 -0
- package/dist/search/v1/orchestrator.d.ts +24 -0
- package/dist/search/v1/orchestrator.d.ts.map +1 -0
- package/dist/search/v1/orchestrator.js +163 -0
- package/dist/search/v1/orchestrator.js.map +1 -0
- package/dist/search/v1/recency-boost.d.ts +9 -0
- package/dist/search/v1/recency-boost.d.ts.map +1 -0
- package/dist/search/v1/recency-boost.js +37 -0
- package/dist/search/v1/recency-boost.js.map +1 -0
- package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
- package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
- package/dist/search/v1/recent-cache-dedup.js +85 -0
- package/dist/search/v1/recent-cache-dedup.js.map +1 -0
- package/dist/search/v1/rss/feed-config.d.ts +21 -0
- package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-config.js +90 -0
- package/dist/search/v1/rss/feed-config.js.map +1 -0
- package/dist/search/v1/rss/feed-parser.d.ts +14 -0
- package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-parser.js +104 -0
- package/dist/search/v1/rss/feed-parser.js.map +1 -0
- package/dist/search/v1/rss/feed-poller.d.ts +22 -0
- package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-poller.js +102 -0
- package/dist/search/v1/rss/feed-poller.js.map +1 -0
- package/dist/search/v1/rss/feed-store.d.ts +30 -0
- package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-store.js +134 -0
- package/dist/search/v1/rss/feed-store.js.map +1 -0
- package/dist/search/v1/rss/rss-engine.d.ts +6 -0
- package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
- package/dist/search/v1/rss/rss-engine.js +28 -0
- package/dist/search/v1/rss/rss-engine.js.map +1 -0
- package/dist/search/v1/v1-provider.d.ts +7 -0
- package/dist/search/v1/v1-provider.d.ts.map +1 -0
- package/dist/search/v1/v1-provider.js +68 -0
- package/dist/search/v1/v1-provider.js.map +1 -0
- package/dist/search/v1/verticals/code.d.ts +4 -0
- package/dist/search/v1/verticals/code.d.ts.map +1 -0
- package/dist/search/v1/verticals/code.js +20 -0
- package/dist/search/v1/verticals/code.js.map +1 -0
- package/dist/search/v1/verticals/docs.d.ts +4 -0
- package/dist/search/v1/verticals/docs.d.ts.map +1 -0
- package/dist/search/v1/verticals/docs.js +20 -0
- package/dist/search/v1/verticals/docs.js.map +1 -0
- package/dist/search/v1/verticals/general.d.ts +4 -0
- package/dist/search/v1/verticals/general.d.ts.map +1 -0
- package/dist/search/v1/verticals/general.js +22 -0
- package/dist/search/v1/verticals/general.js.map +1 -0
- package/dist/search/v1/verticals/news.d.ts +10 -0
- package/dist/search/v1/verticals/news.d.ts.map +1 -0
- package/dist/search/v1/verticals/news.js +52 -0
- package/dist/search/v1/verticals/news.js.map +1 -0
- package/dist/search/v1/verticals/papers.d.ts +4 -0
- package/dist/search/v1/verticals/papers.d.ts.map +1 -0
- package/dist/search/v1/verticals/papers.js +23 -0
- package/dist/search/v1/verticals/papers.js.map +1 -0
- package/dist/search/validator.js +31 -31
- package/dist/search/validator.js.map +1 -1
- package/dist/searxng/bootstrap.d.ts +30 -0
- package/dist/searxng/bootstrap.d.ts.map +1 -1
- package/dist/searxng/bootstrap.js +223 -85
- package/dist/searxng/bootstrap.js.map +1 -1
- package/dist/searxng/docker.d.ts.map +1 -1
- package/dist/searxng/docker.js +69 -60
- package/dist/searxng/docker.js.map +1 -1
- package/dist/searxng/process.d.ts +13 -1
- package/dist/searxng/process.d.ts.map +1 -1
- package/dist/searxng/process.js +231 -164
- package/dist/searxng/process.js.map +1 -1
- package/dist/server/backend-status.d.ts +13 -0
- package/dist/server/backend-status.d.ts.map +1 -0
- package/dist/server/backend-status.js +40 -0
- package/dist/server/backend-status.js.map +1 -0
- package/dist/server/tool-schemas.d.ts +549 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +464 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server/warmup-on-start.d.ts +9 -0
- package/dist/server/warmup-on-start.d.ts.map +1 -0
- package/dist/server/warmup-on-start.js +55 -0
- package/dist/server/warmup-on-start.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +454 -297
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +128 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +177 -44
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +171 -88
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts +2 -2
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +175 -59
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts +2 -2
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +174 -68
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +127 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +107 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +10 -2
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -158
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +350 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +6 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +34 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +78 -8
- package/dist/extraction/trafilatura.d.ts +0 -6
- package/dist/extraction/trafilatura.d.ts.map +0 -1
- package/dist/extraction/trafilatura.js +0 -105
- package/dist/extraction/trafilatura.js.map +0 -1
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -63
- package/dist/search/flashrank.js.map +0 -1
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
const STOPWORDS = /* @__PURE__ */ new Set([
|
|
2
|
+
"the",
|
|
3
|
+
"a",
|
|
4
|
+
"an",
|
|
5
|
+
"what",
|
|
6
|
+
"is",
|
|
7
|
+
"are",
|
|
8
|
+
"was",
|
|
9
|
+
"were",
|
|
10
|
+
"how",
|
|
11
|
+
"why",
|
|
12
|
+
"when",
|
|
13
|
+
"where",
|
|
14
|
+
"who",
|
|
15
|
+
"do",
|
|
16
|
+
"does",
|
|
17
|
+
"did",
|
|
18
|
+
"for",
|
|
19
|
+
"of",
|
|
20
|
+
"to",
|
|
21
|
+
"in",
|
|
22
|
+
"on",
|
|
23
|
+
"with",
|
|
24
|
+
"and",
|
|
25
|
+
"or",
|
|
26
|
+
"but",
|
|
27
|
+
"as",
|
|
28
|
+
"at",
|
|
29
|
+
"by",
|
|
30
|
+
"from",
|
|
31
|
+
"into",
|
|
32
|
+
"about",
|
|
33
|
+
"than",
|
|
34
|
+
"this",
|
|
35
|
+
"that",
|
|
36
|
+
"these",
|
|
37
|
+
"those",
|
|
38
|
+
"it",
|
|
39
|
+
"its",
|
|
40
|
+
"be",
|
|
41
|
+
"been",
|
|
42
|
+
"has",
|
|
43
|
+
"have",
|
|
44
|
+
"had",
|
|
45
|
+
"can",
|
|
46
|
+
"could",
|
|
47
|
+
"should",
|
|
48
|
+
"would",
|
|
49
|
+
"may",
|
|
50
|
+
"might",
|
|
51
|
+
"must",
|
|
52
|
+
"will",
|
|
53
|
+
"shall",
|
|
54
|
+
"i",
|
|
55
|
+
"you",
|
|
56
|
+
"we",
|
|
57
|
+
"they",
|
|
58
|
+
"he",
|
|
59
|
+
"she",
|
|
60
|
+
"them",
|
|
61
|
+
"my",
|
|
62
|
+
"your",
|
|
63
|
+
"our",
|
|
64
|
+
"their",
|
|
65
|
+
"latest",
|
|
66
|
+
"current",
|
|
67
|
+
"newest",
|
|
68
|
+
"recent",
|
|
69
|
+
"best",
|
|
70
|
+
"top",
|
|
71
|
+
"most"
|
|
72
|
+
]);
|
|
73
|
+
const AUTHORITATIVE_TLD = /\.(io|org|dev|edu|gov)$/;
|
|
74
|
+
const KNOWN_DOCS_HOSTS = /* @__PURE__ */ new Set([
|
|
75
|
+
"docs.python.org",
|
|
76
|
+
"developer.mozilla.org",
|
|
77
|
+
"kubernetes.io",
|
|
78
|
+
"cloud.google.com",
|
|
79
|
+
"aws.amazon.com",
|
|
80
|
+
"docs.aws.amazon.com",
|
|
81
|
+
"learn.microsoft.com",
|
|
82
|
+
"docs.microsoft.com",
|
|
83
|
+
"developer.apple.com",
|
|
84
|
+
"docs.docker.com",
|
|
85
|
+
"docs.npmjs.com",
|
|
86
|
+
"docs.github.com",
|
|
87
|
+
"docs.anthropic.com"
|
|
88
|
+
]);
|
|
89
|
+
const KNOWN_SUBJECT_DOMAIN = {
|
|
90
|
+
redis: ["redis.io", "redis.com"],
|
|
91
|
+
postgres: ["postgresql.org"],
|
|
92
|
+
postgresql: ["postgresql.org"],
|
|
93
|
+
mysql: ["mysql.com", "dev.mysql.com"],
|
|
94
|
+
python: ["python.org", "docs.python.org"],
|
|
95
|
+
react: ["react.dev", "reactjs.org"],
|
|
96
|
+
nextjs: ["nextjs.org"],
|
|
97
|
+
vue: ["vuejs.org"],
|
|
98
|
+
angular: ["angular.io", "angular.dev"],
|
|
99
|
+
node: ["nodejs.org"],
|
|
100
|
+
nodejs: ["nodejs.org"],
|
|
101
|
+
rust: ["rust-lang.org", "doc.rust-lang.org"],
|
|
102
|
+
go: ["go.dev", "golang.org"],
|
|
103
|
+
golang: ["go.dev", "golang.org"],
|
|
104
|
+
typescript: ["typescriptlang.org"],
|
|
105
|
+
javascript: ["developer.mozilla.org"],
|
|
106
|
+
anthropic: ["anthropic.com", "docs.anthropic.com"],
|
|
107
|
+
openai: ["openai.com", "platform.openai.com"],
|
|
108
|
+
google: ["google.com", "cloud.google.com"],
|
|
109
|
+
microsoft: ["microsoft.com", "learn.microsoft.com"],
|
|
110
|
+
apple: ["apple.com", "developer.apple.com"],
|
|
111
|
+
github: ["github.com", "docs.github.com"],
|
|
112
|
+
gitlab: ["gitlab.com"],
|
|
113
|
+
docker: ["docker.com", "docs.docker.com"],
|
|
114
|
+
kubernetes: ["kubernetes.io"],
|
|
115
|
+
k8s: ["kubernetes.io"],
|
|
116
|
+
aws: ["aws.amazon.com", "docs.aws.amazon.com"],
|
|
117
|
+
azure: ["azure.microsoft.com", "learn.microsoft.com"],
|
|
118
|
+
gcp: ["cloud.google.com"],
|
|
119
|
+
npm: ["npmjs.com", "docs.npmjs.com"],
|
|
120
|
+
pnpm: ["pnpm.io"],
|
|
121
|
+
yarn: ["yarnpkg.com"],
|
|
122
|
+
mcp: ["modelcontextprotocol.io", "spec.modelcontextprotocol.io", "docs.anthropic.com"]
|
|
123
|
+
};
|
|
124
|
+
function extractSubjects(query) {
|
|
125
|
+
const tokens = query.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((t) => t.length >= 2 && t.length <= 16 && !STOPWORDS.has(t));
|
|
126
|
+
return [...new Set(tokens)];
|
|
127
|
+
}
|
|
128
|
+
function hostOf(url) {
|
|
129
|
+
try {
|
|
130
|
+
return new URL(url).hostname.toLowerCase();
|
|
131
|
+
} catch {
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function applyAuthorityBoost(query, results) {
|
|
136
|
+
if (results.length === 0) return results;
|
|
137
|
+
const subjects = extractSubjects(query);
|
|
138
|
+
const knownDomains = /* @__PURE__ */ new Set();
|
|
139
|
+
for (const s of subjects) {
|
|
140
|
+
const mapped = KNOWN_SUBJECT_DOMAIN[s];
|
|
141
|
+
if (mapped) for (const d of mapped) knownDomains.add(d);
|
|
142
|
+
}
|
|
143
|
+
return results.map((r) => {
|
|
144
|
+
const host = hostOf(r.url);
|
|
145
|
+
if (!host) return r;
|
|
146
|
+
let boost = 0;
|
|
147
|
+
if (knownDomains.has(host)) boost += 0.2;
|
|
148
|
+
else for (const dom of knownDomains) {
|
|
149
|
+
if (host.endsWith(`.${dom}`)) {
|
|
150
|
+
boost += 0.18;
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (boost === 0) {
|
|
155
|
+
for (const subj of subjects) {
|
|
156
|
+
if (host === `${subj}.io` || host === `${subj}.com` || host === `${subj}.org` || host === `${subj}.dev`) {
|
|
157
|
+
boost += 0.15;
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
if (host.startsWith(`${subj}.`) || host.includes(`.${subj}.`)) {
|
|
161
|
+
boost += 0.1;
|
|
162
|
+
break;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (KNOWN_DOCS_HOSTS.has(host)) boost = Math.max(boost, 0.18);
|
|
167
|
+
else if (host.startsWith("docs.")) boost += 0.08;
|
|
168
|
+
if (boost === 0 && AUTHORITATIVE_TLD.test(host)) boost += 0.04;
|
|
169
|
+
if (boost === 0) return r;
|
|
170
|
+
return {
|
|
171
|
+
...r,
|
|
172
|
+
relevance_score: Math.min(1, r.relevance_score + boost)
|
|
173
|
+
};
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
export {
|
|
177
|
+
applyAuthorityBoost
|
|
178
|
+
};
|
|
179
|
+
//# sourceMappingURL=authority-boost.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/reranker/authority-boost.ts"],"sourcesContent":["import type { MergedSearchResult } from '../dedup.js';\n\nconst STOPWORDS = new Set([\n 'the', 'a', 'an', 'what', 'is', 'are', 'was', 'were', 'how', 'why', 'when', 'where', 'who',\n 'do', 'does', 'did', 'for', 'of', 'to', 'in', 'on', 'with', 'and', 'or', 'but', 'as', 'at',\n 'by', 'from', 'into', 'about', 'than', 'this', 'that', 'these', 'those', 'it', 'its', 'be',\n 'been', 'has', 'have', 'had', 'can', 'could', 'should', 'would', 'may', 'might', 'must',\n 'will', 'shall', 'i', 'you', 'we', 'they', 'he', 'she', 'them', 'my', 'your', 'our', 'their',\n 'latest', 'current', 'newest', 'recent', 'best', 'top', 'most',\n]);\n\nconst AUTHORITATIVE_TLD = /\\.(io|org|dev|edu|gov)$/;\nconst KNOWN_DOCS_HOSTS = new Set([\n 'docs.python.org', 'developer.mozilla.org', 'kubernetes.io', 'cloud.google.com',\n 'aws.amazon.com', 'docs.aws.amazon.com', 'learn.microsoft.com', 'docs.microsoft.com',\n 'developer.apple.com', 'docs.docker.com', 'docs.npmjs.com', 'docs.github.com',\n 'docs.anthropic.com',\n]);\n\nconst KNOWN_SUBJECT_DOMAIN: Record<string, string[]> = {\n redis: ['redis.io', 'redis.com'],\n postgres: ['postgresql.org'],\n postgresql: ['postgresql.org'],\n mysql: ['mysql.com', 'dev.mysql.com'],\n python: ['python.org', 'docs.python.org'],\n react: ['react.dev', 'reactjs.org'],\n nextjs: ['nextjs.org'],\n vue: ['vuejs.org'],\n angular: ['angular.io', 'angular.dev'],\n node: ['nodejs.org'],\n nodejs: ['nodejs.org'],\n rust: ['rust-lang.org', 'doc.rust-lang.org'],\n go: ['go.dev', 'golang.org'],\n golang: ['go.dev', 'golang.org'],\n typescript: ['typescriptlang.org'],\n javascript: ['developer.mozilla.org'],\n anthropic: ['anthropic.com', 'docs.anthropic.com'],\n openai: ['openai.com', 'platform.openai.com'],\n google: ['google.com', 'cloud.google.com'],\n microsoft: ['microsoft.com', 'learn.microsoft.com'],\n apple: ['apple.com', 'developer.apple.com'],\n github: ['github.com', 'docs.github.com'],\n gitlab: ['gitlab.com'],\n docker: ['docker.com', 'docs.docker.com'],\n kubernetes: ['kubernetes.io'],\n k8s: ['kubernetes.io'],\n aws: ['aws.amazon.com', 'docs.aws.amazon.com'],\n azure: ['azure.microsoft.com', 'learn.microsoft.com'],\n gcp: ['cloud.google.com'],\n npm: ['npmjs.com', 'docs.npmjs.com'],\n pnpm: ['pnpm.io'],\n yarn: ['yarnpkg.com'],\n mcp: ['modelcontextprotocol.io', 'spec.modelcontextprotocol.io', 'docs.anthropic.com'],\n};\n\nfunction extractSubjects(query: string): string[] {\n const tokens = query\n .toLowerCase()\n .replace(/[^a-z0-9\\s]/g, ' ')\n .split(/\\s+/)\n .filter((t) => t.length >= 2 && t.length <= 16 && !STOPWORDS.has(t));\n return [...new Set(tokens)];\n}\n\nfunction hostOf(url: string): string | null {\n try {\n return new URL(url).hostname.toLowerCase();\n } catch {\n return null;\n }\n}\n\nexport function applyAuthorityBoost(\n query: string,\n results: MergedSearchResult[],\n): MergedSearchResult[] {\n if (results.length === 0) return results;\n const subjects = extractSubjects(query);\n const knownDomains = new Set<string>();\n for (const s of subjects) {\n const mapped = KNOWN_SUBJECT_DOMAIN[s];\n if (mapped) for (const d of mapped) knownDomains.add(d);\n }\n\n return results.map((r) => {\n const host = hostOf(r.url);\n if (!host) return r;\n\n let boost = 0;\n\n if (knownDomains.has(host)) boost += 0.20;\n else for (const dom of knownDomains) {\n if (host.endsWith(`.${dom}`)) { boost += 0.18; break; }\n }\n\n if (boost === 0) {\n for (const subj of subjects) {\n if (host === `${subj}.io` || host === `${subj}.com` || host === `${subj}.org` || host === `${subj}.dev`) {\n boost += 0.15;\n break;\n }\n if (host.startsWith(`${subj}.`) || host.includes(`.${subj}.`)) {\n boost += 0.10;\n break;\n }\n }\n }\n\n if (KNOWN_DOCS_HOSTS.has(host)) boost = Math.max(boost, 0.18);\n else if (host.startsWith('docs.')) boost += 0.08;\n\n if (boost === 0 && AUTHORITATIVE_TLD.test(host)) boost += 0.04;\n\n if (boost === 0) return r;\n\n return {\n ...r,\n relevance_score: Math.min(1, r.relevance_score + boost),\n };\n });\n}\n"],"mappings":"AAEA,MAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EAAO;AAAA,EAAK;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAS;AAAA,EACrF;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAO;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAM;AAAA,EACtF;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAS;AAAA,EAAM;AAAA,EAAO;AAAA,EACtF;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAO;AAAA,EAAS;AAAA,EAAU;AAAA,EAAS;AAAA,EAAO;AAAA,EAAS;AAAA,EACjF;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAK;AAAA,EAAO;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EACrF;AAAA,EAAU;AAAA,EAAW;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAO;AAC1D,CAAC;AAED,MAAM,oBAAoB;AAC1B,MAAM,mBAAmB,oBAAI,IAAI;AAAA,EAC/B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAAiB;AAAA,EAC7D;AAAA,EAAkB;AAAA,EAAuB;AAAA,EAAuB;AAAA,EAChE;AAAA,EAAuB;AAAA,EAAmB;AAAA,EAAkB;AAAA,EAC5D;AACF,CAAC;AAED,MAAM,uBAAiD;AAAA,EACrD,OAAO,CAAC,YAAY,WAAW;AAAA,EAC/B,UAAU,CAAC,gBAAgB;AAAA,EAC3B,YAAY,CAAC,gBAAgB;AAAA,EAC7B,OAAO,CAAC,aAAa,eAAe;AAAA,EACpC,QAAQ,CAAC,cAAc,iBAAiB;AAAA,EACxC,OAAO,CAAC,aAAa,aAAa;AAAA,EAClC,QAAQ,CAAC,YAAY;AAAA,EACrB,KAAK,CAAC,WAAW;AAAA,EACjB,SAAS,CAAC,cAAc,aAAa;AAAA,EACrC,MAAM,CAAC,YAAY;AAAA,EACnB,QAAQ,CAAC,YAAY;AAAA,EACrB,MAAM,CAAC,iBAAiB,mBAAmB;AAAA,EAC3C,IAAI,CAAC,UAAU,YAAY;AAAA,EAC3B,QAAQ,CAAC,UAAU,YAAY;AAAA,EAC/B,YAAY,CAAC,oBAAoB;AAAA,EACjC,YAAY,CAAC,uBAAuB;AAAA,EACpC,WAAW,CAAC,iBAAiB,oBAAoB;AAAA,EACjD,QAAQ,CAAC,cAAc,qBAAqB;AAAA,EAC5C,QAAQ,CAAC,cAAc,kBAAkB;AAAA,EACzC,WAAW,CAAC,iBAAiB,qBAAqB;AAAA,EAClD,OAAO,CAAC,aAAa,qBAAqB;AAAA,EAC1C,QAAQ,CAAC,cAAc,iBAAiB;AAAA,EACxC,QAAQ,CAAC,YAAY;AAAA,EACrB,QAAQ,CAAC,cAAc,iBAAiB;AAAA,EACxC,YAAY,CAAC,eAAe;AAAA,EAC5B,KAAK,CAAC,eAAe;AAAA,EACrB,KAAK,CAAC,kBAAkB,qBAAqB;AAAA,EAC7C,OAAO,CAAC,uBAAuB,qBAAqB;AAAA,EACpD,KAAK,CAAC,kBAAkB;AAAA,EACxB,KAAK,CAAC,aAAa,gBAAgB;AAAA,EACnC,MAAM,CAAC,SAAS;AAAA,EAChB,MAAM,CAAC,aAAa;AAAA,EACpB,KAAK,CAAC,2BAA2B,gCAAgC,oBAAoB;AACvF;AAEA,SAAS,gBAAgB,OAAyB;AAChD,QAAM,SAAS,MACZ,YAAY,EACZ,QAAQ,gBAAgB,GAAG,EAC3B,MAAM,KAAK,EACX,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,UAAU,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;AACrE,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC;AAC5B;AAEA,SAAS,OAAO,KAA4B;AAC1C,MAAI;AACF,WAAO,IAAI,IAAI,GAAG,EAAE,SAAS,YAAY;AAAA,EAC3C,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,oBACd,OACA,SACsB;AACtB,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAM,WAAW,gBAAgB,KAAK;AACtC,QAAM,eAAe,oBAAI,IAAY;AACrC,aAAW,KAAK,UAAU;AACxB,UAAM,SAAS,qBAAqB,CAAC;AACrC,QAAI,OAAQ,YAAW,KAAK,OAAQ,cAAa,IAAI,CAAC;AAAA,EACxD;AAEA,SAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,UAAM,OAAO,OAAO,EAAE,GAAG;AACzB,QAAI,CAAC,KAAM,QAAO;AAElB,QAAI,QAAQ;AAEZ,QAAI,aAAa,IAAI,IAAI,EAAG,UAAS;AAAA,QAChC,YAAW,OAAO,cAAc;AACnC,UAAI,KAAK,SAAS,IAAI,GAAG,EAAE,GAAG;AAAE,iBAAS;AAAM;AAAA,MAAO;AAAA,IACxD;AAEA,QAAI,UAAU,GAAG;AACf,iBAAW,QAAQ,UAAU;AAC3B,YAAI,SAAS,GAAG,IAAI,SAAS,SAAS,GAAG,IAAI,UAAU,SAAS,GAAG,IAAI,UAAU,SAAS,GAAG,IAAI,QAAQ;AACvG,mBAAS;AACT;AAAA,QACF;AACA,YAAI,KAAK,WAAW,GAAG,IAAI,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,GAAG,GAAG;AAC7D,mBAAS;AACT;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,iBAAiB,IAAI,IAAI,EAAG,SAAQ,KAAK,IAAI,OAAO,IAAI;AAAA,aACnD,KAAK,WAAW,OAAO,EAAG,UAAS;AAE5C,QAAI,UAAU,KAAK,kBAAkB,KAAK,IAAI,EAAG,UAAS;AAE1D,QAAI,UAAU,EAAG,QAAO;AAExB,WAAO;AAAA,MACL,GAAG;AAAA,MACH,iBAAiB,KAAK,IAAI,GAAG,EAAE,kBAAkB,KAAK;AAAA,IACxD;AAAA,EACF,CAAC;AACH;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consensus-boost.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/consensus-boost.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAoBtD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,kBAAkB,EAAE,CAavF"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const CONSENSUS_BOOST_BY_ENGINE_COUNT = {
|
|
2
|
+
1: 0,
|
|
3
|
+
2: 0.05,
|
|
4
|
+
3: 0.1
|
|
5
|
+
};
|
|
6
|
+
const CAP_AT_OR_ABOVE_4 = 0.12;
|
|
7
|
+
function applyConsensusBoost(results) {
|
|
8
|
+
if (results.length === 0) return results;
|
|
9
|
+
return results.map((r) => {
|
|
10
|
+
const n = uniqueEngines(r.engines);
|
|
11
|
+
if (n <= 1) return r;
|
|
12
|
+
const boost = n >= 4 ? CAP_AT_OR_ABOVE_4 : CONSENSUS_BOOST_BY_ENGINE_COUNT[n] ?? 0;
|
|
13
|
+
if (boost === 0) return r;
|
|
14
|
+
return {
|
|
15
|
+
...r,
|
|
16
|
+
relevance_score: Math.min(1, r.relevance_score + boost)
|
|
17
|
+
};
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
function uniqueEngines(engines) {
|
|
21
|
+
if (!engines || engines.length === 0) return 0;
|
|
22
|
+
return new Set(engines.map((e) => e.toLowerCase())).size;
|
|
23
|
+
}
|
|
24
|
+
export {
|
|
25
|
+
applyConsensusBoost
|
|
26
|
+
};
|
|
27
|
+
//# sourceMappingURL=consensus-boost.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/reranker/consensus-boost.ts"],"sourcesContent":["import type { MergedSearchResult } from '../dedup.js';\n\n// Engine-consensus boost (Q7). Replaces per-subject authority enumeration with a\n// data-driven signal: URLs returned by multiple independent search engines are\n// likely authoritative regardless of query topic. Generalizes to all subjects;\n// no hand-curated map.\n//\n// Provenance: Reciprocal Rank Fusion's consensus property — documents agreed on\n// by multiple rankers tend to be relevant (Cormack et al., SIGIR 2009;\n// OpenSearch 2.19 RRF documentation; Azure AI Search RRF). We surface engine\n// agreement as a first-class re-rank signal in addition to the existing RRF\n// merge in dedup.ts.\n\nconst CONSENSUS_BOOST_BY_ENGINE_COUNT: Record<number, number> = {\n 1: 0,\n 2: 0.05,\n 3: 0.10,\n};\nconst CAP_AT_OR_ABOVE_4 = 0.12;\n\nexport function applyConsensusBoost(results: MergedSearchResult[]): MergedSearchResult[] {\n if (results.length === 0) return results;\n\n return results.map((r) => {\n const n = uniqueEngines(r.engines);\n if (n <= 1) return r;\n const boost = n >= 4 ? CAP_AT_OR_ABOVE_4 : CONSENSUS_BOOST_BY_ENGINE_COUNT[n] ?? 0;\n if (boost === 0) return r;\n return {\n ...r,\n relevance_score: Math.min(1, r.relevance_score + boost),\n };\n });\n}\n\nfunction uniqueEngines(engines: string[] | undefined): number {\n if (!engines || engines.length === 0) return 0;\n return new Set(engines.map((e) => e.toLowerCase())).size;\n}\n"],"mappings":"AAaA,MAAM,kCAA0D;AAAA,EAC9D,GAAG;AAAA,EACH,GAAG;AAAA,EACH,GAAG;AACL;AACA,MAAM,oBAAoB;AAEnB,SAAS,oBAAoB,SAAqD;AACvF,MAAI,QAAQ,WAAW,EAAG,QAAO;AAEjC,SAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,UAAM,IAAI,cAAc,EAAE,OAAO;AACjC,QAAI,KAAK,EAAG,QAAO;AACnB,UAAM,QAAQ,KAAK,IAAI,oBAAoB,gCAAgC,CAAC,KAAK;AACjF,QAAI,UAAU,EAAG,QAAO;AACxB,WAAO;AAAA,MACL,GAAG;AAAA,MACH,iBAAiB,KAAK,IAAI,GAAG,EAAE,kBAAkB,KAAK;AAAA,IACxD;AAAA,EACF,CAAC;AACH;AAEA,SAAS,cAAc,SAAuC;AAC5D,MAAI,CAAC,WAAW,QAAQ,WAAW,EAAG,QAAO;AAC7C,SAAO,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AACtD;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency-boost.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/recency-boost.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGtD,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAAE,EAC7B,GAAG,GAAE,IAAiB,GACrB,kBAAkB,EAAE,CAOtB"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { hasRecencyIntent, recencyFactor } from "./recency.js";
|
|
2
|
+
function applyRecencyBoost(query, results, now = /* @__PURE__ */ new Date()) {
|
|
3
|
+
if (!hasRecencyIntent(query, now)) return results;
|
|
4
|
+
return results.map((r) => {
|
|
5
|
+
const factor = recencyFactor(r.published_date, now);
|
|
6
|
+
if (factor === 1) return r;
|
|
7
|
+
return { ...r, relevance_score: r.relevance_score * factor };
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
export {
|
|
11
|
+
applyRecencyBoost
|
|
12
|
+
};
|
|
13
|
+
//# sourceMappingURL=recency-boost.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/reranker/recency-boost.ts"],"sourcesContent":["import type { MergedSearchResult } from '../dedup.js';\nimport { hasRecencyIntent, recencyFactor } from './recency.js';\n\nexport function applyRecencyBoost(\n query: string,\n results: MergedSearchResult[],\n now: Date = new Date(),\n): MergedSearchResult[] {\n if (!hasRecencyIntent(query, now)) return results;\n return results.map((r) => {\n const factor = recencyFactor(r.published_date, now);\n if (factor === 1.0) return r;\n return { ...r, relevance_score: r.relevance_score * factor };\n });\n}\n"],"mappings":"AACA,SAAS,kBAAkB,qBAAqB;AAEzC,SAAS,kBACd,OACA,SACA,MAAY,oBAAI,KAAK,GACC;AACtB,MAAI,CAAC,iBAAiB,OAAO,GAAG,EAAG,QAAO;AAC1C,SAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,UAAM,SAAS,cAAc,EAAE,gBAAgB,GAAG;AAClD,QAAI,WAAW,EAAK,QAAO;AAC3B,WAAO,EAAE,GAAG,GAAG,iBAAiB,EAAE,kBAAkB,OAAO;AAAA,EAC7D,CAAC;AACH;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/recency.ts"],"names":[],"mappings":"AAEA,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,IAAiB,GAAG,OAAO,CAM/E;AAED,wBAAgB,aAAa,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,IAAiB,GAAG,MAAM,CAS/F"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const RECENCY_TOKENS = /\b(recent|latest|new|just released|today|this week)\b/i;
|
|
2
|
+
function hasRecencyIntent(query, now = /* @__PURE__ */ new Date()) {
|
|
3
|
+
if (RECENCY_TOKENS.test(query)) return true;
|
|
4
|
+
const yearMatches = query.match(/\b(20\d{2})\b/g);
|
|
5
|
+
if (!yearMatches) return false;
|
|
6
|
+
const currentYear = now.getUTCFullYear();
|
|
7
|
+
return yearMatches.some((y) => parseInt(y, 10) >= currentYear);
|
|
8
|
+
}
|
|
9
|
+
function recencyFactor(publishedDate, now = /* @__PURE__ */ new Date()) {
|
|
10
|
+
if (!publishedDate) return 1;
|
|
11
|
+
const ts = new Date(publishedDate).getTime();
|
|
12
|
+
if (isNaN(ts)) return 1;
|
|
13
|
+
const ageDays = (now.getTime() - ts) / (1e3 * 60 * 60 * 24);
|
|
14
|
+
if (ageDays < 7) return 1.5;
|
|
15
|
+
if (ageDays < 30) return 1.3;
|
|
16
|
+
if (ageDays < 90) return 1.1;
|
|
17
|
+
return 1;
|
|
18
|
+
}
|
|
19
|
+
export {
|
|
20
|
+
hasRecencyIntent,
|
|
21
|
+
recencyFactor
|
|
22
|
+
};
|
|
23
|
+
//# sourceMappingURL=recency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/reranker/recency.ts"],"sourcesContent":["const RECENCY_TOKENS = /\\b(recent|latest|new|just released|today|this week)\\b/i;\n\nexport function hasRecencyIntent(query: string, now: Date = new Date()): boolean {\n if (RECENCY_TOKENS.test(query)) return true;\n const yearMatches = query.match(/\\b(20\\d{2})\\b/g);\n if (!yearMatches) return false;\n const currentYear = now.getUTCFullYear();\n return yearMatches.some((y) => parseInt(y, 10) >= currentYear);\n}\n\nexport function recencyFactor(publishedDate: string | undefined, now: Date = new Date()): number {\n if (!publishedDate) return 1.0;\n const ts = new Date(publishedDate).getTime();\n if (isNaN(ts)) return 1.0;\n const ageDays = (now.getTime() - ts) / (1000 * 60 * 60 * 24);\n if (ageDays < 7) return 1.5;\n if (ageDays < 30) return 1.3;\n if (ageDays < 90) return 1.1;\n return 1.0;\n}\n"],"mappings":"AAAA,MAAM,iBAAiB;AAEhB,SAAS,iBAAiB,OAAe,MAAY,oBAAI,KAAK,GAAY;AAC/E,MAAI,eAAe,KAAK,KAAK,EAAG,QAAO;AACvC,QAAM,cAAc,MAAM,MAAM,gBAAgB;AAChD,MAAI,CAAC,YAAa,QAAO;AACzB,QAAM,cAAc,IAAI,eAAe;AACvC,SAAO,YAAY,KAAK,CAAC,MAAM,SAAS,GAAG,EAAE,KAAK,WAAW;AAC/D;AAEO,SAAS,cAAc,eAAmC,MAAY,oBAAI,KAAK,GAAW;AAC/F,MAAI,CAAC,cAAe,QAAO;AAC3B,QAAM,KAAK,IAAI,KAAK,aAAa,EAAE,QAAQ;AAC3C,MAAI,MAAM,EAAE,EAAG,QAAO;AACtB,QAAM,WAAW,IAAI,QAAQ,IAAI,OAAO,MAAO,KAAK,KAAK;AACzD,MAAI,UAAU,EAAG,QAAO;AACxB,MAAI,UAAU,GAAI,QAAO;AACzB,MAAI,UAAU,GAAI,QAAO;AACzB,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RerankProvider, RerankCandidate, RerankResult } from '../../providers/rerank-provider.js';
|
|
2
|
+
export declare class TransformersRerankProvider implements RerankProvider {
|
|
3
|
+
private tokenizer;
|
|
4
|
+
private model;
|
|
5
|
+
private loadPromise;
|
|
6
|
+
readonly modelId: string;
|
|
7
|
+
constructor();
|
|
8
|
+
warmup(): Promise<void>;
|
|
9
|
+
private load;
|
|
10
|
+
rerank(query: string, candidates: RerankCandidate[], topK?: number): Promise<RerankResult[]>;
|
|
11
|
+
dispose(): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=transformers-rerank-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transformers-rerank-provider.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/transformers-rerank-provider.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,cAAc,EACd,eAAe,EACf,YAAY,EACb,MAAM,oCAAoC,CAAC;AAwC5C,qBAAa,0BAA2B,YAAW,cAAc;IAC/D,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,WAAW,CAAgE;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;;IAMnB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,OAAO,CAAC,IAAI;IA6BN,MAAM,CACV,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,eAAe,EAAE,EAC7B,IAAI,SAAoB,GACvB,OAAO,CAAC,YAAY,EAAE,CAAC;IAkCpB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAa/B"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import {
|
|
3
|
+
AutoTokenizer,
|
|
4
|
+
AutoModelForSequenceClassification,
|
|
5
|
+
env
|
|
6
|
+
} from "@huggingface/transformers";
|
|
7
|
+
import { createLogger } from "../../logger.js";
|
|
8
|
+
import { getConfig } from "../../config.js";
|
|
9
|
+
const log = createLogger("reranker");
|
|
10
|
+
function wrapLoadError(err) {
|
|
11
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
12
|
+
const looksLikeMissingModel = /tokenizer_class|tokenizer_config|preprocessor_config|fetch failed|ENOTFOUND|ECONNREFUSED|ETIMEDOUT|ENETUNREACH/i.test(
|
|
13
|
+
message
|
|
14
|
+
);
|
|
15
|
+
if (looksLikeMissingModel) {
|
|
16
|
+
return new Error(
|
|
17
|
+
`Reranker model not downloaded \u2014 run \`wigolo warmup\` (cause: ${message})`
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
return new Error(`Failed to load reranker model: ${message}`);
|
|
21
|
+
}
|
|
22
|
+
class TransformersRerankProvider {
|
|
23
|
+
tokenizer = null;
|
|
24
|
+
model = null;
|
|
25
|
+
loadPromise = null;
|
|
26
|
+
modelId;
|
|
27
|
+
constructor() {
|
|
28
|
+
this.modelId = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
29
|
+
}
|
|
30
|
+
async warmup() {
|
|
31
|
+
await this.load();
|
|
32
|
+
}
|
|
33
|
+
load() {
|
|
34
|
+
if (this.tokenizer && this.model) {
|
|
35
|
+
return Promise.resolve({ tokenizer: this.tokenizer, model: this.model });
|
|
36
|
+
}
|
|
37
|
+
if (this.loadPromise) return this.loadPromise;
|
|
38
|
+
log.info("Loading rerank model", { modelId: this.modelId });
|
|
39
|
+
const cacheDir = join(getConfig().dataDir, "transformers");
|
|
40
|
+
env.cacheDir = cacheDir;
|
|
41
|
+
this.loadPromise = Promise.all([
|
|
42
|
+
AutoTokenizer.from_pretrained(this.modelId),
|
|
43
|
+
AutoModelForSequenceClassification.from_pretrained(this.modelId)
|
|
44
|
+
]).then(([tokenizer, model]) => {
|
|
45
|
+
this.tokenizer = tokenizer;
|
|
46
|
+
this.model = model;
|
|
47
|
+
return { tokenizer, model };
|
|
48
|
+
}).catch((err) => {
|
|
49
|
+
this.loadPromise = null;
|
|
50
|
+
throw wrapLoadError(err);
|
|
51
|
+
});
|
|
52
|
+
return this.loadPromise;
|
|
53
|
+
}
|
|
54
|
+
async rerank(query, candidates, topK = candidates.length) {
|
|
55
|
+
if (candidates.length === 0) return [];
|
|
56
|
+
const { tokenizer, model } = await this.load();
|
|
57
|
+
const queries = candidates.map(() => query);
|
|
58
|
+
const docs = candidates.map((c) => c.text);
|
|
59
|
+
const inputs = tokenizer(queries, {
|
|
60
|
+
text_pair: docs,
|
|
61
|
+
padding: true,
|
|
62
|
+
truncation: true
|
|
63
|
+
});
|
|
64
|
+
const outputs = await model(inputs);
|
|
65
|
+
const logits = outputs.logits;
|
|
66
|
+
const stride = logits.dims.length >= 2 ? logits.dims[1] : 1;
|
|
67
|
+
const data = logits.data;
|
|
68
|
+
const scored = candidates.map((c, i) => ({
|
|
69
|
+
id: c.id,
|
|
70
|
+
score: Number(data[i * stride])
|
|
71
|
+
}));
|
|
72
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, topK);
|
|
73
|
+
}
|
|
74
|
+
// Release the underlying ONNX session before process exit. Without this,
|
|
75
|
+
// the runtime's worker threads race during C++ destructor teardown and
|
|
76
|
+
// surface as `mutex lock failed: Invalid argument` on macOS.
|
|
77
|
+
async dispose() {
|
|
78
|
+
const model = this.model;
|
|
79
|
+
this.model = null;
|
|
80
|
+
this.tokenizer = null;
|
|
81
|
+
this.loadPromise = null;
|
|
82
|
+
if (!model) return;
|
|
83
|
+
try {
|
|
84
|
+
const m = model;
|
|
85
|
+
if (typeof m.dispose === "function") await m.dispose();
|
|
86
|
+
} catch (err) {
|
|
87
|
+
log.debug("reranker dispose failed", { error: err instanceof Error ? err.message : String(err) });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export {
|
|
92
|
+
TransformersRerankProvider
|
|
93
|
+
};
|
|
94
|
+
//# sourceMappingURL=transformers-rerank-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/reranker/transformers-rerank-provider.ts"],"sourcesContent":["import { join } from 'node:path';\nimport {\n AutoTokenizer,\n AutoModelForSequenceClassification,\n env,\n} from '@huggingface/transformers';\nimport type {\n RerankProvider,\n RerankCandidate,\n RerankResult,\n} from '../../providers/rerank-provider.js';\nimport { createLogger } from '../../logger.js';\nimport { getConfig } from '../../config.js';\n\nconst log = createLogger('reranker');\n\n// Cross-encoder reranker via Transformers.js.\n//\n// The high-level `pipeline('text-classification', ...)` API does not pass\n// `text_pair`, so it can't drive a cross-encoder properly. We therefore\n// load the tokenizer + sequence-classification model directly: feed\n// (query, document) pairs to the tokenizer and read raw logits from the\n// model. ms-marco-MiniLM-L-6-v2 is a single-output regressor (num_labels=1)\n// where higher logit = more relevant, so the logit is used as the rerank\n// score with no further transform.\ntype Tokenizer = Awaited<ReturnType<typeof AutoTokenizer.from_pretrained>>;\ntype Model = Awaited<ReturnType<typeof AutoModelForSequenceClassification.from_pretrained>>;\n\ninterface LogitsTensor {\n data: ArrayLike<number>;\n dims: number[];\n}\n\n// Recognize the noisy huggingface fetch failure signature and replace it\n// with an actionable instruction. Transformers.js parses a config that\n// failed to download, then dereferences `tokenizer_class` on undefined.\nfunction wrapLoadError(err: unknown): Error {\n const message = err instanceof Error ? err.message : String(err);\n const looksLikeMissingModel =\n /tokenizer_class|tokenizer_config|preprocessor_config|fetch failed|ENOTFOUND|ECONNREFUSED|ETIMEDOUT|ENETUNREACH/i.test(\n message,\n );\n if (looksLikeMissingModel) {\n return new Error(\n `Reranker model not downloaded — run \\`wigolo warmup\\` (cause: ${message})`,\n );\n }\n return new Error(`Failed to load reranker model: ${message}`);\n}\n\nexport class TransformersRerankProvider implements RerankProvider {\n private tokenizer: Tokenizer | null = null;\n private model: Model | null = null;\n private loadPromise: Promise<{ tokenizer: Tokenizer; model: Model }> | null = null;\n readonly modelId: string;\n\n constructor() {\n this.modelId = 'Xenova/ms-marco-MiniLM-L-6-v2';\n }\n\n async warmup(): Promise<void> {\n await this.load();\n }\n\n private load(): Promise<{ tokenizer: Tokenizer; model: Model }> {\n if (this.tokenizer && this.model) {\n return Promise.resolve({ tokenizer: this.tokenizer, model: this.model });\n }\n if (this.loadPromise) return this.loadPromise;\n\n log.info('Loading rerank model', { modelId: this.modelId });\n const cacheDir = join(getConfig().dataDir, 'transformers');\n // Direct the library at a writable cache under the wigolo data dir so\n // models don't end up in a user home cache the daemon can't manage.\n env.cacheDir = cacheDir;\n\n this.loadPromise = Promise.all([\n AutoTokenizer.from_pretrained(this.modelId),\n AutoModelForSequenceClassification.from_pretrained(this.modelId),\n ])\n .then(([tokenizer, model]) => {\n this.tokenizer = tokenizer;\n this.model = model;\n return { tokenizer, model };\n })\n .catch((err: unknown) => {\n this.loadPromise = null;\n throw wrapLoadError(err);\n });\n\n return this.loadPromise;\n }\n\n async rerank(\n query: string,\n candidates: RerankCandidate[],\n topK = candidates.length,\n ): Promise<RerankResult[]> {\n if (candidates.length === 0) return [];\n\n const { tokenizer, model } = await this.load();\n\n // Build batch: query repeated against each document.\n const queries = candidates.map(() => query);\n const docs = candidates.map((c) => c.text);\n\n const inputs = tokenizer(queries, {\n text_pair: docs,\n padding: true,\n truncation: true,\n });\n\n const outputs = (await model(inputs)) as { logits: LogitsTensor };\n const logits = outputs.logits;\n // logits shape is [batch, 1] for single-label regression rerankers.\n // For multi-label heads (rare for rerankers) we still take the first\n // value as the relevance score.\n const stride = logits.dims.length >= 2 ? logits.dims[1] : 1;\n const data = logits.data;\n\n const scored: RerankResult[] = candidates.map((c, i) => ({\n id: c.id,\n score: Number(data[i * stride]),\n }));\n\n return scored.sort((a, b) => b.score - a.score).slice(0, topK);\n }\n\n // Release the underlying ONNX session before process exit. Without this,\n // the runtime's worker threads race during C++ destructor teardown and\n // surface as `mutex lock failed: Invalid argument` on macOS.\n async dispose(): Promise<void> {\n const model = this.model;\n this.model = null;\n this.tokenizer = null;\n this.loadPromise = null;\n if (!model) return;\n try {\n const m = model as unknown as { dispose?: () => Promise<unknown> };\n if (typeof m.dispose === 'function') await m.dispose();\n } catch (err) {\n log.debug('reranker dispose failed', { error: err instanceof Error ? err.message : String(err) });\n }\n }\n}\n"],"mappings":"AAAA,SAAS,YAAY;AACrB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAMP,SAAS,oBAAoB;AAC7B,SAAS,iBAAiB;AAE1B,MAAM,MAAM,aAAa,UAAU;AAsBnC,SAAS,cAAc,KAAqB;AAC1C,QAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,QAAM,wBACJ,kHAAkH;AAAA,IAChH;AAAA,EACF;AACF,MAAI,uBAAuB;AACzB,WAAO,IAAI;AAAA,MACT,sEAAiE,OAAO;AAAA,IAC1E;AAAA,EACF;AACA,SAAO,IAAI,MAAM,kCAAkC,OAAO,EAAE;AAC9D;AAEO,MAAM,2BAAqD;AAAA,EACxD,YAA8B;AAAA,EAC9B,QAAsB;AAAA,EACtB,cAAsE;AAAA,EACrE;AAAA,EAET,cAAc;AACZ,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,MAAM,SAAwB;AAC5B,UAAM,KAAK,KAAK;AAAA,EAClB;AAAA,EAEQ,OAAwD;AAC9D,QAAI,KAAK,aAAa,KAAK,OAAO;AAChC,aAAO,QAAQ,QAAQ,EAAE,WAAW,KAAK,WAAW,OAAO,KAAK,MAAM,CAAC;AAAA,IACzE;AACA,QAAI,KAAK,YAAa,QAAO,KAAK;AAElC,QAAI,KAAK,wBAAwB,EAAE,SAAS,KAAK,QAAQ,CAAC;AAC1D,UAAM,WAAW,KAAK,UAAU,EAAE,SAAS,cAAc;AAGzD,QAAI,WAAW;AAEf,SAAK,cAAc,QAAQ,IAAI;AAAA,MAC7B,cAAc,gBAAgB,KAAK,OAAO;AAAA,MAC1C,mCAAmC,gBAAgB,KAAK,OAAO;AAAA,IACjE,CAAC,EACE,KAAK,CAAC,CAAC,WAAW,KAAK,MAAM;AAC5B,WAAK,YAAY;AACjB,WAAK,QAAQ;AACb,aAAO,EAAE,WAAW,MAAM;AAAA,IAC5B,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,cAAc;AACnB,YAAM,cAAc,GAAG;AAAA,IACzB,CAAC;AAEH,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,OACJ,OACA,YACA,OAAO,WAAW,QACO;AACzB,QAAI,WAAW,WAAW,EAAG,QAAO,CAAC;AAErC,UAAM,EAAE,WAAW,MAAM,IAAI,MAAM,KAAK,KAAK;AAG7C,UAAM,UAAU,WAAW,IAAI,MAAM,KAAK;AAC1C,UAAM,OAAO,WAAW,IAAI,CAAC,MAAM,EAAE,IAAI;AAEzC,UAAM,SAAS,UAAU,SAAS;AAAA,MAChC,WAAW;AAAA,MACX,SAAS;AAAA,MACT,YAAY;AAAA,IACd,CAAC;AAED,UAAM,UAAW,MAAM,MAAM,MAAM;AACnC,UAAM,SAAS,QAAQ;AAIvB,UAAM,SAAS,OAAO,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,IAAI;AAC1D,UAAM,OAAO,OAAO;AAEpB,UAAM,SAAyB,WAAW,IAAI,CAAC,GAAG,OAAO;AAAA,MACvD,IAAI,EAAE;AAAA,MACN,OAAO,OAAO,KAAK,IAAI,MAAM,CAAC;AAAA,IAChC,EAAE;AAEF,WAAO,OAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UAAyB;AAC7B,UAAM,QAAQ,KAAK;AACnB,SAAK,QAAQ;AACb,SAAK,YAAY;AACjB,SAAK,cAAc;AACnB,QAAI,CAAC,MAAO;AACZ,QAAI;AACF,YAAM,IAAI;AACV,UAAI,OAAO,EAAE,YAAY,WAAY,OAAM,EAAE,QAAQ;AAAA,IACvD,SAAS,KAAK;AACZ,UAAI,MAAM,2BAA2B,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;AAAA,IAClG;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reciprocal Rank Fusion -- merges multiple ranked lists into a single
|
|
3
|
+
* score map. Each list maps URL -> rank (1-based). The fused score for
|
|
4
|
+
* a URL is the sum of 1/(k + rank) across all lists it appears in.
|
|
5
|
+
*/
|
|
6
|
+
export declare function reciprocalRankFusion(lists: Map<string, number>[], k?: number): Map<string, number>;
|
|
7
|
+
/**
|
|
8
|
+
* Convert a fused score map into a sorted array of [url, score] pairs,
|
|
9
|
+
* descending by score.
|
|
10
|
+
*/
|
|
11
|
+
export declare function sortByRRFScore(scores: Map<string, number>): Array<[string, number]>;
|
|
12
|
+
/**
|
|
13
|
+
* Build a rank map from an ordered array of URLs.
|
|
14
|
+
* First URL gets rank 1, second gets rank 2, etc.
|
|
15
|
+
*/
|
|
16
|
+
export declare function buildRankMap(urls: string[]): Map<string, number>;
|
|
17
|
+
//# sourceMappingURL=rrf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rrf.d.ts","sourceRoot":"","sources":["../../src/search/rrf.ts"],"names":[],"mappings":"AAIA;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAC5B,CAAC,GAAE,MAAW,GACb,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAqBrB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1B,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAEzB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAQhE"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createLogger } from "../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
function reciprocalRankFusion(lists, k = 60) {
|
|
4
|
+
try {
|
|
5
|
+
const scores = /* @__PURE__ */ new Map();
|
|
6
|
+
for (const list of lists) {
|
|
7
|
+
for (const [url, rank] of list) {
|
|
8
|
+
const contribution = 1 / (k + rank);
|
|
9
|
+
scores.set(url, (scores.get(url) ?? 0) + contribution);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
log.debug("RRF fusion complete", {
|
|
13
|
+
inputLists: lists.length,
|
|
14
|
+
uniqueUrls: scores.size
|
|
15
|
+
});
|
|
16
|
+
return scores;
|
|
17
|
+
} catch (err) {
|
|
18
|
+
log.error("RRF fusion failed", { error: String(err) });
|
|
19
|
+
return /* @__PURE__ */ new Map();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function sortByRRFScore(scores) {
|
|
23
|
+
return [...scores.entries()].sort((a, b) => b[1] - a[1]);
|
|
24
|
+
}
|
|
25
|
+
function buildRankMap(urls) {
|
|
26
|
+
const rankMap = /* @__PURE__ */ new Map();
|
|
27
|
+
for (let i = 0; i < urls.length; i++) {
|
|
28
|
+
if (!rankMap.has(urls[i])) {
|
|
29
|
+
rankMap.set(urls[i], i + 1);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return rankMap;
|
|
33
|
+
}
|
|
34
|
+
export {
|
|
35
|
+
buildRankMap,
|
|
36
|
+
reciprocalRankFusion,
|
|
37
|
+
sortByRRFScore
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=rrf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/search/rrf.ts"],"sourcesContent":["import { createLogger } from '../logger.js';\n\nconst log = createLogger('search');\n\n/**\n * Reciprocal Rank Fusion -- merges multiple ranked lists into a single\n * score map. Each list maps URL -> rank (1-based). The fused score for\n * a URL is the sum of 1/(k + rank) across all lists it appears in.\n */\nexport function reciprocalRankFusion(\n lists: Map<string, number>[],\n k: number = 60,\n): Map<string, number> {\n try {\n const scores = new Map<string, number>();\n\n for (const list of lists) {\n for (const [url, rank] of list) {\n const contribution = 1 / (k + rank);\n scores.set(url, (scores.get(url) ?? 0) + contribution);\n }\n }\n\n log.debug('RRF fusion complete', {\n inputLists: lists.length,\n uniqueUrls: scores.size,\n });\n\n return scores;\n } catch (err) {\n log.error('RRF fusion failed', { error: String(err) });\n return new Map();\n }\n}\n\n/**\n * Convert a fused score map into a sorted array of [url, score] pairs,\n * descending by score.\n */\nexport function sortByRRFScore(\n scores: Map<string, number>,\n): Array<[string, number]> {\n return [...scores.entries()].sort((a, b) => b[1] - a[1]);\n}\n\n/**\n * Build a rank map from an ordered array of URLs.\n * First URL gets rank 1, second gets rank 2, etc.\n */\nexport function buildRankMap(urls: string[]): Map<string, number> {\n const rankMap = new Map<string, number>();\n for (let i = 0; i < urls.length; i++) {\n if (!rankMap.has(urls[i])) {\n rankMap.set(urls[i], i + 1);\n }\n }\n return rankMap;\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAO1B,SAAS,qBACd,OACA,IAAY,IACS;AACrB,MAAI;AACF,UAAM,SAAS,oBAAI,IAAoB;AAEvC,eAAW,QAAQ,OAAO;AACxB,iBAAW,CAAC,KAAK,IAAI,KAAK,MAAM;AAC9B,cAAM,eAAe,KAAK,IAAI;AAC9B,eAAO,IAAI,MAAM,OAAO,IAAI,GAAG,KAAK,KAAK,YAAY;AAAA,MACvD;AAAA,IACF;AAEA,QAAI,MAAM,uBAAuB;AAAA,MAC/B,YAAY,MAAM;AAAA,MAClB,YAAY,OAAO;AAAA,IACrB,CAAC;AAED,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,qBAAqB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACrD,WAAO,oBAAI,IAAI;AAAA,EACjB;AACF;AAMO,SAAS,eACd,QACyB;AACzB,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACzD;AAMO,SAAS,aAAa,MAAqC;AAChE,QAAM,UAAU,oBAAI,IAAoB;AACxC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,QAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,GAAG;AACzB,cAAQ,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC;AAAA,IAC5B;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export interface SamplingMessage {
|
|
2
|
+
role: 'user' | 'assistant';
|
|
3
|
+
content: {
|
|
4
|
+
type: 'text';
|
|
5
|
+
text: string;
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
export interface SamplingResponse {
|
|
9
|
+
model: string;
|
|
10
|
+
content: {
|
|
11
|
+
type: string;
|
|
12
|
+
text: string;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export interface SamplingCapableServer {
|
|
16
|
+
getClientCapabilities: () => Record<string, unknown> | null | undefined;
|
|
17
|
+
createMessage: (params: {
|
|
18
|
+
messages: SamplingMessage[];
|
|
19
|
+
maxTokens: number;
|
|
20
|
+
}) => Promise<SamplingResponse>;
|
|
21
|
+
}
|
|
22
|
+
export declare function checkSamplingSupport(server: SamplingCapableServer): boolean;
|
|
23
|
+
export declare function requestSampling(server: SamplingCapableServer, messages: SamplingMessage[], maxTokens: number): Promise<SamplingResponse>;
|
|
24
|
+
export declare function extractTextFromSamplingResponse(response: unknown): string;
|
|
25
|
+
//# sourceMappingURL=sampling.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sampling.d.ts","sourceRoot":"","sources":["../../src/search/sampling.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACzC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACzC;AAED,MAAM,WAAW,qBAAqB;IACpC,qBAAqB,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxE,aAAa,EAAE,CAAC,MAAM,EAAE;QACtB,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC;KACnB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACjC;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAS3E;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,qBAAqB,EAC7B,QAAQ,EAAE,eAAe,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,CAAC,CAsB3B;AAED,wBAAgB,+BAA+B,CAC7C,QAAQ,EAAE,OAAO,GAChB,MAAM,CAiBR"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { createLogger } from "../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
function checkSamplingSupport(server) {
|
|
4
|
+
try {
|
|
5
|
+
const capabilities = server.getClientCapabilities();
|
|
6
|
+
if (!capabilities) return false;
|
|
7
|
+
return capabilities.sampling !== void 0;
|
|
8
|
+
} catch (err) {
|
|
9
|
+
log.debug("sampling support check failed", { error: String(err) });
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
async function requestSampling(server, messages, maxTokens) {
|
|
14
|
+
if (!checkSamplingSupport(server)) {
|
|
15
|
+
throw new Error("Client does not support MCP sampling");
|
|
16
|
+
}
|
|
17
|
+
log.debug("sending sampling request", {
|
|
18
|
+
messageCount: messages.length,
|
|
19
|
+
maxTokens
|
|
20
|
+
});
|
|
21
|
+
const response = await server.createMessage({
|
|
22
|
+
messages,
|
|
23
|
+
maxTokens
|
|
24
|
+
});
|
|
25
|
+
log.debug("sampling response received", {
|
|
26
|
+
model: response?.model,
|
|
27
|
+
contentType: response?.content?.type,
|
|
28
|
+
textLength: response?.content?.text?.length ?? 0
|
|
29
|
+
});
|
|
30
|
+
return response;
|
|
31
|
+
}
|
|
32
|
+
function extractTextFromSamplingResponse(response) {
|
|
33
|
+
try {
|
|
34
|
+
if (!response || typeof response !== "object") return "";
|
|
35
|
+
const resp = response;
|
|
36
|
+
const content = resp.content;
|
|
37
|
+
if (!content) return "";
|
|
38
|
+
if (content.type === "text" && typeof content.text === "string") {
|
|
39
|
+
return content.text.trim();
|
|
40
|
+
}
|
|
41
|
+
return "";
|
|
42
|
+
} catch (err) {
|
|
43
|
+
log.debug("failed to extract text from sampling response", { error: String(err) });
|
|
44
|
+
return "";
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export {
|
|
48
|
+
checkSamplingSupport,
|
|
49
|
+
extractTextFromSamplingResponse,
|
|
50
|
+
requestSampling
|
|
51
|
+
};
|
|
52
|
+
//# sourceMappingURL=sampling.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/search/sampling.ts"],"sourcesContent":["import { createLogger } from '../logger.js';\n\nconst log = createLogger('search');\n\nexport interface SamplingMessage {\n role: 'user' | 'assistant';\n content: { type: 'text'; text: string };\n}\n\nexport interface SamplingResponse {\n model: string;\n content: { type: string; text: string };\n}\n\nexport interface SamplingCapableServer {\n getClientCapabilities: () => Record<string, unknown> | null | undefined;\n createMessage: (params: {\n messages: SamplingMessage[];\n maxTokens: number;\n }) => Promise<SamplingResponse>;\n}\n\nexport function checkSamplingSupport(server: SamplingCapableServer): boolean {\n try {\n const capabilities = server.getClientCapabilities();\n if (!capabilities) return false;\n return capabilities.sampling !== undefined;\n } catch (err) {\n log.debug('sampling support check failed', { error: String(err) });\n return false;\n }\n}\n\nexport async function requestSampling(\n server: SamplingCapableServer,\n messages: SamplingMessage[],\n maxTokens: number,\n): Promise<SamplingResponse> {\n if (!checkSamplingSupport(server)) {\n throw new Error('Client does not support MCP sampling');\n }\n\n log.debug('sending sampling request', {\n messageCount: messages.length,\n maxTokens,\n });\n\n const response = await server.createMessage({\n messages,\n maxTokens,\n });\n\n log.debug('sampling response received', {\n model: response?.model,\n contentType: response?.content?.type,\n textLength: response?.content?.text?.length ?? 0,\n });\n\n return response;\n}\n\nexport function extractTextFromSamplingResponse(\n response: unknown,\n): string {\n try {\n if (!response || typeof response !== 'object') return '';\n\n const resp = response as Record<string, unknown>;\n const content = resp.content as Record<string, unknown> | null | undefined;\n if (!content) return '';\n\n if (content.type === 'text' && typeof content.text === 'string') {\n return content.text.trim();\n }\n\n return '';\n } catch (err) {\n log.debug('failed to extract text from sampling response', { error: String(err) });\n return '';\n }\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAoB1B,SAAS,qBAAqB,QAAwC;AAC3E,MAAI;AACF,UAAM,eAAe,OAAO,sBAAsB;AAClD,QAAI,CAAC,aAAc,QAAO;AAC1B,WAAO,aAAa,aAAa;AAAA,EACnC,SAAS,KAAK;AACZ,QAAI,MAAM,iCAAiC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACjE,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,gBACpB,QACA,UACA,WAC2B;AAC3B,MAAI,CAAC,qBAAqB,MAAM,GAAG;AACjC,UAAM,IAAI,MAAM,sCAAsC;AAAA,EACxD;AAEA,MAAI,MAAM,4BAA4B;AAAA,IACpC,cAAc,SAAS;AAAA,IACvB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,OAAO,cAAc;AAAA,IAC1C;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI,MAAM,8BAA8B;AAAA,IACtC,OAAO,UAAU;AAAA,IACjB,aAAa,UAAU,SAAS;AAAA,IAChC,YAAY,UAAU,SAAS,MAAM,UAAU;AAAA,EACjD,CAAC;AAED,SAAO;AACT;AAEO,SAAS,gCACd,UACQ;AACR,MAAI;AACF,QAAI,CAAC,YAAY,OAAO,aAAa,SAAU,QAAO;AAEtD,UAAM,OAAO;AACb,UAAM,UAAU,KAAK;AACrB,QAAI,CAAC,QAAS,QAAO;AAErB,QAAI,QAAQ,SAAS,UAAU,OAAO,QAAQ,SAAS,UAAU;AAC/D,aAAO,QAAQ,KAAK,KAAK;AAAA,IAC3B;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,iDAAiD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACjF,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAkDtF,qBAAa,aAAc,YAAW,YAAY;IAGpC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,IAAI,SAAa;gBAEY,OAAO,EAAE,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA6D3F"}
|