@staticn0va/wigolo 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +195 -73
- package/SKILL.md +382 -0
- package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
- package/assets/blocks/claude-code/wigolo-command.md +40 -0
- package/assets/blocks/cursor/wigolo.mdc +46 -0
- package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
- package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
- package/assets/skills/wigolo/SKILL.md +50 -0
- package/assets/skills/wigolo/rules/cache-first.md +30 -0
- package/assets/skills/wigolo/rules/synthesis.md +43 -0
- package/assets/skills/wigolo-agent/SKILL.md +73 -0
- package/assets/skills/wigolo-crawl/SKILL.md +60 -0
- package/assets/skills/wigolo-extract/SKILL.md +59 -0
- package/assets/skills/wigolo-fetch/SKILL.md +65 -0
- package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
- package/assets/skills/wigolo-research/SKILL.md +77 -0
- package/assets/skills/wigolo-search/SKILL.md +78 -0
- package/dist/agent/executor.d.ts +33 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +233 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +208 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +13 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +271 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/agent/relevance.d.ts +15 -0
- package/dist/agent/relevance.d.ts.map +1 -0
- package/dist/agent/relevance.js +60 -0
- package/dist/agent/relevance.js.map +1 -0
- package/dist/cache/backfill-embeddings.d.ts +23 -0
- package/dist/cache/backfill-embeddings.d.ts.map +1 -0
- package/dist/cache/backfill-embeddings.js +105 -0
- package/dist/cache/backfill-embeddings.js.map +1 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/db.d.ts +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +94 -22
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +82 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/migrations/runner.d.ts +29 -0
- package/dist/cache/migrations/runner.d.ts.map +1 -0
- package/dist/cache/migrations/runner.js +147 -0
- package/dist/cache/migrations/runner.js.map +1 -0
- package/dist/cache/sqlite-vec-store.d.ts +42 -0
- package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
- package/dist/cache/sqlite-vec-store.js +176 -0
- package/dist/cache/sqlite-vec-store.js.map +1 -0
- package/dist/cache/store.d.ts +46 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +362 -168
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/agents/antigravity.d.ts +20 -0
- package/dist/cli/agents/antigravity.d.ts.map +1 -0
- package/dist/cli/agents/antigravity.js +49 -0
- package/dist/cli/agents/antigravity.js.map +1 -0
- package/dist/cli/agents/claude-code.d.ts +25 -0
- package/dist/cli/agents/claude-code.d.ts.map +1 -0
- package/dist/cli/agents/claude-code.js +111 -0
- package/dist/cli/agents/claude-code.js.map +1 -0
- package/dist/cli/agents/cursor.d.ts +21 -0
- package/dist/cli/agents/cursor.d.ts.map +1 -0
- package/dist/cli/agents/cursor.js +58 -0
- package/dist/cli/agents/cursor.js.map +1 -0
- package/dist/cli/agents/gemini-cli.d.ts +21 -0
- package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
- package/dist/cli/agents/gemini-cli.js +55 -0
- package/dist/cli/agents/gemini-cli.js.map +1 -0
- package/dist/cli/agents/registry.d.ts +21 -0
- package/dist/cli/agents/registry.d.ts.map +1 -0
- package/dist/cli/agents/registry.js +27 -0
- package/dist/cli/agents/registry.js.map +1 -0
- package/dist/cli/agents/utils.d.ts +26 -0
- package/dist/cli/agents/utils.d.ts.map +1 -0
- package/dist/cli/agents/utils.js +136 -0
- package/dist/cli/agents/utils.js.map +1 -0
- package/dist/cli/agents/vscode.d.ts +21 -0
- package/dist/cli/agents/vscode.d.ts.map +1 -0
- package/dist/cli/agents/vscode.js +62 -0
- package/dist/cli/agents/vscode.js.map +1 -0
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +94 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/backfill.d.ts +2 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +58 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +61 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/doctor.d.ts +8 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +318 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +42 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/help.d.ts +6 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +63 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +35 -7
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +201 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +185 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/setup-mcp.d.ts +2 -0
- package/dist/cli/setup-mcp.d.ts.map +1 -0
- package/dist/cli/setup-mcp.js +114 -0
- package/dist/cli/setup-mcp.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/status.d.ts +2 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +31 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/telemetry.d.ts +10 -0
- package/dist/cli/telemetry.d.ts.map +1 -0
- package/dist/cli/telemetry.js +56 -0
- package/dist/cli/telemetry.js.map +1 -0
- package/dist/cli/tui/agents-types.d.ts +28 -0
- package/dist/cli/tui/agents-types.d.ts.map +1 -0
- package/dist/cli/tui/agents-types.js +1 -0
- package/dist/cli/tui/agents-types.js.map +1 -0
- package/dist/cli/tui/agents.d.ts +11 -0
- package/dist/cli/tui/agents.d.ts.map +1 -0
- package/dist/cli/tui/agents.js +93 -0
- package/dist/cli/tui/agents.js.map +1 -0
- package/dist/cli/tui/banner.d.ts +3 -0
- package/dist/cli/tui/banner.d.ts.map +1 -0
- package/dist/cli/tui/banner.js +30 -0
- package/dist/cli/tui/banner.js.map +1 -0
- package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
- package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/AgentSelect.js +116 -0
- package/dist/cli/tui/components/AgentSelect.js.map +1 -0
- package/dist/cli/tui/components/Banner.d.ts +6 -0
- package/dist/cli/tui/components/Banner.d.ts.map +1 -0
- package/dist/cli/tui/components/Banner.js +25 -0
- package/dist/cli/tui/components/Banner.js.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.js +19 -0
- package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
- package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
- package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
- package/dist/cli/tui/components/InstallProgress.js +67 -0
- package/dist/cli/tui/components/InstallProgress.js.map +1 -0
- package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
- package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
- package/dist/cli/tui/components/SkillInstall.js +94 -0
- package/dist/cli/tui/components/SkillInstall.js.map +1 -0
- package/dist/cli/tui/components/Summary.d.ts +22 -0
- package/dist/cli/tui/components/Summary.d.ts.map +1 -0
- package/dist/cli/tui/components/Summary.js +135 -0
- package/dist/cli/tui/components/Summary.js.map +1 -0
- package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
- package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/components/SystemCheck.js +71 -0
- package/dist/cli/tui/components/SystemCheck.js.map +1 -0
- package/dist/cli/tui/components/Verification.d.ts +8 -0
- package/dist/cli/tui/components/Verification.d.ts.map +1 -0
- package/dist/cli/tui/components/Verification.js +63 -0
- package/dist/cli/tui/components/Verification.js.map +1 -0
- package/dist/cli/tui/config-writer-cli.d.ts +12 -0
- package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-cli.js +39 -0
- package/dist/cli/tui/config-writer-cli.js.map +1 -0
- package/dist/cli/tui/config-writer-json.d.ts +16 -0
- package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-json.js +86 -0
- package/dist/cli/tui/config-writer-json.js.map +1 -0
- package/dist/cli/tui/config-writer-toml.d.ts +16 -0
- package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-toml.js +83 -0
- package/dist/cli/tui/config-writer-toml.js.map +1 -0
- package/dist/cli/tui/config-writer.d.ts +25 -0
- package/dist/cli/tui/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/config-writer.js +101 -0
- package/dist/cli/tui/config-writer.js.map +1 -0
- package/dist/cli/tui/detect-helpers.d.ts +6 -0
- package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
- package/dist/cli/tui/detect-helpers.js +45 -0
- package/dist/cli/tui/detect-helpers.js.map +1 -0
- package/dist/cli/tui/extras-prompt.d.ts +7 -0
- package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
- package/dist/cli/tui/extras-prompt.js +42 -0
- package/dist/cli/tui/extras-prompt.js.map +1 -0
- package/dist/cli/tui/flags-types.d.ts +19 -0
- package/dist/cli/tui/flags-types.d.ts.map +1 -0
- package/dist/cli/tui/flags-types.js +23 -0
- package/dist/cli/tui/flags-types.js.map +1 -0
- package/dist/cli/tui/flags.d.ts +5 -0
- package/dist/cli/tui/flags.d.ts.map +1 -0
- package/dist/cli/tui/flags.js +132 -0
- package/dist/cli/tui/flags.js.map +1 -0
- package/dist/cli/tui/format.d.ts +14 -0
- package/dist/cli/tui/format.d.ts.map +1 -0
- package/dist/cli/tui/format.js +37 -0
- package/dist/cli/tui/format.js.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
- package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
- package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
- package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useInstall.js +90 -0
- package/dist/cli/tui/hooks/useInstall.js.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
- package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
- package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
- package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useVerify.js +71 -0
- package/dist/cli/tui/hooks/useVerify.js.map +1 -0
- package/dist/cli/tui/ink-init.d.ts +2 -0
- package/dist/cli/tui/ink-init.d.ts.map +1 -0
- package/dist/cli/tui/ink-init.js +198 -0
- package/dist/cli/tui/ink-init.js.map +1 -0
- package/dist/cli/tui/reporter-auto.d.ts +7 -0
- package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
- package/dist/cli/tui/reporter-auto.js +15 -0
- package/dist/cli/tui/reporter-auto.js.map +1 -0
- package/dist/cli/tui/reporter.d.ts +26 -0
- package/dist/cli/tui/reporter.d.ts.map +1 -0
- package/dist/cli/tui/reporter.js +32 -0
- package/dist/cli/tui/reporter.js.map +1 -0
- package/dist/cli/tui/run-command.d.ts +14 -0
- package/dist/cli/tui/run-command.d.ts.map +1 -0
- package/dist/cli/tui/run-command.js +72 -0
- package/dist/cli/tui/run-command.js.map +1 -0
- package/dist/cli/tui/select-agents.d.ts +6 -0
- package/dist/cli/tui/select-agents.d.ts.map +1 -0
- package/dist/cli/tui/select-agents.js +32 -0
- package/dist/cli/tui/select-agents.js.map +1 -0
- package/dist/cli/tui/status-agents.d.ts +11 -0
- package/dist/cli/tui/status-agents.d.ts.map +1 -0
- package/dist/cli/tui/status-agents.js +53 -0
- package/dist/cli/tui/status-agents.js.map +1 -0
- package/dist/cli/tui/status-cache.d.ts +6 -0
- package/dist/cli/tui/status-cache.d.ts.map +1 -0
- package/dist/cli/tui/status-cache.js +39 -0
- package/dist/cli/tui/status-cache.js.map +1 -0
- package/dist/cli/tui/status-format.d.ts +14 -0
- package/dist/cli/tui/status-format.d.ts.map +1 -0
- package/dist/cli/tui/status-format.js +41 -0
- package/dist/cli/tui/status-format.js.map +1 -0
- package/dist/cli/tui/status-python.d.ts +6 -0
- package/dist/cli/tui/status-python.d.ts.map +1 -0
- package/dist/cli/tui/status-python.js +30 -0
- package/dist/cli/tui/status-python.js.map +1 -0
- package/dist/cli/tui/system-check.d.ts +24 -0
- package/dist/cli/tui/system-check.d.ts.map +1 -0
- package/dist/cli/tui/system-check.js +103 -0
- package/dist/cli/tui/system-check.js.map +1 -0
- package/dist/cli/tui/tui-reporter.d.ts +19 -0
- package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
- package/dist/cli/tui/tui-reporter.js +95 -0
- package/dist/cli/tui/tui-reporter.js.map +1 -0
- package/dist/cli/tui/utils/config-writer.d.ts +3 -0
- package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/utils/config-writer.js +22 -0
- package/dist/cli/tui/utils/config-writer.js.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.js +11 -0
- package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
- package/dist/cli/tui/verify-suggestions.d.ts +5 -0
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
- package/dist/cli/tui/verify-suggestions.js +20 -0
- package/dist/cli/tui/verify-suggestions.js.map +1 -0
- package/dist/cli/tui/verify.d.ts +14 -0
- package/dist/cli/tui/verify.d.ts.map +1 -0
- package/dist/cli/tui/verify.js +101 -0
- package/dist/cli/tui/verify.js.map +1 -0
- package/dist/cli/tui/version.d.ts +2 -0
- package/dist/cli/tui/version.d.ts.map +1 -0
- package/dist/cli/tui/version.js +14 -0
- package/dist/cli/tui/version.js.map +1 -0
- package/dist/cli/uninstall.d.ts +2 -0
- package/dist/cli/uninstall.d.ts.map +1 -0
- package/dist/cli/uninstall.js +57 -0
- package/dist/cli/uninstall.js.map +1 -0
- package/dist/cli/warmup.d.ts +10 -2
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +226 -93
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +28 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +106 -56
- package/dist/config.js.map +1 -1
- package/dist/crawl/crawler.d.ts +6 -0
- package/dist/crawl/crawler.d.ts.map +1 -1
- package/dist/crawl/crawler.js +210 -209
- package/dist/crawl/crawler.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +124 -81
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/crawl/etag-incremental.d.ts +43 -0
- package/dist/crawl/etag-incremental.d.ts.map +1 -0
- package/dist/crawl/etag-incremental.js +94 -0
- package/dist/crawl/etag-incremental.js.map +1 -0
- package/dist/crawl/index-to-vec.d.ts +10 -0
- package/dist/crawl/index-to-vec.d.ts.map +1 -0
- package/dist/crawl/index-to-vec.js +44 -0
- package/dist/crawl/index-to-vec.js.map +1 -0
- package/dist/crawl/mapper.js +136 -164
- package/dist/crawl/mapper.js.map +1 -1
- package/dist/crawl/rate-limiter.js +63 -66
- package/dist/crawl/rate-limiter.js.map +1 -1
- package/dist/crawl/robots.js +58 -57
- package/dist/crawl/robots.js.map +1 -1
- package/dist/crawl/sitemap-first.d.ts +12 -0
- package/dist/crawl/sitemap-first.d.ts.map +1 -0
- package/dist/crawl/sitemap-first.js +47 -0
- package/dist/crawl/sitemap-first.js.map +1 -0
- package/dist/crawl/sitemap.js +33 -32
- package/dist/crawl/sitemap.js.map +1 -1
- package/dist/crawl/url-utils.d.ts +1 -0
- package/dist/crawl/url-utils.d.ts.map +1 -1
- package/dist/crawl/url-utils.js +49 -37
- package/dist/crawl/url-utils.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +33 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +275 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +93 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +233 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/fastembed-provider.d.ts +19 -0
- package/dist/embedding/fastembed-provider.d.ts.map +1 -0
- package/dist/embedding/fastembed-provider.js +51 -0
- package/dist/embedding/fastembed-provider.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +234 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +52 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +27 -23
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/extract.d.ts.map +1 -1
- package/dist/extraction/extract.js +76 -76
- package/dist/extraction/extract.js.map +1 -1
- package/dist/extraction/jsonld.js +50 -54
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +30 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +130 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown-sanitize.d.ts +2 -0
- package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
- package/dist/extraction/markdown-sanitize.js +151 -0
- package/dist/extraction/markdown-sanitize.js.map +1 -0
- package/dist/extraction/markdown.d.ts +11 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +195 -91
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts +8 -0
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +57 -91
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +28 -29
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +135 -72
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +81 -91
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +87 -95
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +46 -54
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +71 -80
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +173 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/extraction/structured.d.ts +4 -0
- package/dist/extraction/structured.d.ts.map +1 -0
- package/dist/extraction/structured.js +163 -0
- package/dist/extraction/structured.js.map +1 -0
- package/dist/extraction/v1/classifier.d.ts +3 -0
- package/dist/extraction/v1/classifier.d.ts.map +1 -0
- package/dist/extraction/v1/classifier.js +110 -0
- package/dist/extraction/v1/classifier.js.map +1 -0
- package/dist/extraction/v1/extract-provider.d.ts +16 -0
- package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
- package/dist/extraction/v1/extract-provider.js +43 -0
- package/dist/extraction/v1/extract-provider.js.map +1 -0
- package/dist/extraction/v1/local-llm.d.ts +8 -0
- package/dist/extraction/v1/local-llm.d.ts.map +1 -0
- package/dist/extraction/v1/local-llm.js +58 -0
- package/dist/extraction/v1/local-llm.js.map +1 -0
- package/dist/extraction/v1/news.d.ts +3 -0
- package/dist/extraction/v1/news.d.ts.map +1 -0
- package/dist/extraction/v1/news.js +61 -0
- package/dist/extraction/v1/news.js.map +1 -0
- package/dist/extraction/v1/product.d.ts +3 -0
- package/dist/extraction/v1/product.d.ts.map +1 -0
- package/dist/extraction/v1/product.js +166 -0
- package/dist/extraction/v1/product.js.map +1 -0
- package/dist/extraction/v1/recipe.d.ts +3 -0
- package/dist/extraction/v1/recipe.d.ts.map +1 -0
- package/dist/extraction/v1/recipe.js +136 -0
- package/dist/extraction/v1/recipe.js.map +1 -0
- package/dist/extraction/v1/routed.d.ts +17 -0
- package/dist/extraction/v1/routed.d.ts.map +1 -0
- package/dist/extraction/v1/routed.js +68 -0
- package/dist/extraction/v1/routed.js.map +1 -0
- package/dist/extraction/v1/schemas/Article.d.ts +11 -0
- package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Article.js +23 -0
- package/dist/extraction/v1/schemas/Article.js.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.js +122 -0
- package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
- package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
- package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Paper.js +156 -0
- package/dist/extraction/v1/schemas/Paper.js.map +1 -0
- package/dist/extraction/v1/schemas/Product.d.ts +17 -0
- package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Product.js +149 -0
- package/dist/extraction/v1/schemas/Product.js.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.js +160 -0
- package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
- package/dist/extraction/v1/schemas/index.d.ts +13 -0
- package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/index.js +44 -0
- package/dist/extraction/v1/schemas/index.js.map +1 -0
- package/dist/extraction/v1/site-extractors.d.ts +5 -0
- package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
- package/dist/extraction/v1/site-extractors.js +31 -0
- package/dist/extraction/v1/site-extractors.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +88 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +56 -26
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +303 -127
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +72 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +89 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/content-check.js +39 -46
- package/dist/fetch/content-check.js.map +1 -1
- package/dist/fetch/http-client.d.ts +4 -0
- package/dist/fetch/http-client.d.ts.map +1 -1
- package/dist/fetch/http-client.js +147 -128
- package/dist/fetch/http-client.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +174 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/fetch/playwright-tier.d.ts +19 -0
- package/dist/fetch/playwright-tier.d.ts.map +1 -0
- package/dist/fetch/playwright-tier.js +76 -0
- package/dist/fetch/playwright-tier.js.map +1 -0
- package/dist/fetch/router.d.ts +49 -3
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +185 -81
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +97 -17
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +31 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +245 -0
- package/dist/instructions.js.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.js +41 -0
- package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
- package/dist/integrations/cloud/llm/cache.d.ts +5 -0
- package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/cache.js +49 -0
- package/dist/integrations/cloud/llm/cache.js.map +1 -0
- package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
- package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/gemini.js +37 -0
- package/dist/integrations/cloud/llm/gemini.js.map +1 -0
- package/dist/integrations/cloud/llm/groq.d.ts +3 -0
- package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/groq.js +74 -0
- package/dist/integrations/cloud/llm/groq.js.map +1 -0
- package/dist/integrations/cloud/llm/hash.d.ts +3 -0
- package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/hash.js +26 -0
- package/dist/integrations/cloud/llm/hash.js.map +1 -0
- package/dist/integrations/cloud/llm/openai.d.ts +3 -0
- package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/openai.js +43 -0
- package/dist/integrations/cloud/llm/openai.js.map +1 -0
- package/dist/integrations/cloud/llm/select.d.ts +5 -0
- package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/select.js +30 -0
- package/dist/integrations/cloud/llm/select.js.map +1 -0
- package/dist/integrations/cloud/llm/types.d.ts +24 -0
- package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/types.js +1 -0
- package/dist/integrations/cloud/llm/types.js.map +1 -0
- package/dist/integrations/cloud/llm/validate.d.ts +6 -0
- package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/validate.js +63 -0
- package/dist/integrations/cloud/llm/validate.js.map +1 -0
- package/dist/logger.d.ts +4 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +71 -30
- package/dist/logger.js.map +1 -1
- package/dist/pdf-parse.d.js +1 -0
- package/dist/pdf-parse.d.js.map +1 -0
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +157 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +71 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +79 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/providers/embed-provider.d.ts +11 -0
- package/dist/providers/embed-provider.d.ts.map +1 -0
- package/dist/providers/embed-provider.js +24 -0
- package/dist/providers/embed-provider.js.map +1 -0
- package/dist/providers/extract-provider.d.ts +23 -0
- package/dist/providers/extract-provider.d.ts.map +1 -0
- package/dist/providers/extract-provider.js +25 -0
- package/dist/providers/extract-provider.js.map +1 -0
- package/dist/providers/rerank-provider.d.ts +16 -0
- package/dist/providers/rerank-provider.d.ts.map +1 -0
- package/dist/providers/rerank-provider.js +28 -0
- package/dist/providers/rerank-provider.js.map +1 -0
- package/dist/providers/search-provider.d.ts +25 -0
- package/dist/providers/search-provider.d.ts.map +1 -0
- package/dist/providers/search-provider.js +44 -0
- package/dist/providers/search-provider.js.map +1 -0
- package/dist/providers/vector-store.d.ts +27 -0
- package/dist/providers/vector-store.d.ts.map +1 -0
- package/dist/providers/vector-store.js +27 -0
- package/dist/providers/vector-store.js.map +1 -0
- package/dist/python-env.d.ts +9 -0
- package/dist/python-env.d.ts.map +1 -0
- package/dist/python-env.js +13 -0
- package/dist/python-env.js.map +1 -0
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +62 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +43 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +44 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +47 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +67 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +74 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +65 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +74 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +1 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +13 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +283 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +86 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +184 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/research/branch-exploration.d.ts +14 -0
- package/dist/research/branch-exploration.d.ts.map +1 -0
- package/dist/research/branch-exploration.js +100 -0
- package/dist/research/branch-exploration.js.map +1 -0
- package/dist/research/brief.d.ts +5 -0
- package/dist/research/brief.d.ts.map +1 -0
- package/dist/research/brief.js +242 -0
- package/dist/research/brief.js.map +1 -0
- package/dist/research/citation-graph.d.ts +9 -0
- package/dist/research/citation-graph.d.ts.map +1 -0
- package/dist/research/citation-graph.js +114 -0
- package/dist/research/citation-graph.js.map +1 -0
- package/dist/research/decompose.d.ts +14 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +439 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +269 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesis-local.d.ts +16 -0
- package/dist/research/synthesis-local.d.ts.map +1 -0
- package/dist/research/synthesis-local.js +73 -0
- package/dist/research/synthesis-local.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +137 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +33 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +244 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/search/dedup.d.ts +1 -0
- package/dist/search/dedup.d.ts.map +1 -1
- package/dist/search/dedup.js +40 -32
- package/dist/search/dedup.js.map +1 -1
- package/dist/search/engines/arxiv.d.ts +7 -0
- package/dist/search/engines/arxiv.d.ts.map +1 -0
- package/dist/search/engines/arxiv.js +70 -0
- package/dist/search/engines/arxiv.js.map +1 -0
- package/dist/search/engines/bing-news.d.ts +7 -0
- package/dist/search/engines/bing-news.d.ts.map +1 -0
- package/dist/search/engines/bing-news.js +97 -0
- package/dist/search/engines/bing-news.js.map +1 -0
- package/dist/search/engines/bing.d.ts +1 -0
- package/dist/search/engines/bing.d.ts.map +1 -1
- package/dist/search/engines/bing.js +100 -44
- package/dist/search/engines/bing.js.map +1 -1
- package/dist/search/engines/devdocs.d.ts +6 -0
- package/dist/search/engines/devdocs.d.ts.map +1 -0
- package/dist/search/engines/devdocs.js +56 -0
- package/dist/search/engines/devdocs.js.map +1 -0
- package/dist/search/engines/duckduckgo.d.ts.map +1 -1
- package/dist/search/engines/duckduckgo.js +56 -44
- package/dist/search/engines/duckduckgo.js.map +1 -1
- package/dist/search/engines/github-code.d.ts +7 -0
- package/dist/search/engines/github-code.d.ts.map +1 -0
- package/dist/search/engines/github-code.js +55 -0
- package/dist/search/engines/github-code.js.map +1 -0
- package/dist/search/engines/hn-algolia.d.ts +7 -0
- package/dist/search/engines/hn-algolia.d.ts.map +1 -0
- package/dist/search/engines/hn-algolia.js +76 -0
- package/dist/search/engines/hn-algolia.js.map +1 -0
- package/dist/search/engines/lobsters.d.ts +7 -0
- package/dist/search/engines/lobsters.d.ts.map +1 -0
- package/dist/search/engines/lobsters.js +83 -0
- package/dist/search/engines/lobsters.js.map +1 -0
- package/dist/search/engines/mdn.d.ts +7 -0
- package/dist/search/engines/mdn.d.ts.map +1 -0
- package/dist/search/engines/mdn.js +48 -0
- package/dist/search/engines/mdn.js.map +1 -0
- package/dist/search/engines/semantic-scholar.d.ts +7 -0
- package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
- package/dist/search/engines/semantic-scholar.js +69 -0
- package/dist/search/engines/semantic-scholar.js.map +1 -0
- package/dist/search/engines/stackoverflow.d.ts +7 -0
- package/dist/search/engines/stackoverflow.d.ts.map +1 -0
- package/dist/search/engines/stackoverflow.js +73 -0
- package/dist/search/engines/stackoverflow.js.map +1 -0
- package/dist/search/engines/startpage.d.ts.map +1 -1
- package/dist/search/engines/startpage.js +65 -46
- package/dist/search/engines/startpage.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +220 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/filters.js +49 -55
- package/dist/search/filters.js.map +1 -1
- package/dist/search/find-similar/crawl-rank.d.ts +9 -0
- package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
- package/dist/search/find-similar/crawl-rank.js +272 -0
- package/dist/search/find-similar/crawl-rank.js.map +1 -0
- package/dist/search/find-similar/mode.d.ts +4 -0
- package/dist/search/find-similar/mode.d.ts.map +1 -0
- package/dist/search/find-similar/mode.js +12 -0
- package/dist/search/find-similar/mode.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +509 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/highlights.d.ts +19 -0
- package/dist/search/highlights.d.ts.map +1 -0
- package/dist/search/highlights.js +167 -0
- package/dist/search/highlights.js.map +1 -0
- package/dist/search/language-filter.d.ts +29 -0
- package/dist/search/language-filter.d.ts.map +1 -0
- package/dist/search/language-filter.js +126 -0
- package/dist/search/language-filter.js.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.js +501 -0
- package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
- package/dist/search/legacy/searxng-provider.d.ts +7 -0
- package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
- package/dist/search/legacy/searxng-provider.js +11 -0
- package/dist/search/legacy/searxng-provider.js.map +1 -0
- package/dist/search/multi-query.d.ts +25 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +228 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/query.js +32 -34
- package/dist/search/query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -1
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +44 -35
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/authority-boost.d.ts +3 -0
- package/dist/search/reranker/authority-boost.d.ts.map +1 -0
- package/dist/search/reranker/authority-boost.js +179 -0
- package/dist/search/reranker/authority-boost.js.map +1 -0
- package/dist/search/reranker/consensus-boost.d.ts +3 -0
- package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
- package/dist/search/reranker/consensus-boost.js +27 -0
- package/dist/search/reranker/consensus-boost.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +13 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +23 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts +12 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.js +78 -0
- package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +39 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +69 -79
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +39 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +6 -0
- package/dist/search/truncate.d.ts.map +1 -0
- package/dist/search/truncate.js +26 -0
- package/dist/search/truncate.js.map +1 -0
- package/dist/search/url-unwrap.d.ts +3 -0
- package/dist/search/url-unwrap.d.ts.map +1 -0
- package/dist/search/url-unwrap.js +43 -0
- package/dist/search/url-unwrap.js.map +1 -0
- package/dist/search/v1/context-rank.d.ts +13 -0
- package/dist/search/v1/context-rank.d.ts.map +1 -0
- package/dist/search/v1/context-rank.js +74 -0
- package/dist/search/v1/context-rank.js.map +1 -0
- package/dist/search/v1/engine-base.d.ts +27 -0
- package/dist/search/v1/engine-base.d.ts.map +1 -0
- package/dist/search/v1/engine-base.js +110 -0
- package/dist/search/v1/engine-base.js.map +1 -0
- package/dist/search/v1/intent-router.d.ts +22 -0
- package/dist/search/v1/intent-router.d.ts.map +1 -0
- package/dist/search/v1/intent-router.js +138 -0
- package/dist/search/v1/intent-router.js.map +1 -0
- package/dist/search/v1/orchestrator.d.ts +24 -0
- package/dist/search/v1/orchestrator.d.ts.map +1 -0
- package/dist/search/v1/orchestrator.js +163 -0
- package/dist/search/v1/orchestrator.js.map +1 -0
- package/dist/search/v1/recency-boost.d.ts +9 -0
- package/dist/search/v1/recency-boost.d.ts.map +1 -0
- package/dist/search/v1/recency-boost.js +37 -0
- package/dist/search/v1/recency-boost.js.map +1 -0
- package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
- package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
- package/dist/search/v1/recent-cache-dedup.js +85 -0
- package/dist/search/v1/recent-cache-dedup.js.map +1 -0
- package/dist/search/v1/rss/feed-config.d.ts +21 -0
- package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-config.js +90 -0
- package/dist/search/v1/rss/feed-config.js.map +1 -0
- package/dist/search/v1/rss/feed-parser.d.ts +14 -0
- package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-parser.js +104 -0
- package/dist/search/v1/rss/feed-parser.js.map +1 -0
- package/dist/search/v1/rss/feed-poller.d.ts +22 -0
- package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-poller.js +102 -0
- package/dist/search/v1/rss/feed-poller.js.map +1 -0
- package/dist/search/v1/rss/feed-store.d.ts +30 -0
- package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-store.js +134 -0
- package/dist/search/v1/rss/feed-store.js.map +1 -0
- package/dist/search/v1/rss/rss-engine.d.ts +6 -0
- package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
- package/dist/search/v1/rss/rss-engine.js +28 -0
- package/dist/search/v1/rss/rss-engine.js.map +1 -0
- package/dist/search/v1/v1-provider.d.ts +7 -0
- package/dist/search/v1/v1-provider.d.ts.map +1 -0
- package/dist/search/v1/v1-provider.js +68 -0
- package/dist/search/v1/v1-provider.js.map +1 -0
- package/dist/search/v1/verticals/code.d.ts +4 -0
- package/dist/search/v1/verticals/code.d.ts.map +1 -0
- package/dist/search/v1/verticals/code.js +20 -0
- package/dist/search/v1/verticals/code.js.map +1 -0
- package/dist/search/v1/verticals/docs.d.ts +4 -0
- package/dist/search/v1/verticals/docs.d.ts.map +1 -0
- package/dist/search/v1/verticals/docs.js +20 -0
- package/dist/search/v1/verticals/docs.js.map +1 -0
- package/dist/search/v1/verticals/general.d.ts +4 -0
- package/dist/search/v1/verticals/general.d.ts.map +1 -0
- package/dist/search/v1/verticals/general.js +22 -0
- package/dist/search/v1/verticals/general.js.map +1 -0
- package/dist/search/v1/verticals/news.d.ts +10 -0
- package/dist/search/v1/verticals/news.d.ts.map +1 -0
- package/dist/search/v1/verticals/news.js +52 -0
- package/dist/search/v1/verticals/news.js.map +1 -0
- package/dist/search/v1/verticals/papers.d.ts +4 -0
- package/dist/search/v1/verticals/papers.d.ts.map +1 -0
- package/dist/search/v1/verticals/papers.js +23 -0
- package/dist/search/v1/verticals/papers.js.map +1 -0
- package/dist/search/validator.js +31 -31
- package/dist/search/validator.js.map +1 -1
- package/dist/searxng/bootstrap.d.ts +30 -0
- package/dist/searxng/bootstrap.d.ts.map +1 -1
- package/dist/searxng/bootstrap.js +223 -85
- package/dist/searxng/bootstrap.js.map +1 -1
- package/dist/searxng/docker.d.ts.map +1 -1
- package/dist/searxng/docker.js +69 -60
- package/dist/searxng/docker.js.map +1 -1
- package/dist/searxng/process.d.ts +13 -1
- package/dist/searxng/process.d.ts.map +1 -1
- package/dist/searxng/process.js +231 -164
- package/dist/searxng/process.js.map +1 -1
- package/dist/server/backend-status.d.ts +13 -0
- package/dist/server/backend-status.d.ts.map +1 -0
- package/dist/server/backend-status.js +40 -0
- package/dist/server/backend-status.js.map +1 -0
- package/dist/server/tool-schemas.d.ts +549 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +464 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server/warmup-on-start.d.ts +9 -0
- package/dist/server/warmup-on-start.d.ts.map +1 -0
- package/dist/server/warmup-on-start.js +55 -0
- package/dist/server/warmup-on-start.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +454 -297
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +128 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +175 -44
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +171 -88
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts +2 -2
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +175 -59
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts +2 -2
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +161 -68
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +127 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +107 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +10 -2
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -158
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +350 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +6 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +34 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +78 -8
- package/dist/extraction/trafilatura.d.ts +0 -6
- package/dist/extraction/trafilatura.d.ts.map +0 -1
- package/dist/extraction/trafilatura.js +0 -105
- package/dist/extraction/trafilatura.js.map +0 -1
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -63
- package/dist/search/flashrank.js.map +0 -1
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
|
|
2
|
+
export declare class HnAlgoliaEngine implements SearchEngine {
|
|
3
|
+
name: string;
|
|
4
|
+
search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
|
|
5
|
+
private parseHits;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=hn-algolia.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hn-algolia.d.ts","sourceRoot":"","sources":["../../../src/search/engines/hn-algolia.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAoCzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAgB;IAEd,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiC1F,OAAO,CAAC,SAAS;CAmClB"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { createLogger } from "../../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
const SNIPPET_LIMIT = 200;
|
|
4
|
+
function toEpoch(dateStr) {
|
|
5
|
+
const d = new Date(dateStr);
|
|
6
|
+
const t = d.getTime();
|
|
7
|
+
if (isNaN(t)) return 0;
|
|
8
|
+
return Math.floor(t / 1e3);
|
|
9
|
+
}
|
|
10
|
+
function asString(v) {
|
|
11
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
12
|
+
}
|
|
13
|
+
function asNumber(v) {
|
|
14
|
+
return typeof v === "number" && isFinite(v) ? v : void 0;
|
|
15
|
+
}
|
|
16
|
+
class HnAlgoliaEngine {
|
|
17
|
+
name = "hn-algolia";
|
|
18
|
+
async search(query, options = {}) {
|
|
19
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
20
|
+
const maxResults = options.maxResults ?? 10;
|
|
21
|
+
const params = new URLSearchParams({
|
|
22
|
+
query,
|
|
23
|
+
hitsPerPage: String(maxResults),
|
|
24
|
+
tags: "story"
|
|
25
|
+
});
|
|
26
|
+
if (options.fromDate || options.toDate) {
|
|
27
|
+
const filters = [];
|
|
28
|
+
if (options.fromDate) {
|
|
29
|
+
const epoch = toEpoch(options.fromDate);
|
|
30
|
+
if (epoch > 0) filters.push(`created_at_i>${epoch}`);
|
|
31
|
+
}
|
|
32
|
+
if (options.toDate) {
|
|
33
|
+
const epoch = toEpoch(options.toDate);
|
|
34
|
+
if (epoch > 0) filters.push(`created_at_i<${epoch}`);
|
|
35
|
+
}
|
|
36
|
+
if (filters.length) params.set("numericFilters", filters.join(","));
|
|
37
|
+
}
|
|
38
|
+
const url = `https://hn.algolia.com/api/v1/search?${params}`;
|
|
39
|
+
log.debug("hn algolia search", { query });
|
|
40
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
41
|
+
if (!response.ok) throw new Error(`HN Algolia returned ${response.status}`);
|
|
42
|
+
const data = await response.json();
|
|
43
|
+
return this.parseHits(data.hits ?? []);
|
|
44
|
+
}
|
|
45
|
+
parseHits(hits) {
|
|
46
|
+
const results = [];
|
|
47
|
+
const total = hits.length;
|
|
48
|
+
for (let i = 0; i < total; i++) {
|
|
49
|
+
const hit = hits[i];
|
|
50
|
+
const title = asString(hit.title);
|
|
51
|
+
if (!title) continue;
|
|
52
|
+
const objectID = asString(hit.objectID);
|
|
53
|
+
const url = asString(hit.url) ?? (objectID ? `https://news.ycombinator.com/item?id=${objectID}` : void 0);
|
|
54
|
+
if (!url) continue;
|
|
55
|
+
const storyText = asString(hit.story_text);
|
|
56
|
+
const points = asNumber(hit.points) ?? 0;
|
|
57
|
+
const comments = asNumber(hit.num_comments) ?? 0;
|
|
58
|
+
const snippet = storyText ? storyText.slice(0, SNIPPET_LIMIT) : `${points} points \xB7 ${comments} comments`;
|
|
59
|
+
const createdAt = asNumber(hit.created_at_i);
|
|
60
|
+
const published_date = createdAt ? new Date(createdAt * 1e3).toISOString() : void 0;
|
|
61
|
+
results.push({
|
|
62
|
+
title,
|
|
63
|
+
url,
|
|
64
|
+
snippet,
|
|
65
|
+
relevance_score: 1 - i / Math.max(total, 1),
|
|
66
|
+
engine: "hn-algolia",
|
|
67
|
+
...published_date ? { published_date } : {}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return results;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export {
|
|
74
|
+
HnAlgoliaEngine
|
|
75
|
+
};
|
|
76
|
+
//# sourceMappingURL=hn-algolia.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/hn-algolia.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface HnHit {\n objectID?: unknown;\n title?: unknown;\n url?: unknown;\n story_text?: unknown;\n points?: unknown;\n num_comments?: unknown;\n created_at_i?: unknown;\n}\n\ninterface HnResponse {\n hits?: HnHit[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction toEpoch(dateStr: string): number {\n const d = new Date(dateStr);\n const t = d.getTime();\n if (isNaN(t)) return 0;\n return Math.floor(t / 1000);\n}\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nexport class HnAlgoliaEngine implements SearchEngine {\n name = 'hn-algolia';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n query,\n hitsPerPage: String(maxResults),\n tags: 'story',\n });\n\n if (options.fromDate || options.toDate) {\n const filters: string[] = [];\n if (options.fromDate) {\n const epoch = toEpoch(options.fromDate);\n if (epoch > 0) filters.push(`created_at_i>${epoch}`);\n }\n if (options.toDate) {\n const epoch = toEpoch(options.toDate);\n if (epoch > 0) filters.push(`created_at_i<${epoch}`);\n }\n if (filters.length) params.set('numericFilters', filters.join(','));\n }\n\n const url = `https://hn.algolia.com/api/v1/search?${params}`;\n log.debug('hn algolia search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`HN Algolia returned ${response.status}`);\n\n const data = (await response.json()) as HnResponse;\n return this.parseHits(data.hits ?? []);\n }\n\n private parseHits(hits: HnHit[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = hits.length;\n\n for (let i = 0; i < total; i++) {\n const hit = hits[i];\n const title = asString(hit.title);\n if (!title) continue;\n\n const objectID = asString(hit.objectID);\n const url = asString(hit.url) ?? (objectID ? `https://news.ycombinator.com/item?id=${objectID}` : undefined);\n if (!url) continue;\n\n const storyText = asString(hit.story_text);\n const points = asNumber(hit.points) ?? 0;\n const comments = asNumber(hit.num_comments) ?? 0;\n const snippet = storyText\n ? storyText.slice(0, SNIPPET_LIMIT)\n : `${points} points · ${comments} comments`;\n\n const createdAt = asNumber(hit.created_at_i);\n const published_date = createdAt ? new Date(createdAt * 1000).toISOString() : undefined;\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'hn-algolia',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAgBjC,MAAM,gBAAgB;AAEtB,SAAS,QAAQ,SAAyB;AACxC,QAAM,IAAI,IAAI,KAAK,OAAO;AAC1B,QAAM,IAAI,EAAE,QAAQ;AACpB,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,KAAK,MAAM,IAAI,GAAI;AAC5B;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEO,MAAM,gBAAwC;AAAA,EACnD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC;AAAA,MACA,aAAa,OAAO,UAAU;AAAA,MAC9B,MAAM;AAAA,IACR,CAAC;AAED,QAAI,QAAQ,YAAY,QAAQ,QAAQ;AACtC,YAAM,UAAoB,CAAC;AAC3B,UAAI,QAAQ,UAAU;AACpB,cAAM,QAAQ,QAAQ,QAAQ,QAAQ;AACtC,YAAI,QAAQ,EAAG,SAAQ,KAAK,gBAAgB,KAAK,EAAE;AAAA,MACrD;AACA,UAAI,QAAQ,QAAQ;AAClB,cAAM,QAAQ,QAAQ,QAAQ,MAAM;AACpC,YAAI,QAAQ,EAAG,SAAQ,KAAK,gBAAgB,KAAK,EAAE;AAAA,MACrD;AACA,UAAI,QAAQ,OAAQ,QAAO,IAAI,kBAAkB,QAAQ,KAAK,GAAG,CAAC;AAAA,IACpE;AAEA,UAAM,MAAM,wCAAwC,MAAM;AAC1D,QAAI,MAAM,qBAAqB,EAAE,MAAM,CAAC;AAExC,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,uBAAuB,SAAS,MAAM,EAAE;AAE1E,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,UAAU,KAAK,QAAQ,CAAC,CAAC;AAAA,EACvC;AAAA,EAEQ,UAAU,MAAkC;AAClD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,UAAI,CAAC,MAAO;AAEZ,YAAM,WAAW,SAAS,IAAI,QAAQ;AACtC,YAAM,MAAM,SAAS,IAAI,GAAG,MAAM,WAAW,wCAAwC,QAAQ,KAAK;AAClG,UAAI,CAAC,IAAK;AAEV,YAAM,YAAY,SAAS,IAAI,UAAU;AACzC,YAAM,SAAS,SAAS,IAAI,MAAM,KAAK;AACvC,YAAM,WAAW,SAAS,IAAI,YAAY,KAAK;AAC/C,YAAM,UAAU,YACZ,UAAU,MAAM,GAAG,aAAa,IAChC,GAAG,MAAM,gBAAa,QAAQ;AAElC,YAAM,YAAY,SAAS,IAAI,YAAY;AAC3C,YAAM,iBAAiB,YAAY,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY,IAAI;AAE9E,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
|
|
2
|
+
export declare class LobstersEngine implements SearchEngine {
|
|
3
|
+
name: string;
|
|
4
|
+
search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
|
|
5
|
+
private parseHits;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=lobsters.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lobsters.d.ts","sourceRoot":"","sources":["../../../src/search/engines/lobsters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqCzF,qBAAa,cAAe,YAAW,YAAY;IACjD,IAAI,SAAc;IAEZ,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAqB1F,OAAO,CAAC,SAAS;CAsClB"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { createLogger } from "../../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
const SNIPPET_LIMIT = 200;
|
|
4
|
+
function asString(v) {
|
|
5
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
6
|
+
}
|
|
7
|
+
function asNumber(v) {
|
|
8
|
+
return typeof v === "number" && isFinite(v) ? v : void 0;
|
|
9
|
+
}
|
|
10
|
+
function extractHits(data) {
|
|
11
|
+
if (Array.isArray(data)) return data;
|
|
12
|
+
if (data && typeof data === "object" && "results" in data) {
|
|
13
|
+
const r = data.results;
|
|
14
|
+
if (Array.isArray(r)) return r;
|
|
15
|
+
}
|
|
16
|
+
return [];
|
|
17
|
+
}
|
|
18
|
+
class LobstersEngine {
|
|
19
|
+
name = "lobsters";
|
|
20
|
+
async search(query, options = {}) {
|
|
21
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
22
|
+
const maxResults = options.maxResults ?? 10;
|
|
23
|
+
const params = new URLSearchParams({ q: query, what: "stories" });
|
|
24
|
+
const url = `https://lobste.rs/search.json?${params}`;
|
|
25
|
+
log.debug("lobsters search", { query });
|
|
26
|
+
const response = await fetch(url, {
|
|
27
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
28
|
+
headers: { Accept: "application/json" }
|
|
29
|
+
});
|
|
30
|
+
if (!response.ok) throw new Error(`Lobsters returned ${response.status}`);
|
|
31
|
+
const data = await response.json();
|
|
32
|
+
const hits = extractHits(data);
|
|
33
|
+
const mapped = this.parseHits(hits);
|
|
34
|
+
const filtered = applyDateFilter(mapped, options);
|
|
35
|
+
return filtered.slice(0, maxResults);
|
|
36
|
+
}
|
|
37
|
+
parseHits(hits) {
|
|
38
|
+
const results = [];
|
|
39
|
+
const total = hits.length;
|
|
40
|
+
for (let i = 0; i < total; i++) {
|
|
41
|
+
const hit = hits[i];
|
|
42
|
+
const title = asString(hit.title);
|
|
43
|
+
if (!title) continue;
|
|
44
|
+
const url = asString(hit.url) ?? asString(hit.short_id_url);
|
|
45
|
+
if (!url) continue;
|
|
46
|
+
const description = asString(hit.description);
|
|
47
|
+
const score = asNumber(hit.score) ?? 0;
|
|
48
|
+
const comments = asNumber(hit.comment_count) ?? 0;
|
|
49
|
+
const snippet = description ? description.slice(0, SNIPPET_LIMIT) : `${score} score \xB7 ${comments} comments`;
|
|
50
|
+
const createdAt = asString(hit.created_at);
|
|
51
|
+
let published_date;
|
|
52
|
+
if (createdAt) {
|
|
53
|
+
const d = new Date(createdAt);
|
|
54
|
+
if (!isNaN(d.getTime())) published_date = d.toISOString();
|
|
55
|
+
}
|
|
56
|
+
results.push({
|
|
57
|
+
title,
|
|
58
|
+
url,
|
|
59
|
+
snippet,
|
|
60
|
+
relevance_score: 1 - i / Math.max(total, 1),
|
|
61
|
+
engine: "lobsters",
|
|
62
|
+
...published_date ? { published_date } : {}
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return results;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function applyDateFilter(results, options) {
|
|
69
|
+
if (!options.fromDate && !options.toDate) return results;
|
|
70
|
+
const from = options.fromDate ? new Date(options.fromDate).getTime() : -Infinity;
|
|
71
|
+
const to = options.toDate ? new Date(options.toDate).getTime() : Infinity;
|
|
72
|
+
if (isNaN(from) || isNaN(to)) return results;
|
|
73
|
+
return results.filter((r) => {
|
|
74
|
+
if (!r.published_date) return false;
|
|
75
|
+
const t = new Date(r.published_date).getTime();
|
|
76
|
+
if (isNaN(t)) return false;
|
|
77
|
+
return t >= from && t <= to;
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
export {
|
|
81
|
+
LobstersEngine
|
|
82
|
+
};
|
|
83
|
+
//# sourceMappingURL=lobsters.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/lobsters.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\n// Lobste.rs /search.json sometimes returns an array directly, sometimes a\n// { results: [...] } envelope. Both shapes are handled defensively.\ninterface LobsterHit {\n short_id?: unknown;\n title?: unknown;\n url?: unknown;\n score?: unknown;\n description?: unknown;\n comment_count?: unknown;\n created_at?: unknown;\n short_id_url?: unknown;\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction extractHits(data: unknown): LobsterHit[] {\n if (Array.isArray(data)) return data as LobsterHit[];\n if (data && typeof data === 'object' && 'results' in data) {\n const r = (data as { results?: unknown }).results;\n if (Array.isArray(r)) return r as LobsterHit[];\n }\n return [];\n}\n\nexport class LobstersEngine implements SearchEngine {\n name = 'lobsters';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({ q: query, what: 'stories' });\n const url = `https://lobste.rs/search.json?${params}`;\n log.debug('lobsters search', { query });\n\n const response = await fetch(url, {\n signal: AbortSignal.timeout(timeoutMs),\n headers: { Accept: 'application/json' },\n });\n if (!response.ok) throw new Error(`Lobsters returned ${response.status}`);\n\n const data = (await response.json()) as unknown;\n const hits = extractHits(data);\n const mapped = this.parseHits(hits);\n const filtered = applyDateFilter(mapped, options);\n return filtered.slice(0, maxResults);\n }\n\n private parseHits(hits: LobsterHit[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = hits.length;\n\n for (let i = 0; i < total; i++) {\n const hit = hits[i];\n const title = asString(hit.title);\n if (!title) continue;\n\n const url = asString(hit.url) ?? asString(hit.short_id_url);\n if (!url) continue;\n\n const description = asString(hit.description);\n const score = asNumber(hit.score) ?? 0;\n const comments = asNumber(hit.comment_count) ?? 0;\n const snippet = description\n ? description.slice(0, SNIPPET_LIMIT)\n : `${score} score · ${comments} comments`;\n\n const createdAt = asString(hit.created_at);\n let published_date: string | undefined;\n if (createdAt) {\n const d = new Date(createdAt);\n if (!isNaN(d.getTime())) published_date = d.toISOString();\n }\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'lobsters',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n\nfunction applyDateFilter(results: RawSearchResult[], options: SearchEngineOptions): RawSearchResult[] {\n if (!options.fromDate && !options.toDate) return results;\n const from = options.fromDate ? new Date(options.fromDate).getTime() : -Infinity;\n const to = options.toDate ? new Date(options.toDate).getTime() : Infinity;\n if (isNaN(from) || isNaN(to)) return results;\n return results.filter((r) => {\n if (!r.published_date) return false;\n const t = new Date(r.published_date).getTime();\n if (isNaN(t)) return false;\n return t >= from && t <= to;\n });\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAejC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,YAAY,MAA6B;AAChD,MAAI,MAAM,QAAQ,IAAI,EAAG,QAAO;AAChC,MAAI,QAAQ,OAAO,SAAS,YAAY,aAAa,MAAM;AACzD,UAAM,IAAK,KAA+B;AAC1C,QAAI,MAAM,QAAQ,CAAC,EAAG,QAAO;AAAA,EAC/B;AACA,SAAO,CAAC;AACV;AAEO,MAAM,eAAuC;AAAA,EAClD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB,EAAE,GAAG,OAAO,MAAM,UAAU,CAAC;AAChE,UAAM,MAAM,iCAAiC,MAAM;AACnD,QAAI,MAAM,mBAAmB,EAAE,MAAM,CAAC;AAEtC,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,QAAQ,mBAAmB;AAAA,IACxC,CAAC;AACD,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,qBAAqB,SAAS,MAAM,EAAE;AAExE,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,SAAS,KAAK,UAAU,IAAI;AAClC,UAAM,WAAW,gBAAgB,QAAQ,OAAO;AAChD,WAAO,SAAS,MAAM,GAAG,UAAU;AAAA,EACrC;AAAA,EAEQ,UAAU,MAAuC;AACvD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,UAAI,CAAC,MAAO;AAEZ,YAAM,MAAM,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,YAAY;AAC1D,UAAI,CAAC,IAAK;AAEV,YAAM,cAAc,SAAS,IAAI,WAAW;AAC5C,YAAM,QAAQ,SAAS,IAAI,KAAK,KAAK;AACrC,YAAM,WAAW,SAAS,IAAI,aAAa,KAAK;AAChD,YAAM,UAAU,cACZ,YAAY,MAAM,GAAG,aAAa,IAClC,GAAG,KAAK,eAAY,QAAQ;AAEhC,YAAM,YAAY,SAAS,IAAI,UAAU;AACzC,UAAI;AACJ,UAAI,WAAW;AACb,cAAM,IAAI,IAAI,KAAK,SAAS;AAC5B,YAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAG,kBAAiB,EAAE,YAAY;AAAA,MAC1D;AAEA,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;AAEA,SAAS,gBAAgB,SAA4B,SAAiD;AACpG,MAAI,CAAC,QAAQ,YAAY,CAAC,QAAQ,OAAQ,QAAO;AACjD,QAAM,OAAO,QAAQ,WAAW,IAAI,KAAK,QAAQ,QAAQ,EAAE,QAAQ,IAAI;AACvE,QAAM,KAAK,QAAQ,SAAS,IAAI,KAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI;AACjE,MAAI,MAAM,IAAI,KAAK,MAAM,EAAE,EAAG,QAAO;AACrC,SAAO,QAAQ,OAAO,CAAC,MAAM;AAC3B,QAAI,CAAC,EAAE,eAAgB,QAAO;AAC9B,UAAM,IAAI,IAAI,KAAK,EAAE,cAAc,EAAE,QAAQ;AAC7C,QAAI,MAAM,CAAC,EAAG,QAAO;AACrB,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B,CAAC;AACH;","names":[]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
|
|
2
|
+
export declare class MdnEngine implements SearchEngine {
|
|
3
|
+
name: string;
|
|
4
|
+
search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
|
|
5
|
+
private parseDocs;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=mdn.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mdn.d.ts","sourceRoot":"","sources":["../../../src/search/engines/mdn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqBzF,qBAAa,SAAU,YAAW,YAAY;IAC5C,IAAI,SAAS;IAEP,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAmB1F,OAAO,CAAC,SAAS;CAwBlB"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { createLogger } from "../../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
const MDN_HOST = "https://developer.mozilla.org";
|
|
4
|
+
function asString(v) {
|
|
5
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
6
|
+
}
|
|
7
|
+
class MdnEngine {
|
|
8
|
+
name = "mdn";
|
|
9
|
+
async search(query, options = {}) {
|
|
10
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
11
|
+
const maxResults = options.maxResults ?? 10;
|
|
12
|
+
const params = new URLSearchParams({
|
|
13
|
+
q: query,
|
|
14
|
+
locale: "en-US",
|
|
15
|
+
size: String(maxResults)
|
|
16
|
+
});
|
|
17
|
+
const url = `${MDN_HOST}/api/v1/search?${params}`;
|
|
18
|
+
log.debug("mdn search", { query });
|
|
19
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
20
|
+
if (!response.ok) throw new Error(`MDN returned ${response.status}`);
|
|
21
|
+
const data = await response.json();
|
|
22
|
+
return this.parseDocs(data.documents ?? []);
|
|
23
|
+
}
|
|
24
|
+
parseDocs(docs) {
|
|
25
|
+
const results = [];
|
|
26
|
+
const total = docs.length;
|
|
27
|
+
for (let i = 0; i < total; i++) {
|
|
28
|
+
const doc = docs[i];
|
|
29
|
+
const title = asString(doc.title);
|
|
30
|
+
const path = asString(doc.mdn_url);
|
|
31
|
+
if (!title || !path) continue;
|
|
32
|
+
const summary = asString(doc.summary) ?? "";
|
|
33
|
+
const url = path.startsWith("http") ? path : `${MDN_HOST}${path}`;
|
|
34
|
+
results.push({
|
|
35
|
+
title,
|
|
36
|
+
url,
|
|
37
|
+
snippet: summary,
|
|
38
|
+
relevance_score: 1 - i / Math.max(total, 1),
|
|
39
|
+
engine: "mdn"
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
return results;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export {
|
|
46
|
+
MdnEngine
|
|
47
|
+
};
|
|
48
|
+
//# sourceMappingURL=mdn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/mdn.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface MdnDoc {\n mdn_url?: unknown;\n title?: unknown;\n summary?: unknown;\n}\n\ninterface MdnResponse {\n documents?: MdnDoc[];\n}\n\nconst MDN_HOST = 'https://developer.mozilla.org';\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nexport class MdnEngine implements SearchEngine {\n name = 'mdn';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n q: query,\n locale: 'en-US',\n size: String(maxResults),\n });\n const url = `${MDN_HOST}/api/v1/search?${params}`;\n log.debug('mdn search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`MDN returned ${response.status}`);\n\n const data = (await response.json()) as MdnResponse;\n return this.parseDocs(data.documents ?? []);\n }\n\n private parseDocs(docs: MdnDoc[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = docs.length;\n\n for (let i = 0; i < total; i++) {\n const doc = docs[i];\n const title = asString(doc.title);\n const path = asString(doc.mdn_url);\n if (!title || !path) continue;\n\n const summary = asString(doc.summary) ?? '';\n const url = path.startsWith('http') ? path : `${MDN_HOST}${path}`;\n\n results.push({\n title,\n url,\n snippet: summary,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'mdn',\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAYjC,MAAM,WAAW;AAEjB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEO,MAAM,UAAkC;AAAA,EAC7C,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,GAAG;AAAA,MACH,QAAQ;AAAA,MACR,MAAM,OAAO,UAAU;AAAA,IACzB,CAAC;AACD,UAAM,MAAM,GAAG,QAAQ,kBAAkB,MAAM;AAC/C,QAAI,MAAM,cAAc,EAAE,MAAM,CAAC;AAEjC,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,gBAAgB,SAAS,MAAM,EAAE;AAEnE,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,UAAU,KAAK,aAAa,CAAC,CAAC;AAAA,EAC5C;AAAA,EAEQ,UAAU,MAAmC;AACnD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,KAAK;AAEnB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,YAAM,OAAO,SAAS,IAAI,OAAO;AACjC,UAAI,CAAC,SAAS,CAAC,KAAM;AAErB,YAAM,UAAU,SAAS,IAAI,OAAO,KAAK;AACzC,YAAM,MAAM,KAAK,WAAW,MAAM,IAAI,OAAO,GAAG,QAAQ,GAAG,IAAI;AAE/D,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA,SAAS;AAAA,QACT,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
|
|
2
|
+
export declare class SemanticScholarEngine implements SearchEngine {
|
|
3
|
+
name: string;
|
|
4
|
+
search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
|
|
5
|
+
private parsePapers;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=semantic-scholar.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic-scholar.d.ts","sourceRoot":"","sources":["../../../src/search/engines/semantic-scholar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAsCzF,qBAAa,qBAAsB,YAAW,YAAY;IACxD,IAAI,SAAsB;IAEpB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA4B1F,OAAO,CAAC,WAAW;CAgCpB"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { createLogger } from "../../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
const SNIPPET_LIMIT = 200;
|
|
4
|
+
function asString(v) {
|
|
5
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
6
|
+
}
|
|
7
|
+
function asNumber(v) {
|
|
8
|
+
return typeof v === "number" && isFinite(v) ? v : void 0;
|
|
9
|
+
}
|
|
10
|
+
function yearOf(iso) {
|
|
11
|
+
const t = new Date(iso).getTime();
|
|
12
|
+
if (isNaN(t)) return void 0;
|
|
13
|
+
return new Date(t).getUTCFullYear();
|
|
14
|
+
}
|
|
15
|
+
class SemanticScholarEngine {
|
|
16
|
+
name = "semantic-scholar";
|
|
17
|
+
async search(query, options = {}) {
|
|
18
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
19
|
+
const maxResults = options.maxResults ?? 10;
|
|
20
|
+
const params = new URLSearchParams({
|
|
21
|
+
query,
|
|
22
|
+
limit: String(maxResults),
|
|
23
|
+
fields: "title,abstract,year,url,authors,externalIds,openAccessPdf"
|
|
24
|
+
});
|
|
25
|
+
if (options.fromDate || options.toDate) {
|
|
26
|
+
const fromY = options.fromDate ? yearOf(options.fromDate) : void 0;
|
|
27
|
+
const toY = options.toDate ? yearOf(options.toDate) : void 0;
|
|
28
|
+
if (fromY !== void 0 || toY !== void 0) {
|
|
29
|
+
params.set("year", `${fromY ?? ""}-${toY ?? ""}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const url = `https://api.semanticscholar.org/graph/v1/paper/search?${params}`;
|
|
33
|
+
log.debug("semantic-scholar search", { query });
|
|
34
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
35
|
+
if (!response.ok) throw new Error(`Semantic Scholar returned ${response.status}`);
|
|
36
|
+
const data = await response.json();
|
|
37
|
+
return this.parsePapers(data.data ?? []);
|
|
38
|
+
}
|
|
39
|
+
parsePapers(papers) {
|
|
40
|
+
const results = [];
|
|
41
|
+
const total = papers.length;
|
|
42
|
+
for (let i = 0; i < total; i++) {
|
|
43
|
+
const p = papers[i];
|
|
44
|
+
const title = asString(p.title);
|
|
45
|
+
if (!title) continue;
|
|
46
|
+
const pdfUrl = asString(p.openAccessPdf?.url);
|
|
47
|
+
const baseUrl = asString(p.url);
|
|
48
|
+
const url = pdfUrl ?? baseUrl;
|
|
49
|
+
if (!url) continue;
|
|
50
|
+
const abstract = asString(p.abstract) ?? "";
|
|
51
|
+
const snippet = abstract.slice(0, SNIPPET_LIMIT);
|
|
52
|
+
const year = asNumber(p.year);
|
|
53
|
+
const published_date = year ? `${year}-01-01T00:00:00.000Z` : void 0;
|
|
54
|
+
results.push({
|
|
55
|
+
title,
|
|
56
|
+
url,
|
|
57
|
+
snippet,
|
|
58
|
+
relevance_score: 1 - i / Math.max(total, 1),
|
|
59
|
+
engine: "semantic-scholar",
|
|
60
|
+
...published_date ? { published_date } : {}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return results;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export {
|
|
67
|
+
SemanticScholarEngine
|
|
68
|
+
};
|
|
69
|
+
//# sourceMappingURL=semantic-scholar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/semantic-scholar.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface OpenAccessPdf {\n url?: unknown;\n}\n\ninterface S2Paper {\n paperId?: unknown;\n title?: unknown;\n abstract?: unknown;\n year?: unknown;\n url?: unknown;\n openAccessPdf?: OpenAccessPdf;\n}\n\ninterface S2Response {\n data?: S2Paper[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction yearOf(iso: string): number | undefined {\n const t = new Date(iso).getTime();\n if (isNaN(t)) return undefined;\n return new Date(t).getUTCFullYear();\n}\n\nexport class SemanticScholarEngine implements SearchEngine {\n name = 'semantic-scholar';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n query,\n limit: String(maxResults),\n fields: 'title,abstract,year,url,authors,externalIds,openAccessPdf',\n });\n\n if (options.fromDate || options.toDate) {\n const fromY = options.fromDate ? yearOf(options.fromDate) : undefined;\n const toY = options.toDate ? yearOf(options.toDate) : undefined;\n if (fromY !== undefined || toY !== undefined) {\n params.set('year', `${fromY ?? ''}-${toY ?? ''}`);\n }\n }\n\n const url = `https://api.semanticscholar.org/graph/v1/paper/search?${params}`;\n log.debug('semantic-scholar search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`Semantic Scholar returned ${response.status}`);\n\n const data = (await response.json()) as S2Response;\n return this.parsePapers(data.data ?? []);\n }\n\n private parsePapers(papers: S2Paper[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = papers.length;\n\n for (let i = 0; i < total; i++) {\n const p = papers[i];\n const title = asString(p.title);\n if (!title) continue;\n\n const pdfUrl = asString(p.openAccessPdf?.url);\n const baseUrl = asString(p.url);\n const url = pdfUrl ?? baseUrl;\n if (!url) continue;\n\n const abstract = asString(p.abstract) ?? '';\n const snippet = abstract.slice(0, SNIPPET_LIMIT);\n\n const year = asNumber(p.year);\n const published_date = year ? `${year}-01-01T00:00:00.000Z` : undefined;\n\n results.push({\n title,\n url,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'semantic-scholar',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAmBjC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,OAAO,KAAiC;AAC/C,QAAM,IAAI,IAAI,KAAK,GAAG,EAAE,QAAQ;AAChC,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,IAAI,KAAK,CAAC,EAAE,eAAe;AACpC;AAEO,MAAM,sBAA8C;AAAA,EACzD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC;AAAA,MACA,OAAO,OAAO,UAAU;AAAA,MACxB,QAAQ;AAAA,IACV,CAAC;AAED,QAAI,QAAQ,YAAY,QAAQ,QAAQ;AACtC,YAAM,QAAQ,QAAQ,WAAW,OAAO,QAAQ,QAAQ,IAAI;AAC5D,YAAM,MAAM,QAAQ,SAAS,OAAO,QAAQ,MAAM,IAAI;AACtD,UAAI,UAAU,UAAa,QAAQ,QAAW;AAC5C,eAAO,IAAI,QAAQ,GAAG,SAAS,EAAE,IAAI,OAAO,EAAE,EAAE;AAAA,MAClD;AAAA,IACF;AAEA,UAAM,MAAM,yDAAyD,MAAM;AAC3E,QAAI,MAAM,2BAA2B,EAAE,MAAM,CAAC;AAE9C,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,6BAA6B,SAAS,MAAM,EAAE;AAEhF,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,YAAY,KAAK,QAAQ,CAAC,CAAC;AAAA,EACzC;AAAA,EAEQ,YAAY,QAAsC;AACxD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,OAAO;AAErB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,IAAI,OAAO,CAAC;AAClB,YAAM,QAAQ,SAAS,EAAE,KAAK;AAC9B,UAAI,CAAC,MAAO;AAEZ,YAAM,SAAS,SAAS,EAAE,eAAe,GAAG;AAC5C,YAAM,UAAU,SAAS,EAAE,GAAG;AAC9B,YAAM,MAAM,UAAU;AACtB,UAAI,CAAC,IAAK;AAEV,YAAM,WAAW,SAAS,EAAE,QAAQ,KAAK;AACzC,YAAM,UAAU,SAAS,MAAM,GAAG,aAAa;AAE/C,YAAM,OAAO,SAAS,EAAE,IAAI;AAC5B,YAAM,iBAAiB,OAAO,GAAG,IAAI,yBAAyB;AAE9D,cAAQ,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
|
|
2
|
+
export declare class StackOverflowEngine implements SearchEngine {
|
|
3
|
+
name: string;
|
|
4
|
+
search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
|
|
5
|
+
private parseItems;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=stackoverflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stackoverflow.d.ts","sourceRoot":"","sources":["../../../src/search/engines/stackoverflow.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAuCzF,qBAAa,mBAAoB,YAAW,YAAY;IACtD,IAAI,SAAmB;IAEjB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgC1F,OAAO,CAAC,UAAU;CA4BnB"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { createLogger } from "../../logger.js";
|
|
2
|
+
const log = createLogger("search");
|
|
3
|
+
const SNIPPET_LIMIT = 200;
|
|
4
|
+
function asString(v) {
|
|
5
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
6
|
+
}
|
|
7
|
+
function asNumber(v) {
|
|
8
|
+
return typeof v === "number" && isFinite(v) ? v : void 0;
|
|
9
|
+
}
|
|
10
|
+
function stripHtml(html) {
|
|
11
|
+
return html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
12
|
+
}
|
|
13
|
+
function toEpochSeconds(iso) {
|
|
14
|
+
const t = new Date(iso).getTime();
|
|
15
|
+
if (isNaN(t)) return void 0;
|
|
16
|
+
return Math.floor(t / 1e3);
|
|
17
|
+
}
|
|
18
|
+
class StackOverflowEngine {
|
|
19
|
+
name = "stackoverflow";
|
|
20
|
+
async search(query, options = {}) {
|
|
21
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
22
|
+
const maxResults = options.maxResults ?? 10;
|
|
23
|
+
const params = new URLSearchParams({
|
|
24
|
+
order: "desc",
|
|
25
|
+
sort: "relevance",
|
|
26
|
+
q: query,
|
|
27
|
+
site: "stackoverflow",
|
|
28
|
+
filter: "withbody",
|
|
29
|
+
pagesize: String(maxResults)
|
|
30
|
+
});
|
|
31
|
+
if (options.fromDate) {
|
|
32
|
+
const epoch = toEpochSeconds(options.fromDate);
|
|
33
|
+
if (epoch !== void 0) params.set("fromdate", String(epoch));
|
|
34
|
+
}
|
|
35
|
+
if (options.toDate) {
|
|
36
|
+
const epoch = toEpochSeconds(options.toDate);
|
|
37
|
+
if (epoch !== void 0) params.set("todate", String(epoch));
|
|
38
|
+
}
|
|
39
|
+
const url = `https://api.stackexchange.com/2.3/search/advanced?${params}`;
|
|
40
|
+
log.debug("stackoverflow search", { query });
|
|
41
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
42
|
+
if (!response.ok) throw new Error(`StackOverflow returned ${response.status}`);
|
|
43
|
+
const data = await response.json();
|
|
44
|
+
return this.parseItems(data.items ?? []);
|
|
45
|
+
}
|
|
46
|
+
parseItems(items) {
|
|
47
|
+
const results = [];
|
|
48
|
+
const total = items.length;
|
|
49
|
+
for (let i = 0; i < total; i++) {
|
|
50
|
+
const item = items[i];
|
|
51
|
+
const title = asString(item.title);
|
|
52
|
+
const link = asString(item.link);
|
|
53
|
+
if (!title || !link) continue;
|
|
54
|
+
const bodyHtml = asString(item.body) ?? "";
|
|
55
|
+
const snippet = stripHtml(bodyHtml).slice(0, SNIPPET_LIMIT);
|
|
56
|
+
const createdAt = asNumber(item.creation_date);
|
|
57
|
+
const published_date = createdAt ? new Date(createdAt * 1e3).toISOString() : void 0;
|
|
58
|
+
results.push({
|
|
59
|
+
title,
|
|
60
|
+
url: link,
|
|
61
|
+
snippet,
|
|
62
|
+
relevance_score: 1 - i / Math.max(total, 1),
|
|
63
|
+
engine: "stackoverflow",
|
|
64
|
+
...published_date ? { published_date } : {}
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return results;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export {
|
|
71
|
+
StackOverflowEngine
|
|
72
|
+
};
|
|
73
|
+
//# sourceMappingURL=stackoverflow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/stackoverflow.ts"],"sourcesContent":["import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\ninterface SoItem {\n title?: unknown;\n link?: unknown;\n body?: unknown;\n creation_date?: unknown;\n}\n\ninterface SoResponse {\n items?: SoItem[];\n}\n\nconst SNIPPET_LIMIT = 200;\n\nfunction asString(v: unknown): string | undefined {\n return typeof v === 'string' && v.length > 0 ? v : undefined;\n}\n\nfunction asNumber(v: unknown): number | undefined {\n return typeof v === 'number' && isFinite(v) ? v : undefined;\n}\n\nfunction stripHtml(html: string): string {\n return html\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\nfunction toEpochSeconds(iso: string): number | undefined {\n const t = new Date(iso).getTime();\n if (isNaN(t)) return undefined;\n return Math.floor(t / 1000);\n}\n\nexport class StackOverflowEngine implements SearchEngine {\n name = 'stackoverflow';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({\n order: 'desc',\n sort: 'relevance',\n q: query,\n site: 'stackoverflow',\n filter: 'withbody',\n pagesize: String(maxResults),\n });\n\n if (options.fromDate) {\n const epoch = toEpochSeconds(options.fromDate);\n if (epoch !== undefined) params.set('fromdate', String(epoch));\n }\n if (options.toDate) {\n const epoch = toEpochSeconds(options.toDate);\n if (epoch !== undefined) params.set('todate', String(epoch));\n }\n\n const url = `https://api.stackexchange.com/2.3/search/advanced?${params}`;\n log.debug('stackoverflow search', { query });\n\n const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n if (!response.ok) throw new Error(`StackOverflow returned ${response.status}`);\n\n const data = (await response.json()) as SoResponse;\n return this.parseItems(data.items ?? []);\n }\n\n private parseItems(items: SoItem[]): RawSearchResult[] {\n const results: RawSearchResult[] = [];\n const total = items.length;\n\n for (let i = 0; i < total; i++) {\n const item = items[i];\n const title = asString(item.title);\n const link = asString(item.link);\n if (!title || !link) continue;\n\n const bodyHtml = asString(item.body) ?? '';\n const snippet = stripHtml(bodyHtml).slice(0, SNIPPET_LIMIT);\n\n const createdAt = asNumber(item.creation_date);\n const published_date = createdAt ? new Date(createdAt * 1000).toISOString() : undefined;\n\n results.push({\n title,\n url: link,\n snippet,\n relevance_score: 1 - i / Math.max(total, 1),\n engine: 'stackoverflow',\n ...(published_date ? { published_date } : {}),\n });\n }\n\n return results;\n }\n}\n"],"mappings":"AACA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAajC,MAAM,gBAAgB;AAEtB,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,EAAE,SAAS,IAAI,IAAI;AACrD;AAEA,SAAS,SAAS,GAAgC;AAChD,SAAO,OAAO,MAAM,YAAY,SAAS,CAAC,IAAI,IAAI;AACpD;AAEA,SAAS,UAAU,MAAsB;AACvC,SAAO,KACJ,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEA,SAAS,eAAe,KAAiC;AACvD,QAAM,IAAI,IAAI,KAAK,GAAG,EAAE,QAAQ;AAChC,MAAI,MAAM,CAAC,EAAG,QAAO;AACrB,SAAO,KAAK,MAAM,IAAI,GAAI;AAC5B;AAEO,MAAM,oBAA4C;AAAA,EACvD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,OAAO;AAAA,MACP,MAAM;AAAA,MACN,GAAG;AAAA,MACH,MAAM;AAAA,MACN,QAAQ;AAAA,MACR,UAAU,OAAO,UAAU;AAAA,IAC7B,CAAC;AAED,QAAI,QAAQ,UAAU;AACpB,YAAM,QAAQ,eAAe,QAAQ,QAAQ;AAC7C,UAAI,UAAU,OAAW,QAAO,IAAI,YAAY,OAAO,KAAK,CAAC;AAAA,IAC/D;AACA,QAAI,QAAQ,QAAQ;AAClB,YAAM,QAAQ,eAAe,QAAQ,MAAM;AAC3C,UAAI,UAAU,OAAW,QAAO,IAAI,UAAU,OAAO,KAAK,CAAC;AAAA,IAC7D;AAEA,UAAM,MAAM,qDAAqD,MAAM;AACvE,QAAI,MAAM,wBAAwB,EAAE,MAAM,CAAC;AAE3C,UAAM,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;AAC5E,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,0BAA0B,SAAS,MAAM,EAAE;AAE7E,UAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,WAAO,KAAK,WAAW,KAAK,SAAS,CAAC,CAAC;AAAA,EACzC;AAAA,EAEQ,WAAW,OAAoC;AACrD,UAAM,UAA6B,CAAC;AACpC,UAAM,QAAQ,MAAM;AAEpB,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,OAAO,MAAM,CAAC;AACpB,YAAM,QAAQ,SAAS,KAAK,KAAK;AACjC,YAAM,OAAO,SAAS,KAAK,IAAI;AAC/B,UAAI,CAAC,SAAS,CAAC,KAAM;AAErB,YAAM,WAAW,SAAS,KAAK,IAAI,KAAK;AACxC,YAAM,UAAU,UAAU,QAAQ,EAAE,MAAM,GAAG,aAAa;AAE1D,YAAM,YAAY,SAAS,KAAK,aAAa;AAC7C,YAAM,iBAAiB,YAAY,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY,IAAI;AAE9E,cAAQ,KAAK;AAAA,QACX;AAAA,QACA,KAAK;AAAA,QACL;AAAA,QACA,iBAAiB,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,QAC1C,QAAQ;AAAA,QACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"startpage.d.ts","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"startpage.d.ts","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAqBzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAe;IAEb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CAkClE"}
|
|
@@ -1,50 +1,69 @@
|
|
|
1
|
-
import { parseHTML } from
|
|
2
|
-
import { createLogger } from
|
|
3
|
-
const log = createLogger(
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
1
|
+
import { parseHTML } from "linkedom";
|
|
2
|
+
import { createLogger } from "../../logger.js";
|
|
3
|
+
const log = createLogger("search");
|
|
4
|
+
const DATE_SNIPPET_PATTERN = /^(\w{3}\s+\d{1,2},\s+\d{4}|\d{4}-\d{2}-\d{2})\s*[·—–-]/;
|
|
5
|
+
function parseDateFromEl(el) {
|
|
6
|
+
if (!el?.textContent) return void 0;
|
|
7
|
+
const text = el.textContent.trim();
|
|
8
|
+
const d = new Date(text);
|
|
9
|
+
return isNaN(d.getTime()) ? void 0 : d.toISOString();
|
|
10
|
+
}
|
|
11
|
+
function parseDateFromSnippet(snippet) {
|
|
12
|
+
const match = snippet.trim().match(DATE_SNIPPET_PATTERN);
|
|
13
|
+
if (!match) return void 0;
|
|
14
|
+
const d = new Date(match[1]);
|
|
15
|
+
return isNaN(d.getTime()) ? void 0 : d.toISOString();
|
|
16
|
+
}
|
|
17
|
+
class StartpageEngine {
|
|
18
|
+
name = "startpage";
|
|
19
|
+
async search(query, options = {}) {
|
|
20
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
21
|
+
const maxResults = options.maxResults ?? 10;
|
|
22
|
+
const params = new URLSearchParams({ query, cat: "web" });
|
|
23
|
+
const url = `https://www.startpage.com/sp/search?${params}`;
|
|
24
|
+
log.debug("scraping startpage", { query });
|
|
25
|
+
const response = await fetch(url, {
|
|
26
|
+
method: "POST",
|
|
27
|
+
headers: {
|
|
28
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
29
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
30
|
+
},
|
|
31
|
+
body: params.toString(),
|
|
32
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
33
|
+
});
|
|
34
|
+
if (!response.ok) throw new Error(`Startpage returned ${response.status}`);
|
|
35
|
+
const html = await response.text();
|
|
36
|
+
return this.parseResults(html, maxResults);
|
|
37
|
+
}
|
|
38
|
+
parseResults(html, maxResults) {
|
|
39
|
+
const { document } = parseHTML(html);
|
|
40
|
+
const results = [];
|
|
41
|
+
const items = document.querySelectorAll(".w-gl__result");
|
|
42
|
+
const total = Math.min(items.length, maxResults);
|
|
43
|
+
for (let i = 0; i < total; i++) {
|
|
44
|
+
const item = items[i];
|
|
45
|
+
const link = item.querySelector(".w-gl__result-title, a.w-gl__result-url");
|
|
46
|
+
const snippetEl = item.querySelector(".w-gl__description");
|
|
47
|
+
const href = link?.getAttribute("href");
|
|
48
|
+
const title = link?.textContent?.trim();
|
|
49
|
+
if (href && title) {
|
|
50
|
+
const dateEl = item.querySelector(".w-gl__result-date, time");
|
|
51
|
+
const snippetText = snippetEl?.textContent?.trim() ?? "";
|
|
52
|
+
const published_date = parseDateFromEl(dateEl) ?? parseDateFromSnippet(snippetText);
|
|
53
|
+
results.push({
|
|
54
|
+
title,
|
|
55
|
+
url: href,
|
|
56
|
+
snippet: snippetText,
|
|
57
|
+
relevance_score: 1 - i / Math.max(items.length, 1),
|
|
58
|
+
engine: "startpage",
|
|
59
|
+
...published_date ? { published_date } : {}
|
|
20
60
|
});
|
|
21
|
-
|
|
22
|
-
throw new Error(`Startpage returned ${response.status}`);
|
|
23
|
-
const html = await response.text();
|
|
24
|
-
return this.parseResults(html, maxResults);
|
|
25
|
-
}
|
|
26
|
-
parseResults(html, maxResults) {
|
|
27
|
-
const { document } = parseHTML(html);
|
|
28
|
-
const results = [];
|
|
29
|
-
const items = document.querySelectorAll('.w-gl__result');
|
|
30
|
-
const total = Math.min(items.length, maxResults);
|
|
31
|
-
for (let i = 0; i < total; i++) {
|
|
32
|
-
const item = items[i];
|
|
33
|
-
const link = item.querySelector('.w-gl__result-title, a.w-gl__result-url');
|
|
34
|
-
const snippetEl = item.querySelector('.w-gl__description');
|
|
35
|
-
const href = link?.getAttribute('href');
|
|
36
|
-
const title = link?.textContent?.trim();
|
|
37
|
-
if (href && title) {
|
|
38
|
-
results.push({
|
|
39
|
-
title,
|
|
40
|
-
url: href,
|
|
41
|
-
snippet: snippetEl?.textContent?.trim() ?? '',
|
|
42
|
-
relevance_score: 1 - i / Math.max(items.length, 1),
|
|
43
|
-
engine: 'startpage',
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
return results;
|
|
61
|
+
}
|
|
48
62
|
}
|
|
63
|
+
return results;
|
|
64
|
+
}
|
|
49
65
|
}
|
|
66
|
+
export {
|
|
67
|
+
StartpageEngine
|
|
68
|
+
};
|
|
50
69
|
//# sourceMappingURL=startpage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"sources":["../../../src/search/engines/startpage.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\nconst DATE_SNIPPET_PATTERN = /^(\\w{3}\\s+\\d{1,2},\\s+\\d{4}|\\d{4}-\\d{2}-\\d{2})\\s*[·—–-]/;\n\nfunction parseDateFromEl(el: { textContent?: string | null } | null): string | undefined {\n if (!el?.textContent) return undefined;\n const text = el.textContent.trim();\n const d = new Date(text);\n return isNaN(d.getTime()) ? undefined : d.toISOString();\n}\n\nfunction parseDateFromSnippet(snippet: string): string | undefined {\n const match = snippet.trim().match(DATE_SNIPPET_PATTERN);\n if (!match) return undefined;\n const d = new Date(match[1]);\n return isNaN(d.getTime()) ? undefined : d.toISOString();\n}\n\nexport class StartpageEngine implements SearchEngine {\n name = 'startpage';\n\n async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {\n const timeoutMs = options.timeoutMs ?? 10000;\n const maxResults = options.maxResults ?? 10;\n\n const params = new URLSearchParams({ query, cat: 'web' });\n const url = `https://www.startpage.com/sp/search?${params}`;\n\n log.debug('scraping startpage', { query });\n\n const response = await fetch(url, {\n method: 'POST',\n headers: {\n 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',\n 'Content-Type': 'application/x-www-form-urlencoded',\n },\n body: params.toString(),\n signal: AbortSignal.timeout(timeoutMs),\n });\n\n if (!response.ok) throw new Error(`Startpage returned ${response.status}`);\n\n const html = await response.text();\n return this.parseResults(html, maxResults);\n }\n\n parseResults(html: string, maxResults: number): RawSearchResult[] {\n const { document } = parseHTML(html);\n const results: RawSearchResult[] = [];\n\n const items = document.querySelectorAll('.w-gl__result');\n const total = Math.min(items.length, maxResults);\n\n for (let i = 0; i < total; i++) {\n const item = items[i];\n const link = item.querySelector('.w-gl__result-title, a.w-gl__result-url');\n const snippetEl = item.querySelector('.w-gl__description');\n\n const href = link?.getAttribute('href');\n const title = link?.textContent?.trim();\n\n if (href && title) {\n // Startpage sometimes shows dates in a dedicated element or snippet prefix\n const dateEl = item.querySelector('.w-gl__result-date, time');\n const snippetText = snippetEl?.textContent?.trim() ?? '';\n const published_date = parseDateFromEl(dateEl) ?? parseDateFromSnippet(snippetText);\n\n results.push({\n title,\n url: href,\n snippet: snippetText,\n relevance_score: 1 - i / Math.max(items.length, 1),\n engine: 'startpage',\n ...(published_date ? { published_date } : {}),\n });\n }\n }\n\n return results;\n }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAE1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,uBAAuB;AAE7B,SAAS,gBAAgB,IAAgE;AACvF,MAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,QAAM,OAAO,GAAG,YAAY,KAAK;AACjC,QAAM,IAAI,IAAI,KAAK,IAAI;AACvB,SAAO,MAAM,EAAE,QAAQ,CAAC,IAAI,SAAY,EAAE,YAAY;AACxD;AAEA,SAAS,qBAAqB,SAAqC;AACjE,QAAM,QAAQ,QAAQ,KAAK,EAAE,MAAM,oBAAoB;AACvD,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC;AAC3B,SAAO,MAAM,EAAE,QAAQ,CAAC,IAAI,SAAY,EAAE,YAAY;AACxD;AAEO,MAAM,gBAAwC;AAAA,EACnD,OAAO;AAAA,EAEP,MAAM,OAAO,OAAe,UAA+B,CAAC,GAA+B;AACzF,UAAM,YAAY,QAAQ,aAAa;AACvC,UAAM,aAAa,QAAQ,cAAc;AAEzC,UAAM,SAAS,IAAI,gBAAgB,EAAE,OAAO,KAAK,MAAM,CAAC;AACxD,UAAM,MAAM,uCAAuC,MAAM;AAEzD,QAAI,MAAM,sBAAsB,EAAE,MAAM,CAAC;AAEzC,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc;AAAA,QACd,gBAAgB;AAAA,MAClB;AAAA,MACA,MAAM,OAAO,SAAS;AAAA,MACtB,QAAQ,YAAY,QAAQ,SAAS;AAAA,IACvC,CAAC;AAED,QAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,sBAAsB,SAAS,MAAM,EAAE;AAEzE,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,WAAO,KAAK,aAAa,MAAM,UAAU;AAAA,EAC3C;AAAA,EAEA,aAAa,MAAc,YAAuC;AAChE,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAA6B,CAAC;AAEpC,UAAM,QAAQ,SAAS,iBAAiB,eAAe;AACvD,UAAM,QAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU;AAE/C,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,OAAO,MAAM,CAAC;AACpB,YAAM,OAAO,KAAK,cAAc,yCAAyC;AACzE,YAAM,YAAY,KAAK,cAAc,oBAAoB;AAEzD,YAAM,OAAO,MAAM,aAAa,MAAM;AACtC,YAAM,QAAQ,MAAM,aAAa,KAAK;AAEtC,UAAI,QAAQ,OAAO;AAEjB,cAAM,SAAS,KAAK,cAAc,0BAA0B;AAC5D,cAAM,cAAc,WAAW,aAAa,KAAK,KAAK;AACtD,cAAM,iBAAiB,gBAAgB,MAAM,KAAK,qBAAqB,WAAW;AAElF,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA,KAAK;AAAA,UACL,SAAS;AAAA,UACT,iBAAiB,IAAI,IAAI,KAAK,IAAI,MAAM,QAAQ,CAAC;AAAA,UACjD,QAAQ;AAAA,UACR,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,QAC7C,CAAC;AAAA,MACH;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}
|