@staticn0va/wigolo 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +146 -227
- package/SKILL.md +382 -0
- package/assets/blocks/claude-code/CLAUDE.md.block +20 -0
- package/assets/blocks/claude-code/wigolo-command.md +40 -0
- package/assets/blocks/cursor/wigolo.mdc +46 -0
- package/assets/blocks/gemini-cli/GEMINI.md.block +18 -0
- package/assets/blocks/vscode/copilot-instructions.md.block +18 -0
- package/assets/skills/wigolo/SKILL.md +50 -0
- package/assets/skills/wigolo/rules/cache-first.md +30 -0
- package/assets/skills/wigolo/rules/synthesis.md +43 -0
- package/assets/skills/wigolo-agent/SKILL.md +73 -0
- package/assets/skills/wigolo-crawl/SKILL.md +60 -0
- package/assets/skills/wigolo-extract/SKILL.md +59 -0
- package/assets/skills/wigolo-fetch/SKILL.md +65 -0
- package/assets/skills/wigolo-find-similar/SKILL.md +72 -0
- package/assets/skills/wigolo-research/SKILL.md +77 -0
- package/assets/skills/wigolo-search/SKILL.md +78 -0
- package/dist/agent/executor.d.ts +33 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +233 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +238 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +13 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +271 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/agent/relevance.d.ts +15 -0
- package/dist/agent/relevance.d.ts.map +1 -0
- package/dist/agent/relevance.js +60 -0
- package/dist/agent/relevance.js.map +1 -0
- package/dist/cache/backfill-embeddings.d.ts +23 -0
- package/dist/cache/backfill-embeddings.d.ts.map +1 -0
- package/dist/cache/backfill-embeddings.js +105 -0
- package/dist/cache/backfill-embeddings.js.map +1 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/db.d.ts +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +94 -22
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +82 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/migrations/runner.d.ts +29 -0
- package/dist/cache/migrations/runner.d.ts.map +1 -0
- package/dist/cache/migrations/runner.js +147 -0
- package/dist/cache/migrations/runner.js.map +1 -0
- package/dist/cache/sqlite-vec-store.d.ts +42 -0
- package/dist/cache/sqlite-vec-store.d.ts.map +1 -0
- package/dist/cache/sqlite-vec-store.js +176 -0
- package/dist/cache/sqlite-vec-store.js.map +1 -0
- package/dist/cache/store.d.ts +47 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +364 -168
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/agents/antigravity.d.ts +20 -0
- package/dist/cli/agents/antigravity.d.ts.map +1 -0
- package/dist/cli/agents/antigravity.js +49 -0
- package/dist/cli/agents/antigravity.js.map +1 -0
- package/dist/cli/agents/claude-code.d.ts +25 -0
- package/dist/cli/agents/claude-code.d.ts.map +1 -0
- package/dist/cli/agents/claude-code.js +111 -0
- package/dist/cli/agents/claude-code.js.map +1 -0
- package/dist/cli/agents/cursor.d.ts +21 -0
- package/dist/cli/agents/cursor.d.ts.map +1 -0
- package/dist/cli/agents/cursor.js +58 -0
- package/dist/cli/agents/cursor.js.map +1 -0
- package/dist/cli/agents/gemini-cli.d.ts +21 -0
- package/dist/cli/agents/gemini-cli.d.ts.map +1 -0
- package/dist/cli/agents/gemini-cli.js +55 -0
- package/dist/cli/agents/gemini-cli.js.map +1 -0
- package/dist/cli/agents/registry.d.ts +21 -0
- package/dist/cli/agents/registry.d.ts.map +1 -0
- package/dist/cli/agents/registry.js +27 -0
- package/dist/cli/agents/registry.js.map +1 -0
- package/dist/cli/agents/utils.d.ts +26 -0
- package/dist/cli/agents/utils.d.ts.map +1 -0
- package/dist/cli/agents/utils.js +136 -0
- package/dist/cli/agents/utils.js.map +1 -0
- package/dist/cli/agents/vscode.d.ts +21 -0
- package/dist/cli/agents/vscode.d.ts.map +1 -0
- package/dist/cli/agents/vscode.js +62 -0
- package/dist/cli/agents/vscode.js.map +1 -0
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +94 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/backfill.d.ts +2 -0
- package/dist/cli/backfill.d.ts.map +1 -0
- package/dist/cli/backfill.js +58 -0
- package/dist/cli/backfill.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +61 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/doctor.d.ts +8 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +344 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +42 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/help.d.ts +6 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +63 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +35 -7
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +201 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +185 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/setup-mcp.d.ts +2 -0
- package/dist/cli/setup-mcp.d.ts.map +1 -0
- package/dist/cli/setup-mcp.js +114 -0
- package/dist/cli/setup-mcp.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/shutdown.d.ts +2 -0
- package/dist/cli/shutdown.d.ts.map +1 -0
- package/dist/cli/shutdown.js +26 -0
- package/dist/cli/shutdown.js.map +1 -0
- package/dist/cli/status.d.ts +2 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +31 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/telemetry.d.ts +10 -0
- package/dist/cli/telemetry.d.ts.map +1 -0
- package/dist/cli/telemetry.js +56 -0
- package/dist/cli/telemetry.js.map +1 -0
- package/dist/cli/tui/agents-types.d.ts +28 -0
- package/dist/cli/tui/agents-types.d.ts.map +1 -0
- package/dist/cli/tui/agents-types.js +1 -0
- package/dist/cli/tui/agents-types.js.map +1 -0
- package/dist/cli/tui/agents.d.ts +11 -0
- package/dist/cli/tui/agents.d.ts.map +1 -0
- package/dist/cli/tui/agents.js +93 -0
- package/dist/cli/tui/agents.js.map +1 -0
- package/dist/cli/tui/banner.d.ts +3 -0
- package/dist/cli/tui/banner.d.ts.map +1 -0
- package/dist/cli/tui/banner.js +30 -0
- package/dist/cli/tui/banner.js.map +1 -0
- package/dist/cli/tui/components/AgentSelect.d.ts +13 -0
- package/dist/cli/tui/components/AgentSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/AgentSelect.js +116 -0
- package/dist/cli/tui/components/AgentSelect.js.map +1 -0
- package/dist/cli/tui/components/Banner.d.ts +6 -0
- package/dist/cli/tui/components/Banner.d.ts.map +1 -0
- package/dist/cli/tui/components/Banner.js +25 -0
- package/dist/cli/tui/components/Banner.js.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts +7 -0
- package/dist/cli/tui/components/BrowserSelect.d.ts.map +1 -0
- package/dist/cli/tui/components/BrowserSelect.js +19 -0
- package/dist/cli/tui/components/BrowserSelect.js.map +1 -0
- package/dist/cli/tui/components/InstallProgress.d.ts +9 -0
- package/dist/cli/tui/components/InstallProgress.d.ts.map +1 -0
- package/dist/cli/tui/components/InstallProgress.js +67 -0
- package/dist/cli/tui/components/InstallProgress.js.map +1 -0
- package/dist/cli/tui/components/SkillInstall.d.ts +14 -0
- package/dist/cli/tui/components/SkillInstall.d.ts.map +1 -0
- package/dist/cli/tui/components/SkillInstall.js +94 -0
- package/dist/cli/tui/components/SkillInstall.js.map +1 -0
- package/dist/cli/tui/components/Summary.d.ts +22 -0
- package/dist/cli/tui/components/Summary.d.ts.map +1 -0
- package/dist/cli/tui/components/Summary.js +135 -0
- package/dist/cli/tui/components/Summary.js.map +1 -0
- package/dist/cli/tui/components/SystemCheck.d.ts +8 -0
- package/dist/cli/tui/components/SystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/components/SystemCheck.js +71 -0
- package/dist/cli/tui/components/SystemCheck.js.map +1 -0
- package/dist/cli/tui/components/Verification.d.ts +8 -0
- package/dist/cli/tui/components/Verification.d.ts.map +1 -0
- package/dist/cli/tui/components/Verification.js +63 -0
- package/dist/cli/tui/components/Verification.js.map +1 -0
- package/dist/cli/tui/config-writer-cli.d.ts +12 -0
- package/dist/cli/tui/config-writer-cli.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-cli.js +39 -0
- package/dist/cli/tui/config-writer-cli.js.map +1 -0
- package/dist/cli/tui/config-writer-json.d.ts +16 -0
- package/dist/cli/tui/config-writer-json.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-json.js +86 -0
- package/dist/cli/tui/config-writer-json.js.map +1 -0
- package/dist/cli/tui/config-writer-toml.d.ts +16 -0
- package/dist/cli/tui/config-writer-toml.d.ts.map +1 -0
- package/dist/cli/tui/config-writer-toml.js +83 -0
- package/dist/cli/tui/config-writer-toml.js.map +1 -0
- package/dist/cli/tui/config-writer.d.ts +25 -0
- package/dist/cli/tui/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/config-writer.js +101 -0
- package/dist/cli/tui/config-writer.js.map +1 -0
- package/dist/cli/tui/detect-helpers.d.ts +6 -0
- package/dist/cli/tui/detect-helpers.d.ts.map +1 -0
- package/dist/cli/tui/detect-helpers.js +45 -0
- package/dist/cli/tui/detect-helpers.js.map +1 -0
- package/dist/cli/tui/extras-prompt.d.ts +7 -0
- package/dist/cli/tui/extras-prompt.d.ts.map +1 -0
- package/dist/cli/tui/extras-prompt.js +42 -0
- package/dist/cli/tui/extras-prompt.js.map +1 -0
- package/dist/cli/tui/flags-types.d.ts +19 -0
- package/dist/cli/tui/flags-types.d.ts.map +1 -0
- package/dist/cli/tui/flags-types.js +23 -0
- package/dist/cli/tui/flags-types.js.map +1 -0
- package/dist/cli/tui/flags.d.ts +5 -0
- package/dist/cli/tui/flags.d.ts.map +1 -0
- package/dist/cli/tui/flags.js +132 -0
- package/dist/cli/tui/flags.js.map +1 -0
- package/dist/cli/tui/format.d.ts +14 -0
- package/dist/cli/tui/format.d.ts.map +1 -0
- package/dist/cli/tui/format.js +37 -0
- package/dist/cli/tui/format.js.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts +6 -0
- package/dist/cli/tui/hooks/useAgentDetect.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useAgentDetect.js +19 -0
- package/dist/cli/tui/hooks/useAgentDetect.js.map +1 -0
- package/dist/cli/tui/hooks/useInstall.d.ts +14 -0
- package/dist/cli/tui/hooks/useInstall.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useInstall.js +90 -0
- package/dist/cli/tui/hooks/useInstall.js.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts +13 -0
- package/dist/cli/tui/hooks/useSystemCheck.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useSystemCheck.js +95 -0
- package/dist/cli/tui/hooks/useSystemCheck.js.map +1 -0
- package/dist/cli/tui/hooks/useVerify.d.ts +14 -0
- package/dist/cli/tui/hooks/useVerify.d.ts.map +1 -0
- package/dist/cli/tui/hooks/useVerify.js +71 -0
- package/dist/cli/tui/hooks/useVerify.js.map +1 -0
- package/dist/cli/tui/ink-init.d.ts +2 -0
- package/dist/cli/tui/ink-init.d.ts.map +1 -0
- package/dist/cli/tui/ink-init.js +198 -0
- package/dist/cli/tui/ink-init.js.map +1 -0
- package/dist/cli/tui/reporter-auto.d.ts +7 -0
- package/dist/cli/tui/reporter-auto.d.ts.map +1 -0
- package/dist/cli/tui/reporter-auto.js +15 -0
- package/dist/cli/tui/reporter-auto.js.map +1 -0
- package/dist/cli/tui/reporter.d.ts +26 -0
- package/dist/cli/tui/reporter.d.ts.map +1 -0
- package/dist/cli/tui/reporter.js +32 -0
- package/dist/cli/tui/reporter.js.map +1 -0
- package/dist/cli/tui/run-command.d.ts +14 -0
- package/dist/cli/tui/run-command.d.ts.map +1 -0
- package/dist/cli/tui/run-command.js +72 -0
- package/dist/cli/tui/run-command.js.map +1 -0
- package/dist/cli/tui/select-agents.d.ts +6 -0
- package/dist/cli/tui/select-agents.d.ts.map +1 -0
- package/dist/cli/tui/select-agents.js +32 -0
- package/dist/cli/tui/select-agents.js.map +1 -0
- package/dist/cli/tui/status-agents.d.ts +11 -0
- package/dist/cli/tui/status-agents.d.ts.map +1 -0
- package/dist/cli/tui/status-agents.js +53 -0
- package/dist/cli/tui/status-agents.js.map +1 -0
- package/dist/cli/tui/status-cache.d.ts +6 -0
- package/dist/cli/tui/status-cache.d.ts.map +1 -0
- package/dist/cli/tui/status-cache.js +39 -0
- package/dist/cli/tui/status-cache.js.map +1 -0
- package/dist/cli/tui/status-format.d.ts +14 -0
- package/dist/cli/tui/status-format.d.ts.map +1 -0
- package/dist/cli/tui/status-format.js +41 -0
- package/dist/cli/tui/status-format.js.map +1 -0
- package/dist/cli/tui/status-python.d.ts +6 -0
- package/dist/cli/tui/status-python.d.ts.map +1 -0
- package/dist/cli/tui/status-python.js +30 -0
- package/dist/cli/tui/status-python.js.map +1 -0
- package/dist/cli/tui/system-check.d.ts +24 -0
- package/dist/cli/tui/system-check.d.ts.map +1 -0
- package/dist/cli/tui/system-check.js +103 -0
- package/dist/cli/tui/system-check.js.map +1 -0
- package/dist/cli/tui/tui-reporter.d.ts +19 -0
- package/dist/cli/tui/tui-reporter.d.ts.map +1 -0
- package/dist/cli/tui/tui-reporter.js +95 -0
- package/dist/cli/tui/tui-reporter.js.map +1 -0
- package/dist/cli/tui/utils/config-writer.d.ts +3 -0
- package/dist/cli/tui/utils/config-writer.d.ts.map +1 -0
- package/dist/cli/tui/utils/config-writer.js +22 -0
- package/dist/cli/tui/utils/config-writer.js.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts +3 -0
- package/dist/cli/tui/utils/suppress-logs.d.ts.map +1 -0
- package/dist/cli/tui/utils/suppress-logs.js +11 -0
- package/dist/cli/tui/utils/suppress-logs.js.map +1 -0
- package/dist/cli/tui/verify-suggestions.d.ts +5 -0
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -0
- package/dist/cli/tui/verify-suggestions.js +20 -0
- package/dist/cli/tui/verify-suggestions.js.map +1 -0
- package/dist/cli/tui/verify.d.ts +14 -0
- package/dist/cli/tui/verify.d.ts.map +1 -0
- package/dist/cli/tui/verify.js +101 -0
- package/dist/cli/tui/verify.js.map +1 -0
- package/dist/cli/tui/version.d.ts +2 -0
- package/dist/cli/tui/version.d.ts.map +1 -0
- package/dist/cli/tui/version.js +14 -0
- package/dist/cli/tui/version.js.map +1 -0
- package/dist/cli/uninstall.d.ts +2 -0
- package/dist/cli/uninstall.d.ts.map +1 -0
- package/dist/cli/uninstall.js +57 -0
- package/dist/cli/uninstall.js.map +1 -0
- package/dist/cli/warmup.d.ts +10 -2
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +226 -93
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +28 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +106 -56
- package/dist/config.js.map +1 -1
- package/dist/crawl/crawler.d.ts +6 -0
- package/dist/crawl/crawler.d.ts.map +1 -1
- package/dist/crawl/crawler.js +210 -209
- package/dist/crawl/crawler.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +124 -81
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/crawl/etag-incremental.d.ts +43 -0
- package/dist/crawl/etag-incremental.d.ts.map +1 -0
- package/dist/crawl/etag-incremental.js +94 -0
- package/dist/crawl/etag-incremental.js.map +1 -0
- package/dist/crawl/index-to-vec.d.ts +10 -0
- package/dist/crawl/index-to-vec.d.ts.map +1 -0
- package/dist/crawl/index-to-vec.js +44 -0
- package/dist/crawl/index-to-vec.js.map +1 -0
- package/dist/crawl/mapper.js +136 -164
- package/dist/crawl/mapper.js.map +1 -1
- package/dist/crawl/rate-limiter.js +63 -66
- package/dist/crawl/rate-limiter.js.map +1 -1
- package/dist/crawl/robots.js +58 -57
- package/dist/crawl/robots.js.map +1 -1
- package/dist/crawl/sitemap-first.d.ts +12 -0
- package/dist/crawl/sitemap-first.d.ts.map +1 -0
- package/dist/crawl/sitemap-first.js +47 -0
- package/dist/crawl/sitemap-first.js.map +1 -0
- package/dist/crawl/sitemap.js +33 -32
- package/dist/crawl/sitemap.js.map +1 -1
- package/dist/crawl/url-utils.d.ts +1 -0
- package/dist/crawl/url-utils.d.ts.map +1 -1
- package/dist/crawl/url-utils.js +49 -37
- package/dist/crawl/url-utils.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +33 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +275 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +93 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +233 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/fastembed-provider.d.ts +19 -0
- package/dist/embedding/fastembed-provider.d.ts.map +1 -0
- package/dist/embedding/fastembed-provider.js +51 -0
- package/dist/embedding/fastembed-provider.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +234 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +52 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +27 -23
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/extract.d.ts.map +1 -1
- package/dist/extraction/extract.js +76 -76
- package/dist/extraction/extract.js.map +1 -1
- package/dist/extraction/jsonld.js +50 -54
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +30 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +130 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown-sanitize.d.ts +2 -0
- package/dist/extraction/markdown-sanitize.d.ts.map +1 -0
- package/dist/extraction/markdown-sanitize.js +151 -0
- package/dist/extraction/markdown-sanitize.js.map +1 -0
- package/dist/extraction/markdown.d.ts +11 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +195 -91
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts +8 -0
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +57 -91
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +28 -29
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +135 -72
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +81 -91
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +87 -95
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +46 -54
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +71 -80
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +173 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/extraction/structured.d.ts +4 -0
- package/dist/extraction/structured.d.ts.map +1 -0
- package/dist/extraction/structured.js +163 -0
- package/dist/extraction/structured.js.map +1 -0
- package/dist/extraction/v1/classifier.d.ts +3 -0
- package/dist/extraction/v1/classifier.d.ts.map +1 -0
- package/dist/extraction/v1/classifier.js +110 -0
- package/dist/extraction/v1/classifier.js.map +1 -0
- package/dist/extraction/v1/extract-provider.d.ts +16 -0
- package/dist/extraction/v1/extract-provider.d.ts.map +1 -0
- package/dist/extraction/v1/extract-provider.js +43 -0
- package/dist/extraction/v1/extract-provider.js.map +1 -0
- package/dist/extraction/v1/local-llm.d.ts +8 -0
- package/dist/extraction/v1/local-llm.d.ts.map +1 -0
- package/dist/extraction/v1/local-llm.js +34 -0
- package/dist/extraction/v1/local-llm.js.map +1 -0
- package/dist/extraction/v1/news.d.ts +3 -0
- package/dist/extraction/v1/news.d.ts.map +1 -0
- package/dist/extraction/v1/news.js +61 -0
- package/dist/extraction/v1/news.js.map +1 -0
- package/dist/extraction/v1/product.d.ts +3 -0
- package/dist/extraction/v1/product.d.ts.map +1 -0
- package/dist/extraction/v1/product.js +166 -0
- package/dist/extraction/v1/product.js.map +1 -0
- package/dist/extraction/v1/recipe.d.ts +3 -0
- package/dist/extraction/v1/recipe.d.ts.map +1 -0
- package/dist/extraction/v1/recipe.js +136 -0
- package/dist/extraction/v1/recipe.js.map +1 -0
- package/dist/extraction/v1/routed.d.ts +17 -0
- package/dist/extraction/v1/routed.d.ts.map +1 -0
- package/dist/extraction/v1/routed.js +68 -0
- package/dist/extraction/v1/routed.js.map +1 -0
- package/dist/extraction/v1/schemas/Article.d.ts +11 -0
- package/dist/extraction/v1/schemas/Article.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Article.js +23 -0
- package/dist/extraction/v1/schemas/Article.js.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts +9 -0
- package/dist/extraction/v1/schemas/CodeSnippet.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js +90 -0
- package/dist/extraction/v1/schemas/CodeSnippet.js.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts +10 -0
- package/dist/extraction/v1/schemas/EventListing.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/EventListing.js +122 -0
- package/dist/extraction/v1/schemas/EventListing.js.map +1 -0
- package/dist/extraction/v1/schemas/Paper.d.ts +10 -0
- package/dist/extraction/v1/schemas/Paper.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Paper.js +156 -0
- package/dist/extraction/v1/schemas/Paper.js.map +1 -0
- package/dist/extraction/v1/schemas/Product.d.ts +17 -0
- package/dist/extraction/v1/schemas/Product.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Product.js +149 -0
- package/dist/extraction/v1/schemas/Product.js.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts +14 -0
- package/dist/extraction/v1/schemas/Recipe.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/Recipe.js +160 -0
- package/dist/extraction/v1/schemas/Recipe.js.map +1 -0
- package/dist/extraction/v1/schemas/index.d.ts +13 -0
- package/dist/extraction/v1/schemas/index.d.ts.map +1 -0
- package/dist/extraction/v1/schemas/index.js +44 -0
- package/dist/extraction/v1/schemas/index.js.map +1 -0
- package/dist/extraction/v1/site-extractors.d.ts +5 -0
- package/dist/extraction/v1/site-extractors.d.ts.map +1 -0
- package/dist/extraction/v1/site-extractors.js +31 -0
- package/dist/extraction/v1/site-extractors.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +88 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +56 -26
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +303 -127
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +72 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +89 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/content-check.js +39 -46
- package/dist/fetch/content-check.js.map +1 -1
- package/dist/fetch/error-describe.d.ts +7 -0
- package/dist/fetch/error-describe.d.ts.map +1 -0
- package/dist/fetch/error-describe.js +37 -0
- package/dist/fetch/error-describe.js.map +1 -0
- package/dist/fetch/http-client.d.ts +4 -0
- package/dist/fetch/http-client.d.ts.map +1 -1
- package/dist/fetch/http-client.js +147 -128
- package/dist/fetch/http-client.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +174 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/fetch/playwright-tier.d.ts +19 -0
- package/dist/fetch/playwright-tier.d.ts.map +1 -0
- package/dist/fetch/playwright-tier.js +76 -0
- package/dist/fetch/playwright-tier.js.map +1 -0
- package/dist/fetch/router.d.ts +49 -3
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +187 -81
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +102 -17
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +31 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +245 -0
- package/dist/instructions.js.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts +3 -0
- package/dist/integrations/cloud/llm/anthropic.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/anthropic.js +41 -0
- package/dist/integrations/cloud/llm/anthropic.js.map +1 -0
- package/dist/integrations/cloud/llm/cache.d.ts +5 -0
- package/dist/integrations/cloud/llm/cache.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/cache.js +49 -0
- package/dist/integrations/cloud/llm/cache.js.map +1 -0
- package/dist/integrations/cloud/llm/gemini.d.ts +3 -0
- package/dist/integrations/cloud/llm/gemini.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/gemini.js +37 -0
- package/dist/integrations/cloud/llm/gemini.js.map +1 -0
- package/dist/integrations/cloud/llm/groq.d.ts +3 -0
- package/dist/integrations/cloud/llm/groq.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/groq.js +74 -0
- package/dist/integrations/cloud/llm/groq.js.map +1 -0
- package/dist/integrations/cloud/llm/hash.d.ts +3 -0
- package/dist/integrations/cloud/llm/hash.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/hash.js +26 -0
- package/dist/integrations/cloud/llm/hash.js.map +1 -0
- package/dist/integrations/cloud/llm/model-select.d.ts +5 -0
- package/dist/integrations/cloud/llm/model-select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/model-select.js +32 -0
- package/dist/integrations/cloud/llm/model-select.js.map +1 -0
- package/dist/integrations/cloud/llm/openai.d.ts +3 -0
- package/dist/integrations/cloud/llm/openai.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/openai.js +43 -0
- package/dist/integrations/cloud/llm/openai.js.map +1 -0
- package/dist/integrations/cloud/llm/run.d.ts +27 -0
- package/dist/integrations/cloud/llm/run.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/run.js +99 -0
- package/dist/integrations/cloud/llm/run.js.map +1 -0
- package/dist/integrations/cloud/llm/select.d.ts +5 -0
- package/dist/integrations/cloud/llm/select.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/select.js +30 -0
- package/dist/integrations/cloud/llm/select.js.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts +19 -0
- package/dist/integrations/cloud/llm/text-adapters.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/text-adapters.js +103 -0
- package/dist/integrations/cloud/llm/text-adapters.js.map +1 -0
- package/dist/integrations/cloud/llm/types.d.ts +24 -0
- package/dist/integrations/cloud/llm/types.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/types.js +1 -0
- package/dist/integrations/cloud/llm/types.js.map +1 -0
- package/dist/integrations/cloud/llm/validate.d.ts +6 -0
- package/dist/integrations/cloud/llm/validate.d.ts.map +1 -0
- package/dist/integrations/cloud/llm/validate.js +63 -0
- package/dist/integrations/cloud/llm/validate.js.map +1 -0
- package/dist/logger.d.ts +4 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +71 -30
- package/dist/logger.js.map +1 -1
- package/dist/pdf-parse.d.js +1 -0
- package/dist/pdf-parse.d.js.map +1 -0
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +157 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +71 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +79 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/providers/embed-provider.d.ts +11 -0
- package/dist/providers/embed-provider.d.ts.map +1 -0
- package/dist/providers/embed-provider.js +24 -0
- package/dist/providers/embed-provider.js.map +1 -0
- package/dist/providers/extract-provider.d.ts +23 -0
- package/dist/providers/extract-provider.d.ts.map +1 -0
- package/dist/providers/extract-provider.js +25 -0
- package/dist/providers/extract-provider.js.map +1 -0
- package/dist/providers/rerank-provider.d.ts +17 -0
- package/dist/providers/rerank-provider.d.ts.map +1 -0
- package/dist/providers/rerank-provider.js +41 -0
- package/dist/providers/rerank-provider.js.map +1 -0
- package/dist/providers/search-provider.d.ts +25 -0
- package/dist/providers/search-provider.d.ts.map +1 -0
- package/dist/providers/search-provider.js +44 -0
- package/dist/providers/search-provider.js.map +1 -0
- package/dist/providers/vector-store.d.ts +27 -0
- package/dist/providers/vector-store.d.ts.map +1 -0
- package/dist/providers/vector-store.js +27 -0
- package/dist/providers/vector-store.js.map +1 -0
- package/dist/python-env.d.ts +9 -0
- package/dist/python-env.d.ts.map +1 -0
- package/dist/python-env.js +13 -0
- package/dist/python-env.js.map +1 -0
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +62 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +43 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +44 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +47 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +67 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +74 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +65 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +74 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +1 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +13 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +283 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +86 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +184 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/research/branch-exploration.d.ts +14 -0
- package/dist/research/branch-exploration.d.ts.map +1 -0
- package/dist/research/branch-exploration.js +100 -0
- package/dist/research/branch-exploration.js.map +1 -0
- package/dist/research/brief.d.ts +6 -0
- package/dist/research/brief.d.ts.map +1 -0
- package/dist/research/brief.js +246 -0
- package/dist/research/brief.js.map +1 -0
- package/dist/research/citation-graph.d.ts +9 -0
- package/dist/research/citation-graph.d.ts.map +1 -0
- package/dist/research/citation-graph.js +114 -0
- package/dist/research/citation-graph.js.map +1 -0
- package/dist/research/decompose.d.ts +14 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +439 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +269 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesis-local.d.ts +19 -0
- package/dist/research/synthesis-local.d.ts.map +1 -0
- package/dist/research/synthesis-local.js +62 -0
- package/dist/research/synthesis-local.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +137 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +33 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +244 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/search/dedup.d.ts +1 -0
- package/dist/search/dedup.d.ts.map +1 -1
- package/dist/search/dedup.js +40 -32
- package/dist/search/dedup.js.map +1 -1
- package/dist/search/engines/arxiv.d.ts +7 -0
- package/dist/search/engines/arxiv.d.ts.map +1 -0
- package/dist/search/engines/arxiv.js +70 -0
- package/dist/search/engines/arxiv.js.map +1 -0
- package/dist/search/engines/bing-news.d.ts +7 -0
- package/dist/search/engines/bing-news.d.ts.map +1 -0
- package/dist/search/engines/bing-news.js +97 -0
- package/dist/search/engines/bing-news.js.map +1 -0
- package/dist/search/engines/bing.d.ts +1 -0
- package/dist/search/engines/bing.d.ts.map +1 -1
- package/dist/search/engines/bing.js +100 -44
- package/dist/search/engines/bing.js.map +1 -1
- package/dist/search/engines/devdocs.d.ts +6 -0
- package/dist/search/engines/devdocs.d.ts.map +1 -0
- package/dist/search/engines/devdocs.js +56 -0
- package/dist/search/engines/devdocs.js.map +1 -0
- package/dist/search/engines/duckduckgo.d.ts.map +1 -1
- package/dist/search/engines/duckduckgo.js +56 -44
- package/dist/search/engines/duckduckgo.js.map +1 -1
- package/dist/search/engines/github-code.d.ts +7 -0
- package/dist/search/engines/github-code.d.ts.map +1 -0
- package/dist/search/engines/github-code.js +55 -0
- package/dist/search/engines/github-code.js.map +1 -0
- package/dist/search/engines/hn-algolia.d.ts +7 -0
- package/dist/search/engines/hn-algolia.d.ts.map +1 -0
- package/dist/search/engines/hn-algolia.js +76 -0
- package/dist/search/engines/hn-algolia.js.map +1 -0
- package/dist/search/engines/lobsters.d.ts +7 -0
- package/dist/search/engines/lobsters.d.ts.map +1 -0
- package/dist/search/engines/lobsters.js +83 -0
- package/dist/search/engines/lobsters.js.map +1 -0
- package/dist/search/engines/mdn.d.ts +7 -0
- package/dist/search/engines/mdn.d.ts.map +1 -0
- package/dist/search/engines/mdn.js +48 -0
- package/dist/search/engines/mdn.js.map +1 -0
- package/dist/search/engines/semantic-scholar.d.ts +7 -0
- package/dist/search/engines/semantic-scholar.d.ts.map +1 -0
- package/dist/search/engines/semantic-scholar.js +69 -0
- package/dist/search/engines/semantic-scholar.js.map +1 -0
- package/dist/search/engines/stackoverflow.d.ts +7 -0
- package/dist/search/engines/stackoverflow.d.ts.map +1 -0
- package/dist/search/engines/stackoverflow.js +73 -0
- package/dist/search/engines/stackoverflow.js.map +1 -0
- package/dist/search/engines/startpage.d.ts.map +1 -1
- package/dist/search/engines/startpage.js +65 -46
- package/dist/search/engines/startpage.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +220 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/filters.d.ts.map +1 -1
- package/dist/search/filters.js +58 -54
- package/dist/search/filters.js.map +1 -1
- package/dist/search/find-similar/crawl-rank.d.ts +9 -0
- package/dist/search/find-similar/crawl-rank.d.ts.map +1 -0
- package/dist/search/find-similar/crawl-rank.js +272 -0
- package/dist/search/find-similar/crawl-rank.js.map +1 -0
- package/dist/search/find-similar/mode.d.ts +4 -0
- package/dist/search/find-similar/mode.d.ts.map +1 -0
- package/dist/search/find-similar/mode.js +12 -0
- package/dist/search/find-similar/mode.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +509 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/highlights.d.ts +19 -0
- package/dist/search/highlights.d.ts.map +1 -0
- package/dist/search/highlights.js +167 -0
- package/dist/search/highlights.js.map +1 -0
- package/dist/search/language-filter.d.ts +29 -0
- package/dist/search/language-filter.d.ts.map +1 -0
- package/dist/search/language-filter.js +126 -0
- package/dist/search/language-filter.js.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts +4 -0
- package/dist/search/legacy/searxng-orchestrator.d.ts.map +1 -0
- package/dist/search/legacy/searxng-orchestrator.js +501 -0
- package/dist/search/legacy/searxng-orchestrator.js.map +1 -0
- package/dist/search/legacy/searxng-provider.d.ts +7 -0
- package/dist/search/legacy/searxng-provider.d.ts.map +1 -0
- package/dist/search/legacy/searxng-provider.js +11 -0
- package/dist/search/legacy/searxng-provider.js.map +1 -0
- package/dist/search/multi-query.d.ts +25 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +228 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/query.js +32 -34
- package/dist/search/query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -1
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +44 -35
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/authority-boost.d.ts +3 -0
- package/dist/search/reranker/authority-boost.d.ts.map +1 -0
- package/dist/search/reranker/authority-boost.js +179 -0
- package/dist/search/reranker/authority-boost.js.map +1 -0
- package/dist/search/reranker/consensus-boost.d.ts +3 -0
- package/dist/search/reranker/consensus-boost.d.ts.map +1 -0
- package/dist/search/reranker/consensus-boost.js +27 -0
- package/dist/search/reranker/consensus-boost.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +13 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +23 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts +13 -0
- package/dist/search/reranker/transformers-rerank-provider.d.ts.map +1 -0
- package/dist/search/reranker/transformers-rerank-provider.js +94 -0
- package/dist/search/reranker/transformers-rerank-provider.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +39 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +69 -79
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +39 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +6 -0
- package/dist/search/truncate.d.ts.map +1 -0
- package/dist/search/truncate.js +26 -0
- package/dist/search/truncate.js.map +1 -0
- package/dist/search/url-unwrap.d.ts +3 -0
- package/dist/search/url-unwrap.d.ts.map +1 -0
- package/dist/search/url-unwrap.js +43 -0
- package/dist/search/url-unwrap.js.map +1 -0
- package/dist/search/v1/context-rank.d.ts +13 -0
- package/dist/search/v1/context-rank.d.ts.map +1 -0
- package/dist/search/v1/context-rank.js +74 -0
- package/dist/search/v1/context-rank.js.map +1 -0
- package/dist/search/v1/engine-base.d.ts +27 -0
- package/dist/search/v1/engine-base.d.ts.map +1 -0
- package/dist/search/v1/engine-base.js +110 -0
- package/dist/search/v1/engine-base.js.map +1 -0
- package/dist/search/v1/intent-router.d.ts +22 -0
- package/dist/search/v1/intent-router.d.ts.map +1 -0
- package/dist/search/v1/intent-router.js +138 -0
- package/dist/search/v1/intent-router.js.map +1 -0
- package/dist/search/v1/orchestrator.d.ts +24 -0
- package/dist/search/v1/orchestrator.d.ts.map +1 -0
- package/dist/search/v1/orchestrator.js +163 -0
- package/dist/search/v1/orchestrator.js.map +1 -0
- package/dist/search/v1/recency-boost.d.ts +9 -0
- package/dist/search/v1/recency-boost.d.ts.map +1 -0
- package/dist/search/v1/recency-boost.js +37 -0
- package/dist/search/v1/recency-boost.js.map +1 -0
- package/dist/search/v1/recent-cache-dedup.d.ts +6 -0
- package/dist/search/v1/recent-cache-dedup.d.ts.map +1 -0
- package/dist/search/v1/recent-cache-dedup.js +85 -0
- package/dist/search/v1/recent-cache-dedup.js.map +1 -0
- package/dist/search/v1/rss/feed-config.d.ts +21 -0
- package/dist/search/v1/rss/feed-config.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-config.js +90 -0
- package/dist/search/v1/rss/feed-config.js.map +1 -0
- package/dist/search/v1/rss/feed-parser.d.ts +14 -0
- package/dist/search/v1/rss/feed-parser.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-parser.js +104 -0
- package/dist/search/v1/rss/feed-parser.js.map +1 -0
- package/dist/search/v1/rss/feed-poller.d.ts +22 -0
- package/dist/search/v1/rss/feed-poller.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-poller.js +102 -0
- package/dist/search/v1/rss/feed-poller.js.map +1 -0
- package/dist/search/v1/rss/feed-store.d.ts +30 -0
- package/dist/search/v1/rss/feed-store.d.ts.map +1 -0
- package/dist/search/v1/rss/feed-store.js +134 -0
- package/dist/search/v1/rss/feed-store.js.map +1 -0
- package/dist/search/v1/rss/rss-engine.d.ts +6 -0
- package/dist/search/v1/rss/rss-engine.d.ts.map +1 -0
- package/dist/search/v1/rss/rss-engine.js +28 -0
- package/dist/search/v1/rss/rss-engine.js.map +1 -0
- package/dist/search/v1/v1-provider.d.ts +7 -0
- package/dist/search/v1/v1-provider.d.ts.map +1 -0
- package/dist/search/v1/v1-provider.js +68 -0
- package/dist/search/v1/v1-provider.js.map +1 -0
- package/dist/search/v1/verticals/code.d.ts +4 -0
- package/dist/search/v1/verticals/code.d.ts.map +1 -0
- package/dist/search/v1/verticals/code.js +20 -0
- package/dist/search/v1/verticals/code.js.map +1 -0
- package/dist/search/v1/verticals/docs.d.ts +4 -0
- package/dist/search/v1/verticals/docs.d.ts.map +1 -0
- package/dist/search/v1/verticals/docs.js +20 -0
- package/dist/search/v1/verticals/docs.js.map +1 -0
- package/dist/search/v1/verticals/general.d.ts +4 -0
- package/dist/search/v1/verticals/general.d.ts.map +1 -0
- package/dist/search/v1/verticals/general.js +22 -0
- package/dist/search/v1/verticals/general.js.map +1 -0
- package/dist/search/v1/verticals/news.d.ts +10 -0
- package/dist/search/v1/verticals/news.d.ts.map +1 -0
- package/dist/search/v1/verticals/news.js +52 -0
- package/dist/search/v1/verticals/news.js.map +1 -0
- package/dist/search/v1/verticals/papers.d.ts +4 -0
- package/dist/search/v1/verticals/papers.d.ts.map +1 -0
- package/dist/search/v1/verticals/papers.js +23 -0
- package/dist/search/v1/verticals/papers.js.map +1 -0
- package/dist/search/validator.js +31 -31
- package/dist/search/validator.js.map +1 -1
- package/dist/searxng/bootstrap.d.ts +30 -0
- package/dist/searxng/bootstrap.d.ts.map +1 -1
- package/dist/searxng/bootstrap.js +223 -85
- package/dist/searxng/bootstrap.js.map +1 -1
- package/dist/searxng/docker.d.ts.map +1 -1
- package/dist/searxng/docker.js +69 -60
- package/dist/searxng/docker.js.map +1 -1
- package/dist/searxng/process.d.ts +13 -1
- package/dist/searxng/process.d.ts.map +1 -1
- package/dist/searxng/process.js +231 -164
- package/dist/searxng/process.js.map +1 -1
- package/dist/server/backend-status.d.ts +13 -0
- package/dist/server/backend-status.d.ts.map +1 -0
- package/dist/server/backend-status.js +40 -0
- package/dist/server/backend-status.js.map +1 -0
- package/dist/server/tool-schemas.d.ts +549 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +464 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server/warmup-on-start.d.ts +9 -0
- package/dist/server/warmup-on-start.d.ts.map +1 -0
- package/dist/server/warmup-on-start.js +55 -0
- package/dist/server/warmup-on-start.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +454 -297
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +128 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +177 -44
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +171 -88
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts +2 -2
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +175 -59
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts +2 -2
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +174 -68
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +127 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +107 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +10 -2
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -158
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +350 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +6 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +34 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +78 -8
- package/dist/extraction/trafilatura.d.ts +0 -6
- package/dist/extraction/trafilatura.d.ts.map +0 -1
- package/dist/extraction/trafilatura.js +0 -105
- package/dist/extraction/trafilatura.js.map +0 -1
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -63
- package/dist/search/flashrank.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAiB5C,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACpD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExD,MAAM,mBAAmB,GAAG;IAC1B,uHAAuH;IACvH,iHAAiH;IACjH,uGAAuG;CACxG,CAAC;AAEF,SAAS,oBAAoB,CAAC,MAAqC;IACjE,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC;IAC9C,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,IAAI,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzD,iEAAiE;QACjE,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAA4B,EAAE;IACzE,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC;IAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAEzC,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACvF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,CAAC;YAEhB,IAAI,GAAG,YAAY,cAAc,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpD,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,YAAY,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAExF,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBACxC,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,OAAO;gBACP,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,cAAe,SAAQ,KAAK;IACa;IAA7C,YAAY,OAAe,EAAkB,SAAkB;QAC7D,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,cAAS,GAAT,SAAS,CAAS;QAE7D,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,KAAK,UAAU,kBAAkB,CAC/B,WAAmB,EACnB,OAAyB,EACzB,SAAiB,EACjB,YAAoB,EACpB,MAAuC;IAEvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,6BAA6B,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QAEtE,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/D,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACjC,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ;gBAClB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;YACtE,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,qBAAqB,CAAC,GAAG,CAAE,GAA6B,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC/G,MAAM,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC;YACzC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,cAAc,CAAC,uCAAuC,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;YAED,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,cAAc,CAAC,wBAAwB,YAAY,UAAU,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC/F,CAAC;YAED,6BAA6B;YAC7B,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,SAAS;QACX,CAAC;QAED,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,IAAY,CAAC;QACjB,IAAI,SAA6B,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC9C,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,UAAU;YACpB,IAAI;YACJ,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,OAAO;YACP,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"sources":["../../src/fetch/http-client.ts"],"sourcesContent":["import { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nexport interface HttpFetchOptions {\n headers?: Record<string, string>;\n timeoutMs?: number;\n conditionalHeaders?: {\n ifNoneMatch?: string;\n ifModifiedSince?: string;\n };\n}\n\nexport interface HttpFetchResult {\n url: string;\n finalUrl: string;\n html: string;\n contentType: string;\n statusCode: number;\n headers: Record<string, string>;\n rawBuffer?: Buffer;\n}\n\nconst RETRYABLE_STATUSES = new Set([429, 502, 503]);\nconst RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);\nconst REDIRECT_STATUSES = new Set([301, 302, 307, 308]);\n\nconst DEFAULT_USER_AGENTS = [\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n];\n\nfunction getRotatingUserAgent(config: { userAgent?: string | null }): string {\n if (config.userAgent) return config.userAgent;\n return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];\n}\n\nfunction isRetryableError(err: unknown): boolean {\n if (err instanceof Error) {\n const code = (err as NodeJS.ErrnoException).code;\n if (code && RETRYABLE_ERROR_CODES.has(code)) return true;\n // AbortSignal timeout throws DOMException with name TimeoutError\n if (err.name === 'TimeoutError') return true;\n }\n return false;\n}\n\nfunction backoffMs(attempt: number): number {\n return 500 * Math.pow(2, attempt) + Math.random() * 500;\n}\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport async function httpFetch(url: string, options: HttpFetchOptions = {}): Promise<HttpFetchResult> {\n const config = getConfig();\n const logger = createLogger('fetch');\n const maxRetries = config.fetchMaxRetries;\n const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;\n const maxRedirects = config.maxRedirects;\n\n let lastError: unknown;\n\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n if (attempt > 0) {\n const delay = backoffMs(attempt - 1);\n logger.debug('retrying after backoff', { attempt, delayMs: delay, url });\n await sleep(delay);\n }\n\n try {\n const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);\n return result;\n } catch (err) {\n lastError = err;\n\n if (err instanceof HttpFetchError && !err.retryable) {\n throw err;\n }\n\n const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);\n\n if (!retryable || attempt >= maxRetries) {\n throw err;\n }\n\n logger.warn('fetch failed, will retry', {\n attempt,\n url,\n error: err instanceof Error ? err.message : String(err),\n });\n }\n }\n\n throw lastError;\n}\n\nclass HttpFetchError extends Error {\n constructor(message: string, public readonly retryable: boolean) {\n super(message);\n this.name = 'HttpFetchError';\n }\n}\n\nasync function fetchWithRedirects(\n originalUrl: string,\n options: HttpFetchOptions,\n timeoutMs: number,\n maxRedirects: number,\n logger: ReturnType<typeof createLogger>,\n): Promise<HttpFetchResult> {\n const visited = new Set<string>();\n let currentUrl = originalUrl;\n let redirectCount = 0;\n\n while (true) {\n if (visited.has(currentUrl)) {\n throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);\n }\n visited.add(currentUrl);\n\n logger.debug('fetching', { url: currentUrl, attempt: redirectCount });\n\n const signal = AbortSignal.timeout(timeoutMs);\n\n let response: Response;\n try {\n const ua = getRotatingUserAgent(getConfig());\n const mergedHeaders: Record<string, string> = { 'User-Agent': ua, ...options.headers };\n // Conditional GET: inject If-None-Match / If-Modified-Since so the\n // server can return 304 + no body when the resource hasn't changed.\n // Callers (eg. etag-incremental crawl) wire these from the persisted\n // crawl_etags row for the URL.\n if (options.conditionalHeaders?.ifNoneMatch) {\n mergedHeaders['If-None-Match'] = options.conditionalHeaders.ifNoneMatch;\n }\n if (options.conditionalHeaders?.ifModifiedSince) {\n mergedHeaders['If-Modified-Since'] = options.conditionalHeaders.ifModifiedSince;\n }\n response = await fetch(currentUrl, {\n headers: mergedHeaders,\n redirect: 'manual',\n signal,\n });\n } catch (err) {\n const isTimeout = err instanceof Error && err.name === 'TimeoutError';\n const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has((err as NodeJS.ErrnoException).code ?? '');\n const retryable = isTimeout || isConnErr;\n throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });\n }\n\n if (response.status === 304) {\n const headers: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n headers[key] = value;\n });\n // Drain so the connection can be released; ignore the (empty) body.\n try { await response.arrayBuffer(); } catch { /* */ }\n return {\n url: originalUrl,\n finalUrl: currentUrl,\n html: '',\n contentType: response.headers.get('content-type') ?? '',\n statusCode: 304,\n headers,\n };\n }\n\n if (REDIRECT_STATUSES.has(response.status)) {\n const location = response.headers.get('location');\n if (!location) {\n throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);\n }\n\n redirectCount++;\n if (redirectCount > maxRedirects) {\n throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);\n }\n\n // Resolve relative redirects\n currentUrl = new URL(location, currentUrl).toString();\n continue;\n }\n\n if (RETRYABLE_STATUSES.has(response.status)) {\n throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);\n }\n\n const contentType = response.headers.get('content-type') ?? '';\n const headers: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n headers[key] = value;\n });\n\n const isPdf = contentType.includes('application/pdf');\n let html: string;\n let rawBuffer: Buffer | undefined;\n\n if (isPdf) {\n const arrayBuf = await response.arrayBuffer();\n rawBuffer = Buffer.from(arrayBuf);\n html = '';\n } else {\n html = await response.text();\n }\n\n return {\n url: originalUrl,\n finalUrl: currentUrl,\n html,\n contentType,\n statusCode: response.status,\n headers,\n rawBuffer,\n };\n }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAqB7B,MAAM,qBAAqB,oBAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC;AAClD,MAAM,wBAAwB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,CAAC;AACjF,MAAM,oBAAoB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAEtD,MAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AACF;AAEA,SAAS,qBAAqB,QAA+C;AAC3E,MAAI,OAAO,UAAW,QAAO,OAAO;AACpC,SAAO,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,oBAAoB,MAAM,CAAC;AACnF;AAEA,SAAS,iBAAiB,KAAuB;AAC/C,MAAI,eAAe,OAAO;AACxB,UAAM,OAAQ,IAA8B;AAC5C,QAAI,QAAQ,sBAAsB,IAAI,IAAI,EAAG,QAAO;AAEpD,QAAI,IAAI,SAAS,eAAgB,QAAO;AAAA,EAC1C;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAAyB;AAC1C,SAAO,MAAM,KAAK,IAAI,GAAG,OAAO,IAAI,KAAK,OAAO,IAAI;AACtD;AAEA,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AAEA,eAAsB,UAAU,KAAa,UAA4B,CAAC,GAA6B;AACrG,QAAM,SAAS,UAAU;AACzB,QAAM,SAAS,aAAa,OAAO;AACnC,QAAM,aAAa,OAAO;AAC1B,QAAM,YAAY,QAAQ,aAAa,OAAO;AAC9C,QAAM,eAAe,OAAO;AAE5B,MAAI;AAEJ,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,QAAI,UAAU,GAAG;AACf,YAAM,QAAQ,UAAU,UAAU,CAAC;AACnC,aAAO,MAAM,0BAA0B,EAAE,SAAS,SAAS,OAAO,IAAI,CAAC;AACvE,YAAM,MAAM,KAAK;AAAA,IACnB;AAEA,QAAI;AACF,YAAM,SAAS,MAAM,mBAAmB,KAAK,SAAS,WAAW,cAAc,MAAM;AACrF,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,kBAAY;AAEZ,UAAI,eAAe,kBAAkB,CAAC,IAAI,WAAW;AACnD,cAAM;AAAA,MACR;AAEA,YAAM,YAAY,eAAe,iBAAiB,IAAI,YAAY,iBAAiB,GAAG;AAEtF,UAAI,CAAC,aAAa,WAAW,YAAY;AACvC,cAAM;AAAA,MACR;AAEA,aAAO,KAAK,4BAA4B;AAAA,QACtC;AAAA,QACA;AAAA,QACA,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM;AACR;AAEA,MAAM,uBAAuB,MAAM;AAAA,EACjC,YAAY,SAAiC,WAAoB;AAC/D,UAAM,OAAO;AAD8B;AAE3C,SAAK,OAAO;AAAA,EACd;AAAA,EAH6C;AAI/C;AAEA,eAAe,mBACb,aACA,SACA,WACA,cACA,QAC0B;AAC1B,QAAM,UAAU,oBAAI,IAAY;AAChC,MAAI,aAAa;AACjB,MAAI,gBAAgB;AAEpB,SAAO,MAAM;AACX,QAAI,QAAQ,IAAI,UAAU,GAAG;AAC3B,YAAM,IAAI,eAAe,6BAA6B,UAAU,IAAI,KAAK;AAAA,IAC3E;AACA,YAAQ,IAAI,UAAU;AAEtB,WAAO,MAAM,YAAY,EAAE,KAAK,YAAY,SAAS,cAAc,CAAC;AAEpE,UAAM,SAAS,YAAY,QAAQ,SAAS;AAE5C,QAAI;AACJ,QAAI;AACF,YAAM,KAAK,qBAAqB,UAAU,CAAC;AAC3C,YAAM,gBAAwC,EAAE,cAAc,IAAI,GAAG,QAAQ,QAAQ;AAKrF,UAAI,QAAQ,oBAAoB,aAAa;AAC3C,sBAAc,eAAe,IAAI,QAAQ,mBAAmB;AAAA,MAC9D;AACA,UAAI,QAAQ,oBAAoB,iBAAiB;AAC/C,sBAAc,mBAAmB,IAAI,QAAQ,mBAAmB;AAAA,MAClE;AACA,iBAAW,MAAM,MAAM,YAAY;AAAA,QACjC,SAAS;AAAA,QACT,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,YAAM,YAAY,eAAe,SAAS,IAAI,SAAS;AACvD,YAAM,YAAY,eAAe,SAAS,sBAAsB,IAAK,IAA8B,QAAQ,EAAE;AAC7G,YAAM,YAAY,aAAa;AAC/B,YAAM,OAAO,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC;AAAA,IACxF;AAEA,QAAI,SAAS,WAAW,KAAK;AAC3B,YAAMA,WAAkC,CAAC;AACzC,eAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,QAAAA,SAAQ,GAAG,IAAI;AAAA,MACjB,CAAC;AAED,UAAI;AAAE,cAAM,SAAS,YAAY;AAAA,MAAG,QAAQ;AAAA,MAAQ;AACpD,aAAO;AAAA,QACL,KAAK;AAAA,QACL,UAAU;AAAA,QACV,MAAM;AAAA,QACN,aAAa,SAAS,QAAQ,IAAI,cAAc,KAAK;AAAA,QACrD,YAAY;AAAA,QACZ,SAAAA;AAAA,MACF;AAAA,IACF;AAEA,QAAI,kBAAkB,IAAI,SAAS,MAAM,GAAG;AAC1C,YAAM,WAAW,SAAS,QAAQ,IAAI,UAAU;AAChD,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,eAAe,uCAAuC,UAAU,IAAI,KAAK;AAAA,MACrF;AAEA;AACA,UAAI,gBAAgB,cAAc;AAChC,cAAM,IAAI,eAAe,wBAAwB,YAAY,UAAU,WAAW,IAAI,KAAK;AAAA,MAC7F;AAGA,mBAAa,IAAI,IAAI,UAAU,UAAU,EAAE,SAAS;AACpD;AAAA,IACF;AAEA,QAAI,mBAAmB,IAAI,SAAS,MAAM,GAAG;AAC3C,YAAM,IAAI,eAAe,QAAQ,SAAS,MAAM,SAAS,UAAU,IAAI,IAAI;AAAA,IAC7E;AAEA,UAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,UAAM,UAAkC,CAAC;AACzC,aAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,cAAQ,GAAG,IAAI;AAAA,IACjB,CAAC;AAED,UAAM,QAAQ,YAAY,SAAS,iBAAiB;AACpD,QAAI;AACJ,QAAI;AAEJ,QAAI,OAAO;AACT,YAAM,WAAW,MAAM,SAAS,YAAY;AAC5C,kBAAY,OAAO,KAAK,QAAQ;AAChC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,MAAM,SAAS,KAAK;AAAA,IAC7B;AAEA,WAAO;AAAA,MACL,KAAK;AAAA,MACL,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA,YAAY,SAAS;AAAA,MACrB;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;","names":["headers"]}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type Browser, type BrowserContext } from 'playwright';
|
|
2
|
+
export interface DomainRoutingStats {
|
|
3
|
+
domain: string;
|
|
4
|
+
successCount: number;
|
|
5
|
+
failureCount: number;
|
|
6
|
+
preferChromium: boolean;
|
|
7
|
+
lastSuccess?: string;
|
|
8
|
+
lastFailure?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface ConnectionResult {
|
|
11
|
+
connected: boolean;
|
|
12
|
+
error?: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function shouldUseLightpanda(domain: string): boolean;
|
|
15
|
+
export declare function recordSuccess(domain: string): void;
|
|
16
|
+
export declare function recordFailure(domain: string): void;
|
|
17
|
+
export declare function getDomainStats(domain: string): DomainRoutingStats | null;
|
|
18
|
+
export declare class LightpandaAdapter {
|
|
19
|
+
private browser;
|
|
20
|
+
private url;
|
|
21
|
+
constructor(url?: string);
|
|
22
|
+
connect(): Promise<ConnectionResult>;
|
|
23
|
+
disconnect(): Promise<void>;
|
|
24
|
+
isHealthy(): Promise<boolean>;
|
|
25
|
+
getContext(): Promise<BrowserContext | null>;
|
|
26
|
+
getBrowser(): Browser | null;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=lightpanda.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lightpanda.d.ts","sourceRoot":"","sources":["../../src/fetch/lightpanda.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAQzE,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAoC3D;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAgBlD;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAuBlD;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CA4BxE;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,GAAG,CAAS;gBAER,GAAG,CAAC,EAAE,MAAM;IAKlB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAwBpC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAa3B,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAc7B,UAAU,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAgBlD,UAAU,IAAI,OAAO,GAAG,IAAI;CAG7B"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { chromium } from "playwright";
|
|
2
|
+
import { getConfig } from "../config.js";
|
|
3
|
+
import { createLogger } from "../logger.js";
|
|
4
|
+
import { isCDPReachable } from "./cdp-client.js";
|
|
5
|
+
import { getDatabase } from "../cache/db.js";
|
|
6
|
+
const log = createLogger("fetch");
|
|
7
|
+
function shouldUseLightpanda(domain) {
|
|
8
|
+
try {
|
|
9
|
+
const config = getConfig();
|
|
10
|
+
if (!config.lightpandaEnabled || !config.lightpandaUrl) {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
const db = getDatabase();
|
|
14
|
+
const row = db.prepare(
|
|
15
|
+
"SELECT failure_count, prefer_chromium FROM lightpanda_routing WHERE domain = ?"
|
|
16
|
+
).get(domain);
|
|
17
|
+
if (!row) return true;
|
|
18
|
+
if (row.prefer_chromium === 1) {
|
|
19
|
+
log.debug("domain prefers chromium over lightpanda", { domain });
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
if (row.failure_count >= config.lightpandaFailureThreshold) {
|
|
23
|
+
log.debug("domain failure threshold reached for lightpanda", {
|
|
24
|
+
domain,
|
|
25
|
+
failures: row.failure_count,
|
|
26
|
+
threshold: config.lightpandaFailureThreshold
|
|
27
|
+
});
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
return true;
|
|
31
|
+
} catch (err) {
|
|
32
|
+
log.warn("shouldUseLightpanda check failed, defaulting to chromium", {
|
|
33
|
+
domain,
|
|
34
|
+
error: String(err)
|
|
35
|
+
});
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function recordSuccess(domain) {
|
|
40
|
+
try {
|
|
41
|
+
const db = getDatabase();
|
|
42
|
+
db.prepare(`
|
|
43
|
+
INSERT INTO lightpanda_routing (domain, success_count, last_success, last_updated)
|
|
44
|
+
VALUES (?, 1, datetime('now'), datetime('now'))
|
|
45
|
+
ON CONFLICT(domain) DO UPDATE SET
|
|
46
|
+
success_count = success_count + 1,
|
|
47
|
+
last_success = datetime('now'),
|
|
48
|
+
last_updated = datetime('now')
|
|
49
|
+
`).run(domain);
|
|
50
|
+
log.debug("recorded lightpanda success", { domain });
|
|
51
|
+
} catch (err) {
|
|
52
|
+
log.warn("failed to record lightpanda success", { domain, error: String(err) });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function recordFailure(domain) {
|
|
56
|
+
try {
|
|
57
|
+
const config = getConfig();
|
|
58
|
+
const db = getDatabase();
|
|
59
|
+
db.prepare(`
|
|
60
|
+
INSERT INTO lightpanda_routing (domain, failure_count, last_failure, last_updated)
|
|
61
|
+
VALUES (?, 1, datetime('now'), datetime('now'))
|
|
62
|
+
ON CONFLICT(domain) DO UPDATE SET
|
|
63
|
+
failure_count = failure_count + 1,
|
|
64
|
+
last_failure = datetime('now'),
|
|
65
|
+
last_updated = datetime('now'),
|
|
66
|
+
prefer_chromium = CASE
|
|
67
|
+
WHEN failure_count + 1 >= ?
|
|
68
|
+
THEN 1
|
|
69
|
+
ELSE prefer_chromium
|
|
70
|
+
END
|
|
71
|
+
`).run(domain, config.lightpandaFailureThreshold);
|
|
72
|
+
log.debug("recorded lightpanda failure", { domain });
|
|
73
|
+
} catch (err) {
|
|
74
|
+
log.warn("failed to record lightpanda failure", { domain, error: String(err) });
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function getDomainStats(domain) {
|
|
78
|
+
try {
|
|
79
|
+
const db = getDatabase();
|
|
80
|
+
const row = db.prepare(
|
|
81
|
+
"SELECT * FROM lightpanda_routing WHERE domain = ?"
|
|
82
|
+
).get(domain);
|
|
83
|
+
if (!row) return null;
|
|
84
|
+
return {
|
|
85
|
+
domain: row.domain,
|
|
86
|
+
successCount: row.success_count,
|
|
87
|
+
failureCount: row.failure_count,
|
|
88
|
+
preferChromium: row.prefer_chromium === 1,
|
|
89
|
+
lastSuccess: row.last_success ?? void 0,
|
|
90
|
+
lastFailure: row.last_failure ?? void 0
|
|
91
|
+
};
|
|
92
|
+
} catch (err) {
|
|
93
|
+
log.warn("getDomainStats failed", { domain, error: String(err) });
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
class LightpandaAdapter {
|
|
98
|
+
browser = null;
|
|
99
|
+
url;
|
|
100
|
+
constructor(url) {
|
|
101
|
+
const config = getConfig();
|
|
102
|
+
this.url = url ?? config.lightpandaUrl ?? "http://localhost:9222";
|
|
103
|
+
}
|
|
104
|
+
async connect() {
|
|
105
|
+
try {
|
|
106
|
+
const reachable = await isCDPReachable(this.url);
|
|
107
|
+
if (!reachable) {
|
|
108
|
+
log.debug("lightpanda CDP not reachable", { url: this.url });
|
|
109
|
+
return { connected: false, error: "CDP endpoint not reachable" };
|
|
110
|
+
}
|
|
111
|
+
this.browser = await chromium.connectOverCDP(this.url);
|
|
112
|
+
log.info("connected to lightpanda via CDP", { url: this.url });
|
|
113
|
+
return { connected: true };
|
|
114
|
+
} catch (err) {
|
|
115
|
+
log.warn("lightpanda connection failed", {
|
|
116
|
+
url: this.url,
|
|
117
|
+
error: err instanceof Error ? err.message : String(err)
|
|
118
|
+
});
|
|
119
|
+
this.browser = null;
|
|
120
|
+
return {
|
|
121
|
+
connected: false,
|
|
122
|
+
error: err instanceof Error ? err.message : String(err)
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async disconnect() {
|
|
127
|
+
try {
|
|
128
|
+
if (this.browser) {
|
|
129
|
+
await this.browser.close();
|
|
130
|
+
this.browser = null;
|
|
131
|
+
log.debug("disconnected from lightpanda");
|
|
132
|
+
}
|
|
133
|
+
} catch (err) {
|
|
134
|
+
log.warn("lightpanda disconnect error", { error: String(err) });
|
|
135
|
+
this.browser = null;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async isHealthy() {
|
|
139
|
+
try {
|
|
140
|
+
if (!this.browser) return false;
|
|
141
|
+
if ("isConnected" in this.browser && typeof this.browser.isConnected === "function") {
|
|
142
|
+
if (!this.browser.isConnected()) return false;
|
|
143
|
+
}
|
|
144
|
+
return await isCDPReachable(this.url);
|
|
145
|
+
} catch {
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
async getContext() {
|
|
150
|
+
try {
|
|
151
|
+
if (!this.browser) {
|
|
152
|
+
const result = await this.connect();
|
|
153
|
+
if (!result.connected || !this.browser) return null;
|
|
154
|
+
}
|
|
155
|
+
const contexts = this.browser.contexts();
|
|
156
|
+
if (contexts.length > 0) return contexts[0];
|
|
157
|
+
return await this.browser.newContext();
|
|
158
|
+
} catch (err) {
|
|
159
|
+
log.warn("failed to get lightpanda context", { error: String(err) });
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
getBrowser() {
|
|
164
|
+
return this.browser;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
export {
|
|
168
|
+
LightpandaAdapter,
|
|
169
|
+
getDomainStats,
|
|
170
|
+
recordFailure,
|
|
171
|
+
recordSuccess,
|
|
172
|
+
shouldUseLightpanda
|
|
173
|
+
};
|
|
174
|
+
//# sourceMappingURL=lightpanda.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/fetch/lightpanda.ts"],"sourcesContent":["import { chromium, type Browser, type BrowserContext } from 'playwright';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\nimport { isCDPReachable } from './cdp-client.js';\nimport { getDatabase } from '../cache/db.js';\n\nconst log = createLogger('fetch');\n\nexport interface DomainRoutingStats {\n domain: string;\n successCount: number;\n failureCount: number;\n preferChromium: boolean;\n lastSuccess?: string;\n lastFailure?: string;\n}\n\nexport interface ConnectionResult {\n connected: boolean;\n error?: string;\n}\n\nexport function shouldUseLightpanda(domain: string): boolean {\n try {\n const config = getConfig();\n if (!config.lightpandaEnabled || !config.lightpandaUrl) {\n return false;\n }\n\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT failure_count, prefer_chromium FROM lightpanda_routing WHERE domain = ?',\n ).get(domain) as { failure_count: number; prefer_chromium: number } | undefined;\n\n if (!row) return true;\n\n if (row.prefer_chromium === 1) {\n log.debug('domain prefers chromium over lightpanda', { domain });\n return false;\n }\n\n if (row.failure_count >= config.lightpandaFailureThreshold) {\n log.debug('domain failure threshold reached for lightpanda', {\n domain,\n failures: row.failure_count,\n threshold: config.lightpandaFailureThreshold,\n });\n return false;\n }\n\n return true;\n } catch (err) {\n log.warn('shouldUseLightpanda check failed, defaulting to chromium', {\n domain,\n error: String(err),\n });\n return false;\n }\n}\n\nexport function recordSuccess(domain: string): void {\n try {\n const db = getDatabase();\n db.prepare(`\n INSERT INTO lightpanda_routing (domain, success_count, last_success, last_updated)\n VALUES (?, 1, datetime('now'), datetime('now'))\n ON CONFLICT(domain) DO UPDATE SET\n success_count = success_count + 1,\n last_success = datetime('now'),\n last_updated = datetime('now')\n `).run(domain);\n\n log.debug('recorded lightpanda success', { domain });\n } catch (err) {\n log.warn('failed to record lightpanda success', { domain, error: String(err) });\n }\n}\n\nexport function recordFailure(domain: string): void {\n try {\n const config = getConfig();\n const db = getDatabase();\n\n db.prepare(`\n INSERT INTO lightpanda_routing (domain, failure_count, last_failure, last_updated)\n VALUES (?, 1, datetime('now'), datetime('now'))\n ON CONFLICT(domain) DO UPDATE SET\n failure_count = failure_count + 1,\n last_failure = datetime('now'),\n last_updated = datetime('now'),\n prefer_chromium = CASE\n WHEN failure_count + 1 >= ?\n THEN 1\n ELSE prefer_chromium\n END\n `).run(domain, config.lightpandaFailureThreshold);\n\n log.debug('recorded lightpanda failure', { domain });\n } catch (err) {\n log.warn('failed to record lightpanda failure', { domain, error: String(err) });\n }\n}\n\nexport function getDomainStats(domain: string): DomainRoutingStats | null {\n try {\n const db = getDatabase();\n const row = db.prepare(\n 'SELECT * FROM lightpanda_routing WHERE domain = ?',\n ).get(domain) as {\n domain: string;\n success_count: number;\n failure_count: number;\n prefer_chromium: number;\n last_success: string | null;\n last_failure: string | null;\n } | undefined;\n\n if (!row) return null;\n\n return {\n domain: row.domain,\n successCount: row.success_count,\n failureCount: row.failure_count,\n preferChromium: row.prefer_chromium === 1,\n lastSuccess: row.last_success ?? undefined,\n lastFailure: row.last_failure ?? undefined,\n };\n } catch (err) {\n log.warn('getDomainStats failed', { domain, error: String(err) });\n return null;\n }\n}\n\nexport class LightpandaAdapter {\n private browser: Browser | null = null;\n private url: string;\n\n constructor(url?: string) {\n const config = getConfig();\n this.url = url ?? config.lightpandaUrl ?? 'http://localhost:9222';\n }\n\n async connect(): Promise<ConnectionResult> {\n try {\n const reachable = await isCDPReachable(this.url);\n if (!reachable) {\n log.debug('lightpanda CDP not reachable', { url: this.url });\n return { connected: false, error: 'CDP endpoint not reachable' };\n }\n\n this.browser = await chromium.connectOverCDP(this.url);\n log.info('connected to lightpanda via CDP', { url: this.url });\n return { connected: true };\n } catch (err) {\n log.warn('lightpanda connection failed', {\n url: this.url,\n error: err instanceof Error ? err.message : String(err),\n });\n this.browser = null;\n return {\n connected: false,\n error: err instanceof Error ? err.message : String(err),\n };\n }\n }\n\n async disconnect(): Promise<void> {\n try {\n if (this.browser) {\n await this.browser.close();\n this.browser = null;\n log.debug('disconnected from lightpanda');\n }\n } catch (err) {\n log.warn('lightpanda disconnect error', { error: String(err) });\n this.browser = null;\n }\n }\n\n async isHealthy(): Promise<boolean> {\n try {\n if (!this.browser) return false;\n\n if ('isConnected' in this.browser && typeof this.browser.isConnected === 'function') {\n if (!this.browser.isConnected()) return false;\n }\n\n return await isCDPReachable(this.url);\n } catch {\n return false;\n }\n }\n\n async getContext(): Promise<BrowserContext | null> {\n try {\n if (!this.browser) {\n const result = await this.connect();\n if (!result.connected || !this.browser) return null;\n }\n\n const contexts = this.browser.contexts();\n if (contexts.length > 0) return contexts[0];\n return await this.browser.newContext();\n } catch (err) {\n log.warn('failed to get lightpanda context', { error: String(err) });\n return null;\n }\n }\n\n getBrowser(): Browser | null {\n return this.browser;\n }\n}\n"],"mappings":"AAAA,SAAS,gBAAmD;AAC5D,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB;AAC/B,SAAS,mBAAmB;AAE5B,MAAM,MAAM,aAAa,OAAO;AAgBzB,SAAS,oBAAoB,QAAyB;AAC3D,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,QAAI,CAAC,OAAO,qBAAqB,CAAC,OAAO,eAAe;AACtD,aAAO;AAAA,IACT;AAEA,UAAM,KAAK,YAAY;AACvB,UAAM,MAAM,GAAG;AAAA,MACb;AAAA,IACF,EAAE,IAAI,MAAM;AAEZ,QAAI,CAAC,IAAK,QAAO;AAEjB,QAAI,IAAI,oBAAoB,GAAG;AAC7B,UAAI,MAAM,2CAA2C,EAAE,OAAO,CAAC;AAC/D,aAAO;AAAA,IACT;AAEA,QAAI,IAAI,iBAAiB,OAAO,4BAA4B;AAC1D,UAAI,MAAM,mDAAmD;AAAA,QAC3D;AAAA,QACA,UAAU,IAAI;AAAA,QACd,WAAW,OAAO;AAAA,MACpB,CAAC;AACD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,KAAK,4DAA4D;AAAA,MACnE;AAAA,MACA,OAAO,OAAO,GAAG;AAAA,IACnB,CAAC;AACD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,cAAc,QAAsB;AAClD,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,OAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOV,EAAE,IAAI,MAAM;AAEb,QAAI,MAAM,+BAA+B,EAAE,OAAO,CAAC;AAAA,EACrD,SAAS,KAAK;AACZ,QAAI,KAAK,uCAAuC,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EAChF;AACF;AAEO,SAAS,cAAc,QAAsB;AAClD,MAAI;AACF,UAAM,SAAS,UAAU;AACzB,UAAM,KAAK,YAAY;AAEvB,OAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAYV,EAAE,IAAI,QAAQ,OAAO,0BAA0B;AAEhD,QAAI,MAAM,+BAA+B,EAAE,OAAO,CAAC;AAAA,EACrD,SAAS,KAAK;AACZ,QAAI,KAAK,uCAAuC,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EAChF;AACF;AAEO,SAAS,eAAe,QAA2C;AACxE,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,UAAM,MAAM,GAAG;AAAA,MACb;AAAA,IACF,EAAE,IAAI,MAAM;AASZ,QAAI,CAAC,IAAK,QAAO;AAEjB,WAAO;AAAA,MACL,QAAQ,IAAI;AAAA,MACZ,cAAc,IAAI;AAAA,MAClB,cAAc,IAAI;AAAA,MAClB,gBAAgB,IAAI,oBAAoB;AAAA,MACxC,aAAa,IAAI,gBAAgB;AAAA,MACjC,aAAa,IAAI,gBAAgB;AAAA,IACnC;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,KAAK,yBAAyB,EAAE,QAAQ,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,WAAO;AAAA,EACT;AACF;AAEO,MAAM,kBAAkB;AAAA,EACrB,UAA0B;AAAA,EAC1B;AAAA,EAER,YAAY,KAAc;AACxB,UAAM,SAAS,UAAU;AACzB,SAAK,MAAM,OAAO,OAAO,iBAAiB;AAAA,EAC5C;AAAA,EAEA,MAAM,UAAqC;AACzC,QAAI;AACF,YAAM,YAAY,MAAM,eAAe,KAAK,GAAG;AAC/C,UAAI,CAAC,WAAW;AACd,YAAI,MAAM,gCAAgC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC3D,eAAO,EAAE,WAAW,OAAO,OAAO,6BAA6B;AAAA,MACjE;AAEA,WAAK,UAAU,MAAM,SAAS,eAAe,KAAK,GAAG;AACrD,UAAI,KAAK,mCAAmC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC7D,aAAO,EAAE,WAAW,KAAK;AAAA,IAC3B,SAAS,KAAK;AACZ,UAAI,KAAK,gCAAgC;AAAA,QACvC,KAAK,KAAK;AAAA,QACV,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AACD,WAAK,UAAU;AACf,aAAO;AAAA,QACL,WAAW;AAAA,QACX,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,QAAI;AACF,UAAI,KAAK,SAAS;AAChB,cAAM,KAAK,QAAQ,MAAM;AACzB,aAAK,UAAU;AACf,YAAI,MAAM,8BAA8B;AAAA,MAC1C;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,KAAK,+BAA+B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC9D,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAM,YAA8B;AAClC,QAAI;AACF,UAAI,CAAC,KAAK,QAAS,QAAO;AAE1B,UAAI,iBAAiB,KAAK,WAAW,OAAO,KAAK,QAAQ,gBAAgB,YAAY;AACnF,YAAI,CAAC,KAAK,QAAQ,YAAY,EAAG,QAAO;AAAA,MAC1C;AAEA,aAAO,MAAM,eAAe,KAAK,GAAG;AAAA,IACtC,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAA6C;AACjD,QAAI;AACF,UAAI,CAAC,KAAK,SAAS;AACjB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAClC,YAAI,CAAC,OAAO,aAAa,CAAC,KAAK,QAAS,QAAO;AAAA,MACjD;AAEA,YAAM,WAAW,KAAK,QAAQ,SAAS;AACvC,UAAI,SAAS,SAAS,EAAG,QAAO,SAAS,CAAC;AAC1C,aAAO,MAAM,KAAK,QAAQ,WAAW;AAAA,IACvC,SAAS,KAAK;AACZ,UAAI,KAAK,oCAAoC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACnE,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,aAA6B;AAC3B,WAAO,KAAK;AAAA,EACd;AACF;","names":[]}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type Browser, type BrowserContext } from 'playwright';
|
|
2
|
+
export interface InstallStatus {
|
|
3
|
+
installed: boolean;
|
|
4
|
+
hint?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function detectPlaywrightInstall(): Promise<InstallStatus>;
|
|
7
|
+
export declare function shouldEscalate(body: string): boolean;
|
|
8
|
+
export declare function getDaemonBrowser(): Promise<{
|
|
9
|
+
browser: Browser;
|
|
10
|
+
context: BrowserContext;
|
|
11
|
+
}>;
|
|
12
|
+
export declare function closeDaemonBrowser(): Promise<void>;
|
|
13
|
+
export declare function fetchWithPlaywright(url: string, opts?: {
|
|
14
|
+
timeoutMs?: number;
|
|
15
|
+
}): Promise<{
|
|
16
|
+
html: string;
|
|
17
|
+
text: string;
|
|
18
|
+
}>;
|
|
19
|
+
//# sourceMappingURL=playwright-tier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-tier.d.ts","sourceRoot":"","sources":["../../src/fetch/playwright-tier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAMzE,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,aAAa,CAAC,CAQtE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAKpD;AAMD,wBAAsB,gBAAgB,IAAI,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAAC,CAsB/F;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAWjI"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { chromium } from "playwright";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { createLogger } from "../logger.js";
|
|
4
|
+
const log = createLogger("playwright-tier");
|
|
5
|
+
async function detectPlaywrightInstall() {
|
|
6
|
+
try {
|
|
7
|
+
const exec = chromium.executablePath();
|
|
8
|
+
if (exec && existsSync(exec)) return { installed: true };
|
|
9
|
+
return { installed: false, hint: "npx playwright install chromium" };
|
|
10
|
+
} catch {
|
|
11
|
+
return { installed: false, hint: "npx playwright install chromium" };
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function shouldEscalate(body) {
|
|
15
|
+
if (!body) return true;
|
|
16
|
+
if (body.length < 500) return true;
|
|
17
|
+
if (/enable javascript/i.test(body)) return true;
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
let _browser = null;
|
|
21
|
+
let _ctx = null;
|
|
22
|
+
let _launching = null;
|
|
23
|
+
async function getDaemonBrowser() {
|
|
24
|
+
if (_browser && _ctx) return { browser: _browser, context: _ctx };
|
|
25
|
+
if (_launching) return _launching;
|
|
26
|
+
_launching = (async () => {
|
|
27
|
+
try {
|
|
28
|
+
const status = await detectPlaywrightInstall();
|
|
29
|
+
if (!status.installed) {
|
|
30
|
+
const err = new Error("playwright_not_installed");
|
|
31
|
+
err.hint = status.hint;
|
|
32
|
+
throw err;
|
|
33
|
+
}
|
|
34
|
+
const browser = await chromium.launch({ headless: true });
|
|
35
|
+
const context = await browser.newContext();
|
|
36
|
+
_browser = browser;
|
|
37
|
+
_ctx = context;
|
|
38
|
+
log.info("Playwright daemon browser launched");
|
|
39
|
+
return { browser, context };
|
|
40
|
+
} finally {
|
|
41
|
+
_launching = null;
|
|
42
|
+
}
|
|
43
|
+
})();
|
|
44
|
+
return _launching;
|
|
45
|
+
}
|
|
46
|
+
async function closeDaemonBrowser() {
|
|
47
|
+
_launching = null;
|
|
48
|
+
if (_ctx) {
|
|
49
|
+
await _ctx.close();
|
|
50
|
+
_ctx = null;
|
|
51
|
+
}
|
|
52
|
+
if (_browser) {
|
|
53
|
+
await _browser.close();
|
|
54
|
+
_browser = null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async function fetchWithPlaywright(url, opts = {}) {
|
|
58
|
+
const { context } = await getDaemonBrowser();
|
|
59
|
+
const page = await context.newPage();
|
|
60
|
+
try {
|
|
61
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: opts.timeoutMs ?? 3e4 });
|
|
62
|
+
const html = await page.content();
|
|
63
|
+
const text = await page.evaluate(() => document.body?.innerText ?? "");
|
|
64
|
+
return { html, text };
|
|
65
|
+
} finally {
|
|
66
|
+
await page.close();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
export {
|
|
70
|
+
closeDaemonBrowser,
|
|
71
|
+
detectPlaywrightInstall,
|
|
72
|
+
fetchWithPlaywright,
|
|
73
|
+
getDaemonBrowser,
|
|
74
|
+
shouldEscalate
|
|
75
|
+
};
|
|
76
|
+
//# sourceMappingURL=playwright-tier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/fetch/playwright-tier.ts"],"sourcesContent":["import { chromium, type Browser, type BrowserContext } from 'playwright';\nimport { existsSync } from 'node:fs';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('playwright-tier');\n\nexport interface InstallStatus {\n installed: boolean;\n hint?: string;\n}\n\nexport async function detectPlaywrightInstall(): Promise<InstallStatus> {\n try {\n const exec = chromium.executablePath();\n if (exec && existsSync(exec)) return { installed: true };\n return { installed: false, hint: 'npx playwright install chromium' };\n } catch {\n return { installed: false, hint: 'npx playwright install chromium' };\n }\n}\n\nexport function shouldEscalate(body: string): boolean {\n if (!body) return true;\n if (body.length < 500) return true;\n if (/enable javascript/i.test(body)) return true;\n return false;\n}\n\nlet _browser: Browser | null = null;\nlet _ctx: BrowserContext | null = null;\nlet _launching: Promise<{ browser: Browser; context: BrowserContext }> | null = null;\n\nexport async function getDaemonBrowser(): Promise<{ browser: Browser; context: BrowserContext }> {\n if (_browser && _ctx) return { browser: _browser, context: _ctx };\n if (_launching) return _launching;\n _launching = (async () => {\n try {\n const status = await detectPlaywrightInstall();\n if (!status.installed) {\n const err = new Error('playwright_not_installed') as Error & { hint?: string };\n err.hint = status.hint;\n throw err;\n }\n const browser = await chromium.launch({ headless: true });\n const context = await browser.newContext();\n _browser = browser;\n _ctx = context;\n log.info('Playwright daemon browser launched');\n return { browser, context };\n } finally {\n _launching = null;\n }\n })();\n return _launching;\n}\n\nexport async function closeDaemonBrowser(): Promise<void> {\n _launching = null;\n if (_ctx) { await _ctx.close(); _ctx = null; }\n if (_browser) { await _browser.close(); _browser = null; }\n}\n\nexport async function fetchWithPlaywright(url: string, opts: { timeoutMs?: number } = {}): Promise<{ html: string; text: string }> {\n const { context } = await getDaemonBrowser();\n const page = await context.newPage();\n try {\n await page.goto(url, { waitUntil: 'domcontentloaded', timeout: opts.timeoutMs ?? 30000 });\n const html = await page.content();\n const text = await page.evaluate(() => document.body?.innerText ?? '');\n return { html, text };\n } finally {\n await page.close();\n }\n}\n"],"mappings":"AAAA,SAAS,gBAAmD;AAC5D,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,iBAAiB;AAO1C,eAAsB,0BAAkD;AACtE,MAAI;AACF,UAAM,OAAO,SAAS,eAAe;AACrC,QAAI,QAAQ,WAAW,IAAI,EAAG,QAAO,EAAE,WAAW,KAAK;AACvD,WAAO,EAAE,WAAW,OAAO,MAAM,kCAAkC;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,WAAW,OAAO,MAAM,kCAAkC;AAAA,EACrE;AACF;AAEO,SAAS,eAAe,MAAuB;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,MAAI,KAAK,SAAS,IAAK,QAAO;AAC9B,MAAI,qBAAqB,KAAK,IAAI,EAAG,QAAO;AAC5C,SAAO;AACT;AAEA,IAAI,WAA2B;AAC/B,IAAI,OAA8B;AAClC,IAAI,aAA4E;AAEhF,eAAsB,mBAA2E;AAC/F,MAAI,YAAY,KAAM,QAAO,EAAE,SAAS,UAAU,SAAS,KAAK;AAChE,MAAI,WAAY,QAAO;AACvB,gBAAc,YAAY;AACxB,QAAI;AACF,YAAM,SAAS,MAAM,wBAAwB;AAC7C,UAAI,CAAC,OAAO,WAAW;AACrB,cAAM,MAAM,IAAI,MAAM,0BAA0B;AAChD,YAAI,OAAO,OAAO;AAClB,cAAM;AAAA,MACR;AACA,YAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,YAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,iBAAW;AACX,aAAO;AACP,UAAI,KAAK,oCAAoC;AAC7C,aAAO,EAAE,SAAS,QAAQ;AAAA,IAC5B,UAAE;AACA,mBAAa;AAAA,IACf;AAAA,EACF,GAAG;AACH,SAAO;AACT;AAEA,eAAsB,qBAAoC;AACxD,eAAa;AACb,MAAI,MAAM;AAAE,UAAM,KAAK,MAAM;AAAG,WAAO;AAAA,EAAM;AAC7C,MAAI,UAAU;AAAE,UAAM,SAAS,MAAM;AAAG,eAAW;AAAA,EAAM;AAC3D;AAEA,eAAsB,oBAAoB,KAAa,OAA+B,CAAC,GAA4C;AACjI,QAAM,EAAE,QAAQ,IAAI,MAAM,iBAAiB;AAC3C,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,MAAI;AACF,UAAM,KAAK,KAAK,KAAK,EAAE,WAAW,oBAAoB,SAAS,KAAK,aAAa,IAAM,CAAC;AACxF,UAAM,OAAO,MAAM,KAAK,QAAQ;AAChC,UAAM,OAAO,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,aAAa,EAAE;AACrE,WAAO,EAAE,MAAM,KAAK;AAAA,EACtB,UAAE;AACA,UAAM,KAAK,MAAM;AAAA,EACnB;AACF;","names":[]}
|
package/dist/fetch/router.d.ts
CHANGED
|
@@ -1,14 +1,31 @@
|
|
|
1
|
-
import type { RawFetchResult } from '../types.js';
|
|
1
|
+
import type { RawFetchResult, BrowserAction, Mode, StageError } from '../types.js';
|
|
2
2
|
export interface RouterFetchOptions {
|
|
3
3
|
renderJs?: 'auto' | 'always' | 'never';
|
|
4
4
|
useAuth?: boolean;
|
|
5
5
|
headers?: Record<string, string>;
|
|
6
6
|
screenshot?: boolean;
|
|
7
|
+
actions?: BrowserAction[];
|
|
8
|
+
force_refresh?: boolean;
|
|
9
|
+
mode?: Mode;
|
|
10
|
+
/**
|
|
11
|
+
* Conditional-GET headers. When set, the HTTP path sends them with the
|
|
12
|
+
* request and a 304 response is returned as RawFetchResult with
|
|
13
|
+
* statusCode=304 + html=''. Routes that always escalate to Playwright
|
|
14
|
+
* (renderJs=always, useAuth, actions) ignore these headers.
|
|
15
|
+
*/
|
|
16
|
+
conditionalHeaders?: {
|
|
17
|
+
ifNoneMatch?: string;
|
|
18
|
+
ifModifiedSince?: string;
|
|
19
|
+
};
|
|
7
20
|
}
|
|
8
21
|
export interface HttpClient {
|
|
9
22
|
fetch(url: string, options?: {
|
|
10
23
|
headers?: Record<string, string>;
|
|
11
24
|
timeoutMs?: number;
|
|
25
|
+
conditionalHeaders?: {
|
|
26
|
+
ifNoneMatch?: string;
|
|
27
|
+
ifModifiedSince?: string;
|
|
28
|
+
};
|
|
12
29
|
}): Promise<{
|
|
13
30
|
url: string;
|
|
14
31
|
finalUrl: string;
|
|
@@ -25,17 +42,46 @@ export interface BrowserPoolInterface {
|
|
|
25
42
|
storageStatePath?: string;
|
|
26
43
|
userDataDir?: string;
|
|
27
44
|
screenshot?: boolean;
|
|
45
|
+
actions?: BrowserAction[];
|
|
46
|
+
cdpUrl?: string;
|
|
28
47
|
}): Promise<RawFetchResult>;
|
|
29
48
|
}
|
|
49
|
+
export type HttpFetcher = (url: string, options?: {
|
|
50
|
+
headers?: Record<string, string>;
|
|
51
|
+
timeoutMs?: number;
|
|
52
|
+
}) => Promise<{
|
|
53
|
+
url: string;
|
|
54
|
+
html: string;
|
|
55
|
+
text: string;
|
|
56
|
+
}>;
|
|
57
|
+
export type PlaywrightFetcher = (url: string, options?: {
|
|
58
|
+
timeoutMs?: number;
|
|
59
|
+
}) => Promise<{
|
|
60
|
+
html: string;
|
|
61
|
+
text: string;
|
|
62
|
+
}>;
|
|
63
|
+
export interface SmartRouterOptions {
|
|
64
|
+
httpClient?: HttpClient;
|
|
65
|
+
browserPool?: BrowserPoolInterface;
|
|
66
|
+
httpFetcher?: HttpFetcher;
|
|
67
|
+
playwrightFetcher?: PlaywrightFetcher;
|
|
68
|
+
}
|
|
30
69
|
interface DomainStats {
|
|
31
70
|
failureCount: number;
|
|
32
71
|
preferPlaywright: boolean;
|
|
33
72
|
}
|
|
34
73
|
export declare class SmartRouter {
|
|
35
|
-
private readonly httpClient;
|
|
36
|
-
private readonly browserPool;
|
|
37
74
|
private readonly domainMap;
|
|
75
|
+
private readonly httpClient?;
|
|
76
|
+
private readonly browserPool?;
|
|
77
|
+
private readonly httpFetcher;
|
|
78
|
+
private readonly playwrightFetcher;
|
|
38
79
|
constructor(httpClient: HttpClient, browserPool: BrowserPoolInterface);
|
|
80
|
+
constructor(options: SmartRouterOptions);
|
|
81
|
+
private makeDefaultHttpFetcher;
|
|
82
|
+
fetch(url: string, options: RouterFetchOptions & {
|
|
83
|
+
mode: 'stealth';
|
|
84
|
+
}): Promise<RawFetchResult | StageError>;
|
|
39
85
|
fetch(url: string, options?: RouterFetchOptions): Promise<RawFetchResult>;
|
|
40
86
|
getDomainStats(domain: string): DomainStats | undefined;
|
|
41
87
|
private ensureStats;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEnF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE;YACnB,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,eAAe,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;KACH,GACA,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChK,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KAC/D,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE1D,MAAM,MAAM,iBAAiB,GAAG,CAC9B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,KAC7B,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE7C,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,WAAW,CAAC,EAAE,oBAAoB,CAAC;IACnC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAuB;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;gBAE1C,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,oBAAoB;gBACzD,OAAO,EAAE,kBAAkB;IAiCvC,OAAO,CAAC,sBAAsB;IAUxB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,GAAG,UAAU,CAAC;IAC3G,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC;IA0J/E,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}
|