argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
link_source.py — 把「消费者入口」指回本仓库来源(符号链接,不复制)。
|
|
4
|
+
|
|
5
|
+
标准化原则:
|
|
6
|
+
1. 磁盘上只应有一份 argo 代码:本仓库(scripts/ 的上一级)。
|
|
7
|
+
2. Skill 目录 / 文档入口若需要出现在主机约定位置,用 **symlink** 指向来源,
|
|
8
|
+
禁止 rsync/cp 出第二份业务树。
|
|
9
|
+
3. **链接必须直连来源(1 跳)**。宿主入口挂到另一个入口上(如
|
|
10
|
+
`~/.workbuddy/skills/argo -> ~/.claude/skills/argo -> 来源`)语义上能解析,
|
|
11
|
+
但多出一环依赖:中间那环被删改或指偏,入口就成断链,而工具此前对此
|
|
12
|
+
完全无感。间接链接会被本脚本**重建为直连**(只替换 symlink 本身)。
|
|
13
|
+
4. **目标路径绝不写死在代码里**。来源优先级:
|
|
14
|
+
CLI `--to PATH`(可重复)
|
|
15
|
+
→ 环境变量 `ARGO_LINK_TARGETS`(os.pathsep 分隔,如 `:` / `;`)
|
|
16
|
+
→ 来源根目录下的 `installs.local.yaml`(本机声明,应 gitignore)
|
|
17
|
+
5. 注册表派生仍只走 `sync_backends.py`(config.yaml → backends/*),与链接无关。
|
|
18
|
+
|
|
19
|
+
用法:
|
|
20
|
+
# 本机 installs.local.yaml 示例见 installs.local.yaml.example
|
|
21
|
+
python3 scripts/link_source.py
|
|
22
|
+
python3 scripts/link_source.py --to /any/consumer/path/argo
|
|
23
|
+
ARGO_LINK_TARGETS="$HOME/.claude/skills/argo:$HOME/.agents/skills/argo" \\
|
|
24
|
+
python3 scripts/link_source.py
|
|
25
|
+
python3 scripts/link_source.py --check # 只校验;间接链接记 warn 并非零退出
|
|
26
|
+
python3 scripts/link_source.py --dry-run
|
|
27
|
+
|
|
28
|
+
退出码:0 成功;1 无目标 / 链接失败 / 校验不一致(含「只经多跳间接指向来源」)。
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import argparse
|
|
34
|
+
import os
|
|
35
|
+
import shutil
|
|
36
|
+
import subprocess
|
|
37
|
+
import sys
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
|
|
40
|
+
SOURCE = Path(__file__).resolve().parent.parent
|
|
41
|
+
LOCAL_INSTALLS = SOURCE / "installs.local.yaml"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _load_local_targets() -> list[Path]:
|
|
45
|
+
if not LOCAL_INSTALLS.exists():
|
|
46
|
+
return []
|
|
47
|
+
try:
|
|
48
|
+
from yaml_load import load as _yaml_load
|
|
49
|
+
except ImportError:
|
|
50
|
+
print("[warn] 需要 PyYAML 才能读 installs.local.yaml", file=sys.stderr)
|
|
51
|
+
return []
|
|
52
|
+
data = _yaml_load(LOCAL_INSTALLS) or {}
|
|
53
|
+
raw = data.get("link_targets") or data.get("targets") or []
|
|
54
|
+
if not isinstance(raw, list):
|
|
55
|
+
return []
|
|
56
|
+
out: list[Path] = []
|
|
57
|
+
for item in raw:
|
|
58
|
+
if not item:
|
|
59
|
+
continue
|
|
60
|
+
p = Path(str(item)).expanduser()
|
|
61
|
+
# 相对路径相对来源根;规范化但不跟随符号链接(见 _normalize_target)
|
|
62
|
+
out.append(_normalize_target(SOURCE / p if not p.is_absolute() else p))
|
|
63
|
+
return out
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _normalize_target(p: Path) -> Path:
|
|
67
|
+
"""规范化目标路径:展开 ~、补成绝对路径、消掉 . 与 ..,**但不跟随符号链接**。
|
|
68
|
+
|
|
69
|
+
为什么不用 resolve():宿主入口本来就应该是一条 symlink。resolve() 会先把
|
|
70
|
+
「已经指向来源的链接」折叠成来源路径,于是 link_one 看到的目标是「来源本体」
|
|
71
|
+
而直接跳过重建,check_targets 也无法区分「直连」与「隔了一层的间接链接」——
|
|
72
|
+
校验常年报 ok,实际入口却挂在中转链接上(2026-09-14 实测踩到)。
|
|
73
|
+
"""
|
|
74
|
+
p = Path(p).expanduser()
|
|
75
|
+
if not p.is_absolute():
|
|
76
|
+
p = Path.cwd() / p
|
|
77
|
+
return Path(os.path.normpath(str(p)))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _hops_to_source(target: Path) -> int:
|
|
81
|
+
"""目标到来源的符号链接跳数。
|
|
82
|
+
|
|
83
|
+
0 = 目标就是来源本体;1 = 直连;>1 = 间接链接(可解析但多一跳依赖);
|
|
84
|
+
-1 = 未指向来源 / 断链 / 成环(超过 8 跳按未指向处理,避免死循环)。
|
|
85
|
+
"""
|
|
86
|
+
src = SOURCE.resolve()
|
|
87
|
+
cur = target
|
|
88
|
+
for hops in range(8):
|
|
89
|
+
try:
|
|
90
|
+
resolved = cur.resolve()
|
|
91
|
+
except (OSError, RuntimeError):
|
|
92
|
+
# 断链 / 符号链接成环(py3.13 的 resolve() 对环抛 RuntimeError)
|
|
93
|
+
return -1
|
|
94
|
+
if resolved != src:
|
|
95
|
+
return -1
|
|
96
|
+
if not cur.is_symlink():
|
|
97
|
+
return hops
|
|
98
|
+
try:
|
|
99
|
+
raw = os.readlink(cur)
|
|
100
|
+
except OSError:
|
|
101
|
+
return -1
|
|
102
|
+
nxt = Path(raw)
|
|
103
|
+
if not nxt.is_absolute():
|
|
104
|
+
nxt = cur.parent / nxt
|
|
105
|
+
cur = Path(os.path.normpath(str(nxt)))
|
|
106
|
+
return -1
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _env_targets() -> list[Path]:
|
|
110
|
+
raw = os.environ.get("ARGO_LINK_TARGETS", "").strip()
|
|
111
|
+
if not raw:
|
|
112
|
+
return []
|
|
113
|
+
return [_normalize_target(p) for p in raw.split(os.pathsep) if p.strip()]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def resolve_targets(cli: list[Path] | None) -> list[Path]:
|
|
117
|
+
"""无默认路径:没有 CLI / env / local 文件则空列表。"""
|
|
118
|
+
seen: set[str] = set()
|
|
119
|
+
ordered: list[Path] = []
|
|
120
|
+
for group in (cli or [], _env_targets(), _load_local_targets()):
|
|
121
|
+
for p in group:
|
|
122
|
+
key = _normalize_target(p)
|
|
123
|
+
if str(key) in seen:
|
|
124
|
+
continue
|
|
125
|
+
seen.add(str(key))
|
|
126
|
+
ordered.append(key)
|
|
127
|
+
return ordered
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# 判定「目录像 argo 来源/旧副本」的特征文件。用于 --force 迁走目录前的软校验,
|
|
131
|
+
# 防止用户把 --to 错指到任意数据目录,--force 下整目录被 rename 迁走的误伤。
|
|
132
|
+
_ARGO_MARKERS = ("scripts/search.py", "SKILL.md")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _looks_like_argo_copy(path: Path) -> bool:
|
|
136
|
+
"""路径是否为 argo 来源/旧副本(含脚本与 SKILL 文档),而非任意数据目录。"""
|
|
137
|
+
if not path.is_dir():
|
|
138
|
+
return False
|
|
139
|
+
return any((path / marker).exists() for marker in _ARGO_MARKERS)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def link_one(target: Path, *, dry_run: bool, force: bool) -> int:
|
|
143
|
+
"""将 target 设为指向 SOURCE 的 symlink(间接链接会被重建为直连)。"""
|
|
144
|
+
source = SOURCE.resolve()
|
|
145
|
+
depth = _hops_to_source(target)
|
|
146
|
+
|
|
147
|
+
if depth == 0:
|
|
148
|
+
print(f"[skip] 目标就是真源本体: {target}")
|
|
149
|
+
return 0
|
|
150
|
+
|
|
151
|
+
if depth == 1:
|
|
152
|
+
print(f"[ok] 已指向真源: {target} -> {source}")
|
|
153
|
+
return 0
|
|
154
|
+
|
|
155
|
+
if depth > 1:
|
|
156
|
+
# 间接链接:能解析到来源,但中间还挂了一环。重建为直连只替换符号链接
|
|
157
|
+
# 本身(不触碰任何目录内容),因此无需 --force。
|
|
158
|
+
if dry_run:
|
|
159
|
+
print(f"[dry] 将把 {depth} 跳间接链接重建为直连: {target}")
|
|
160
|
+
return 0
|
|
161
|
+
# 先建同目录临时链接再原子替换:避免「已 unlink、symlink 又失败」的
|
|
162
|
+
# 窗口里宿主入口凭空消失(入口消失会让 Skill/工具调用直接找不到 argo)。
|
|
163
|
+
tmp = target.with_name(target.name + ".relink-tmp")
|
|
164
|
+
try:
|
|
165
|
+
if tmp.is_symlink() or tmp.exists():
|
|
166
|
+
tmp.unlink()
|
|
167
|
+
os.symlink(source, tmp, target_is_directory=True)
|
|
168
|
+
os.replace(tmp, target)
|
|
169
|
+
except OSError as e:
|
|
170
|
+
try:
|
|
171
|
+
if tmp.is_symlink() or tmp.exists():
|
|
172
|
+
tmp.unlink()
|
|
173
|
+
except OSError:
|
|
174
|
+
pass
|
|
175
|
+
print(f"[fail] 重建直连失败: {target} ({e})", file=sys.stderr)
|
|
176
|
+
return 1
|
|
177
|
+
print(f"[relink] {target} -> {source}(原为 {depth} 跳间接链接,已重建为直连)")
|
|
178
|
+
return 0
|
|
179
|
+
|
|
180
|
+
if target.exists() or target.is_symlink():
|
|
181
|
+
if not force:
|
|
182
|
+
print(
|
|
183
|
+
f"[fail] 目标已存在且不是指向真源的链接: {target}\n"
|
|
184
|
+
f" 若确认可替换,加 --force(会先移走/删除该路径)",
|
|
185
|
+
file=sys.stderr,
|
|
186
|
+
)
|
|
187
|
+
return 1
|
|
188
|
+
# 软校验(dry-run 与实际执行都触发):目录若不像 argo 来源/旧副本
|
|
189
|
+
# (无 scripts/search.py 或 SKILL.md),拒绝 --force 迁走,防止 --to
|
|
190
|
+
# 误指用户数据目录被整目录 rename 的误伤。
|
|
191
|
+
if target.is_dir() and not _looks_like_argo_copy(target):
|
|
192
|
+
print(
|
|
193
|
+
f"[fail] 目录不像 argo 真源/旧副本,拒绝 --force 迁走: {target}\n"
|
|
194
|
+
f" 若确要替换,先手动备份该目录,或删掉后再执行。",
|
|
195
|
+
file=sys.stderr,
|
|
196
|
+
)
|
|
197
|
+
return 1
|
|
198
|
+
if dry_run:
|
|
199
|
+
print(f"[dry] 将替换已有路径: {target}")
|
|
200
|
+
else:
|
|
201
|
+
if target.is_symlink() or target.is_file():
|
|
202
|
+
target.unlink()
|
|
203
|
+
elif target.is_dir():
|
|
204
|
+
# 旧多副本:整目录移走备份,避免误删用户未入库文件
|
|
205
|
+
bak = target.with_name(target.name + ".bak-before-link")
|
|
206
|
+
if bak.exists():
|
|
207
|
+
shutil.rmtree(bak) if bak.is_dir() and not bak.is_symlink() else bak.unlink(missing_ok=True)
|
|
208
|
+
target.rename(bak)
|
|
209
|
+
print(f"[bak] 旧副本已移至 {bak}")
|
|
210
|
+
else:
|
|
211
|
+
target.unlink(missing_ok=True)
|
|
212
|
+
|
|
213
|
+
parent = target.parent
|
|
214
|
+
if dry_run:
|
|
215
|
+
print(f"[dry] ln -sfn {source} {target}")
|
|
216
|
+
return 0
|
|
217
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
218
|
+
# atomic-ish: symlink in place
|
|
219
|
+
try:
|
|
220
|
+
os.symlink(source, target, target_is_directory=True)
|
|
221
|
+
except OSError:
|
|
222
|
+
# Windows 无开发者模式/管理员权限时 symlink 会失败:
|
|
223
|
+
# 目录链接退化为 junction(mklink /J,不需要管理员权限),
|
|
224
|
+
# 语义与 symlink 基本一致(resolve() 同样指向来源)。
|
|
225
|
+
if os.name == "nt":
|
|
226
|
+
r = subprocess.run(
|
|
227
|
+
["cmd", "/c", "mklink", "/J", str(target), str(source)],
|
|
228
|
+
capture_output=True, text=True,
|
|
229
|
+
encoding="utf-8", errors="replace", timeout=30,
|
|
230
|
+
)
|
|
231
|
+
# junction 不是 symlink(is_symlink() 为 False,_hops_to_source 会
|
|
232
|
+
# 把它当普通目录),此处按「解析后是否等于来源」独立判定。
|
|
233
|
+
try:
|
|
234
|
+
linked = target.resolve() == SOURCE.resolve()
|
|
235
|
+
except OSError:
|
|
236
|
+
linked = False
|
|
237
|
+
if r.returncode == 0 and linked:
|
|
238
|
+
print(f"[link] {target} -> {source} (junction)")
|
|
239
|
+
return 0
|
|
240
|
+
print(f"[fail] 创建链接失败: {target} -> {source}", file=sys.stderr)
|
|
241
|
+
return 1
|
|
242
|
+
print(f"[link] {target} -> {source}")
|
|
243
|
+
return 0
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def check_targets(targets: list[Path]) -> int:
|
|
247
|
+
bad = 0
|
|
248
|
+
print(f"真源: {SOURCE.resolve()}")
|
|
249
|
+
if not targets:
|
|
250
|
+
print("[fail] 无校验目标(请 --to / ARGO_LINK_TARGETS / installs.local.yaml)")
|
|
251
|
+
return 1
|
|
252
|
+
for t in targets:
|
|
253
|
+
depth = _hops_to_source(t)
|
|
254
|
+
if depth == 0:
|
|
255
|
+
print(f"[ok] {t} 即真源本体")
|
|
256
|
+
elif depth == 1:
|
|
257
|
+
print(f"[ok] {t} -> {SOURCE.resolve()}")
|
|
258
|
+
elif depth > 1:
|
|
259
|
+
# 间接链接仍能解析到来源,但多了一环依赖——按「校验不一致」计失败,
|
|
260
|
+
# 提示直接重跑(不带 --check)即可重建为直连。
|
|
261
|
+
print(f"[warn] {t} 经 {depth} 跳间接指向真源(非直连);"
|
|
262
|
+
f"去掉 --check 重跑即可重建为直连")
|
|
263
|
+
bad += 1
|
|
264
|
+
else:
|
|
265
|
+
print(f"[miss] {t} 未指向真源(exists={t.exists()} symlink={t.is_symlink()})")
|
|
266
|
+
bad += 1
|
|
267
|
+
return 1 if bad else 0
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def maybe_sync_backends() -> int:
|
|
271
|
+
script = SOURCE / "scripts" / "sync_backends.py"
|
|
272
|
+
if not script.exists():
|
|
273
|
+
return 0
|
|
274
|
+
# timeout(2026-09-29 补):sync_backends 是纯本地文件操作,正常秒回;
|
|
275
|
+
# 但无 timeout 的 subprocess 一旦被挂住(NFS/锁/杀软扫描),本命令会
|
|
276
|
+
# 无限期卡死。60s 对本地脚本是极宽裕的上界。
|
|
277
|
+
r = subprocess.run([sys.executable, str(script)], cwd=str(SOURCE), timeout=60)
|
|
278
|
+
if r.returncode != 0:
|
|
279
|
+
return r.returncode
|
|
280
|
+
return subprocess.run(
|
|
281
|
+
[sys.executable, str(script), "--check"], cwd=str(SOURCE),
|
|
282
|
+
timeout=60).returncode
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def main(argv: list[str] | None = None) -> int:
|
|
286
|
+
p = argparse.ArgumentParser(
|
|
287
|
+
description="将消费者入口 symlink 到 argo 真源(无内置默认路径)",
|
|
288
|
+
)
|
|
289
|
+
p.add_argument(
|
|
290
|
+
"--to",
|
|
291
|
+
action="append",
|
|
292
|
+
type=Path,
|
|
293
|
+
default=None,
|
|
294
|
+
help="链接目标路径(可重复)。无内置默认值。",
|
|
295
|
+
)
|
|
296
|
+
p.add_argument("--check", action="store_true", help="只校验目标是否指向真源")
|
|
297
|
+
p.add_argument("--dry-run", action="store_true")
|
|
298
|
+
p.add_argument(
|
|
299
|
+
"--force",
|
|
300
|
+
action="store_true",
|
|
301
|
+
help="目标已存在时替换:symlink/file 删除;目录改名为 *.bak-before-link",
|
|
302
|
+
)
|
|
303
|
+
p.add_argument(
|
|
304
|
+
"--with-backends",
|
|
305
|
+
action="store_true",
|
|
306
|
+
help="链接前先跑 sync_backends.py 派生 backends",
|
|
307
|
+
)
|
|
308
|
+
args = p.parse_args(argv)
|
|
309
|
+
|
|
310
|
+
targets = resolve_targets(args.to)
|
|
311
|
+
if not targets:
|
|
312
|
+
print(
|
|
313
|
+
"未指定任何链接目标。\n"
|
|
314
|
+
" 方式 1: python3 scripts/link_source.py --to <path> [--to <path2>]\n"
|
|
315
|
+
" 方式 2: export ARGO_LINK_TARGETS='path1:path2'\n"
|
|
316
|
+
f" 方式 3: 在真源写 {LOCAL_INSTALLS.name}(见 installs.local.yaml.example)\n"
|
|
317
|
+
"代码内不固化任何主机 skill/MCP 路径。",
|
|
318
|
+
file=sys.stderr,
|
|
319
|
+
)
|
|
320
|
+
return 1
|
|
321
|
+
|
|
322
|
+
if args.check:
|
|
323
|
+
return check_targets(targets)
|
|
324
|
+
|
|
325
|
+
if args.with_backends:
|
|
326
|
+
print(f"== sync_backends @ {SOURCE} ==")
|
|
327
|
+
rc = maybe_sync_backends()
|
|
328
|
+
if rc != 0:
|
|
329
|
+
return rc
|
|
330
|
+
|
|
331
|
+
code = 0
|
|
332
|
+
for t in targets:
|
|
333
|
+
code = link_one(t, dry_run=args.dry_run, force=args.force) or code
|
|
334
|
+
if args.dry_run:
|
|
335
|
+
return code
|
|
336
|
+
return check_targets(targets) if code == 0 else code
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
if __name__ == "__main__":
|
|
340
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""本地个性化引擎插件 — 不随公开仓库提交。
|
|
3
|
+
|
|
4
|
+
包含漏洞查询类引擎(OSV.dev + CISA KEV),因启发式杀毒检测
|
|
5
|
+
(HEUR:HackTool/VulnScan)从公开构建器中移出。
|
|
6
|
+
本地文件,不提交 git;删除后对应引擎自动消失,不影响其他功能。
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
import urllib.request
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from engines_base import safe_search, rank_score, http_open, note_failure
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# ── OSV.dev 开源漏洞库(免 key,必须 POST)─────────────────────────────────
|
|
21
|
+
|
|
22
|
+
_OSV_ECOSYSTEM_PREFIX: dict[str, str] = {
|
|
23
|
+
"pypi": "PyPI", "npm": "npm", "go": "Go", "crates": "crates.io",
|
|
24
|
+
"cargo": "crates.io", "maven": "Maven", "nuget": "NuGet",
|
|
25
|
+
"packagist": "Packagist", "composer": "Packagist",
|
|
26
|
+
"rubygems": "RubyGems", "gem": "RubyGems", "hex": "Hex",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
_OSV_NPM_HINTS: frozenset[str] = frozenset({
|
|
30
|
+
"lodash", "react", "vue", "express", "axios", "next", "webpack",
|
|
31
|
+
"typescript", "eslint", "vite", "jest", "moment", "jquery", "angular",
|
|
32
|
+
"svelte", "nuxt", "rollup", "babel", "prettier", "cheerio", "socket.io",
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
_OSV_GO_HINTS: frozenset[str] = frozenset({
|
|
36
|
+
"golang.org/x/net", "golang.org/x/crypto", "golang.org/x/text",
|
|
37
|
+
"github.com/gin-gonic/gin", "github.com/gorilla/websocket",
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _osv_resolve_ecosystem(query: str) -> tuple[str, str]:
|
|
42
|
+
q = (query or "").strip()
|
|
43
|
+
if ":" in q:
|
|
44
|
+
head, _, tail = q.partition(":")
|
|
45
|
+
eco = _OSV_ECOSYSTEM_PREFIX.get(head.strip().lower())
|
|
46
|
+
if eco and tail.strip():
|
|
47
|
+
return tail.strip(), eco
|
|
48
|
+
name = q
|
|
49
|
+
if name.startswith("@") or "/" in name:
|
|
50
|
+
if name.startswith("@"):
|
|
51
|
+
return name, "npm"
|
|
52
|
+
if name.startswith(("github.com/", "gitlab.com/", "golang.org/")):
|
|
53
|
+
return name, "Go"
|
|
54
|
+
return name, "npm"
|
|
55
|
+
if name.lower() in _OSV_NPM_HINTS:
|
|
56
|
+
return name, "npm"
|
|
57
|
+
if name.lower() in _OSV_GO_HINTS:
|
|
58
|
+
return name, "Go"
|
|
59
|
+
return name, "PyPI"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _build_osv_engine(spec: dict[str, Any]) -> Any:
|
|
63
|
+
"""OSV.dev 漏洞查询(按包名,POST /v1/query;免 key、免限次)。"""
|
|
64
|
+
timeout = spec.get("timeout", 15)
|
|
65
|
+
|
|
66
|
+
@safe_search
|
|
67
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None,
|
|
68
|
+
**kwargs) -> list[dict[str, Any]]:
|
|
69
|
+
to = _timeout or timeout
|
|
70
|
+
q = (query or "").strip()
|
|
71
|
+
if not q:
|
|
72
|
+
return []
|
|
73
|
+
if re.match(r"^CVE-\d{4}-\d{4,}$", q, re.I):
|
|
74
|
+
return []
|
|
75
|
+
pkg, eco = _osv_resolve_ecosystem(q)
|
|
76
|
+
body = json.dumps({"package": {"name": pkg, "ecosystem": eco}}).encode("utf-8")
|
|
77
|
+
req = urllib.request.Request(
|
|
78
|
+
"https://api.osv.dev/v1/query", data=body,
|
|
79
|
+
headers={"Content-Type": "application/json"})
|
|
80
|
+
eng_name = spec.get("_name") or "osv"
|
|
81
|
+
try:
|
|
82
|
+
with http_open(req, timeout=to, engine=eng_name) as resp:
|
|
83
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
84
|
+
except Exception as e:
|
|
85
|
+
if isinstance(e, (json.JSONDecodeError, UnicodeDecodeError)):
|
|
86
|
+
note_failure(eng_name, "parse", "json_decode", str(e))
|
|
87
|
+
import logging
|
|
88
|
+
logging.getLogger("unified_search.engines").warning(f"OSV 失败: {e}")
|
|
89
|
+
return []
|
|
90
|
+
results = []
|
|
91
|
+
for i, v in enumerate((data.get("vulns") or [])[:n]):
|
|
92
|
+
vid = str(v.get("id") or "").strip()
|
|
93
|
+
if not vid:
|
|
94
|
+
continue
|
|
95
|
+
summary = str(v.get("summary") or "").strip()
|
|
96
|
+
affected_bits = []
|
|
97
|
+
for aff in (v.get("affected") or [])[:2]:
|
|
98
|
+
if not isinstance(aff, dict):
|
|
99
|
+
continue
|
|
100
|
+
for rng in (aff.get("ranges") or [])[:2]:
|
|
101
|
+
if not isinstance(rng, dict):
|
|
102
|
+
continue
|
|
103
|
+
events = rng.get("events") or []
|
|
104
|
+
introduced = next((e.get("introduced") for e in events
|
|
105
|
+
if isinstance(e, dict) and e.get("introduced")), "")
|
|
106
|
+
fixed = next((e.get("fixed") for e in events
|
|
107
|
+
if isinstance(e, dict) and e.get("fixed")), "")
|
|
108
|
+
if introduced or fixed:
|
|
109
|
+
affected_bits.append(
|
|
110
|
+
f"{introduced or '?'} ≤ 受影响 < {fixed or '未修复'}")
|
|
111
|
+
aliases = [a for a in (v.get("aliases") or []) if a][:3]
|
|
112
|
+
snippet = " · ".join(p for p in (
|
|
113
|
+
summary, f"生态 {eco}",
|
|
114
|
+
affected_bits[0] if affected_bits else "",
|
|
115
|
+
" ".join(aliases),
|
|
116
|
+
) if p)[:300]
|
|
117
|
+
results.append({
|
|
118
|
+
"title": f"{vid} [{eco}] {summary}"[:500] or vid,
|
|
119
|
+
"url": f"https://osv.dev/vulnerability/{vid}",
|
|
120
|
+
"snippet": snippet,
|
|
121
|
+
"source": "osv",
|
|
122
|
+
"published_at": str(v.get("published") or "")[:10],
|
|
123
|
+
"score": rank_score(0.85, i),
|
|
124
|
+
})
|
|
125
|
+
return results
|
|
126
|
+
return _engine
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ── CISA KEV 已知在野利用漏洞(免 key,全量文件 + 本地过滤)──────────────
|
|
130
|
+
|
|
131
|
+
_KEV_CACHE: dict[str, Any] = {"at": 0.0, "items": None}
|
|
132
|
+
_KEV_TTL_S = 6 * 3600
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _kev_disk_path():
|
|
136
|
+
try:
|
|
137
|
+
from argo_paths import state_path
|
|
138
|
+
return state_path("cisa_kev_cache.json")
|
|
139
|
+
except Exception:
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _kev_load(timeout: float) -> list[dict[str, Any]]:
|
|
144
|
+
now = time.time()
|
|
145
|
+
cached = _KEV_CACHE.get("items")
|
|
146
|
+
if cached and (now - float(_KEV_CACHE.get("at") or 0)) < _KEV_TTL_S:
|
|
147
|
+
return cached
|
|
148
|
+
disk_p = _kev_disk_path()
|
|
149
|
+
if disk_p is not None and disk_p.exists():
|
|
150
|
+
try:
|
|
151
|
+
st = disk_p.stat()
|
|
152
|
+
if (now - st.st_mtime) < _KEV_TTL_S:
|
|
153
|
+
disk_items = json.loads(disk_p.read_text(encoding="utf-8"))
|
|
154
|
+
if isinstance(disk_items, list) and disk_items:
|
|
155
|
+
_KEV_CACHE["items"] = disk_items
|
|
156
|
+
_KEV_CACHE["at"] = st.st_mtime
|
|
157
|
+
return disk_items
|
|
158
|
+
except Exception:
|
|
159
|
+
pass
|
|
160
|
+
req = urllib.request.Request(
|
|
161
|
+
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
|
|
162
|
+
headers={"User-Agent": "argo-search/1.0 (+cisa-kev)", "Accept": "application/json"})
|
|
163
|
+
try:
|
|
164
|
+
with http_open(req, timeout=timeout, engine="cisa_kev") as resp:
|
|
165
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
166
|
+
except Exception as e:
|
|
167
|
+
if isinstance(e, (json.JSONDecodeError, UnicodeDecodeError)):
|
|
168
|
+
note_failure("cisa_kev", "parse", "json_decode", str(e))
|
|
169
|
+
import logging
|
|
170
|
+
logging.getLogger("unified_search.engines").warning(f"CISA KEV 失败: {e}")
|
|
171
|
+
return cached or []
|
|
172
|
+
items = data.get("vulnerabilities") or []
|
|
173
|
+
if not isinstance(items, list):
|
|
174
|
+
return cached or []
|
|
175
|
+
_KEV_CACHE["items"] = items
|
|
176
|
+
_KEV_CACHE["at"] = now
|
|
177
|
+
if disk_p is not None and items:
|
|
178
|
+
try:
|
|
179
|
+
from argo_paths import atomic_write_json
|
|
180
|
+
atomic_write_json(disk_p, items)
|
|
181
|
+
except Exception:
|
|
182
|
+
pass
|
|
183
|
+
return items
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _build_cisa_kev_engine(spec: dict[str, Any]) -> Any:
|
|
187
|
+
"""CISA KEV 在野利用漏洞目录(本地过滤全量目录)。"""
|
|
188
|
+
timeout = spec.get("timeout", 30)
|
|
189
|
+
|
|
190
|
+
@safe_search
|
|
191
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None,
|
|
192
|
+
**kwargs) -> list[dict[str, Any]]:
|
|
193
|
+
to = _timeout or timeout
|
|
194
|
+
q = (query or "").strip()
|
|
195
|
+
if not q:
|
|
196
|
+
return []
|
|
197
|
+
items = _kev_load(to)
|
|
198
|
+
if not items:
|
|
199
|
+
return []
|
|
200
|
+
cve_q = q.upper() if re.match(r"^CVE-\d{4}-\d{4,}$", q.strip(), re.I) else ""
|
|
201
|
+
matched = []
|
|
202
|
+
for v in items:
|
|
203
|
+
if not isinstance(v, dict):
|
|
204
|
+
continue
|
|
205
|
+
if cve_q:
|
|
206
|
+
if str(v.get("cveID") or "").strip().upper() == cve_q:
|
|
207
|
+
matched.append(v)
|
|
208
|
+
continue
|
|
209
|
+
terms = [t for t in re.split(r"[\s,]+", q.lower()) if t]
|
|
210
|
+
hay = " ".join(str(v.get(k) or "") for k in (
|
|
211
|
+
"cveID", "vendorProject", "product", "vulnerabilityName",
|
|
212
|
+
"shortDescription", "requiredAction", "cwes")).lower()
|
|
213
|
+
if all(t in hay for t in terms):
|
|
214
|
+
matched.append(v)
|
|
215
|
+
matched.sort(key=lambda x: str(x.get("dateAdded") or ""), reverse=True)
|
|
216
|
+
results = []
|
|
217
|
+
for i, v in enumerate(matched[:n]):
|
|
218
|
+
cve = str(v.get("cveID") or "").strip()
|
|
219
|
+
if not cve:
|
|
220
|
+
continue
|
|
221
|
+
vendor = str(v.get("vendorProject") or "").strip()
|
|
222
|
+
product = str(v.get("product") or "").strip()
|
|
223
|
+
name = str(v.get("vulnerabilityName") or "").strip()
|
|
224
|
+
desc = str(v.get("shortDescription") or "").strip()
|
|
225
|
+
due = str(v.get("dueDate") or "").strip()
|
|
226
|
+
ransom = str(v.get("knownRansomwareCampaignUse") or "").strip()
|
|
227
|
+
bits = [b for b in (
|
|
228
|
+
f"{vendor} {product}".strip(),
|
|
229
|
+
f"修复期限 {due}" if due else "",
|
|
230
|
+
"已知勒索软件利用" if ransom.lower() == "known" else "",
|
|
231
|
+
desc,
|
|
232
|
+
) if b]
|
|
233
|
+
results.append({
|
|
234
|
+
"title": f"{cve} — {name or product or vendor}"[:500],
|
|
235
|
+
"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog",
|
|
236
|
+
"snippet": " · ".join(bits)[:300],
|
|
237
|
+
"source": "cisa_kev",
|
|
238
|
+
"published_at": str(v.get("dateAdded") or "")[:10],
|
|
239
|
+
"score": rank_score(0.9, i),
|
|
240
|
+
})
|
|
241
|
+
return results
|
|
242
|
+
return _engine
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# ── 引擎声明(本地个性化配置,不提交公开仓库)────────────────────────────────
|
|
246
|
+
#
|
|
247
|
+
# 字段与原 engines/specs/osv.yaml、engines/specs/cisa_kev.yaml 逐项对齐——
|
|
248
|
+
# 那两个声明已加 _ 前缀改名停用(公开仓每次启动刷两行「未知引擎类型」warning),
|
|
249
|
+
# 元数据随之并入本文件,本机路由行为(explicit_only / family / priority / qps
|
|
250
|
+
# 等)与改名前完全一致。
|
|
251
|
+
|
|
252
|
+
_LOCAL_ENGINE_SPECS: dict[str, dict[str, Any]] = {
|
|
253
|
+
"osv": {
|
|
254
|
+
"type": "osv",
|
|
255
|
+
"enabled": True,
|
|
256
|
+
"explicit_only": True,
|
|
257
|
+
"cost_tier": "free",
|
|
258
|
+
"label": "OSV 开源漏洞库",
|
|
259
|
+
"family": "security",
|
|
260
|
+
"priority": 44,
|
|
261
|
+
"qps": 2,
|
|
262
|
+
"period": "second",
|
|
263
|
+
"cost_per_call": 0.0,
|
|
264
|
+
"cost_unit": "free",
|
|
265
|
+
"method": "POST",
|
|
266
|
+
"url": "https://api.osv.dev/v1/query",
|
|
267
|
+
"timeout": 15,
|
|
268
|
+
"canary_query": "requests",
|
|
269
|
+
"desc": "OSV.dev 开源软件漏洞库(按包查已知漏洞与影响版本区间,免 key;生态 PyPI/npm/Go/crates 等)",
|
|
270
|
+
"description": "OSV.dev 开源漏洞查询(按包名)",
|
|
271
|
+
"coverage": ["security", "package"],
|
|
272
|
+
"recommended": False,
|
|
273
|
+
"latency_ms": 1500,
|
|
274
|
+
"headers": {"Content-Type": "application/json"},
|
|
275
|
+
"quality_queries": [
|
|
276
|
+
{"query": "requests"},
|
|
277
|
+
{"query": "lodash"},
|
|
278
|
+
{"query": "log4j-core"},
|
|
279
|
+
],
|
|
280
|
+
},
|
|
281
|
+
"cisa_kev": {
|
|
282
|
+
"type": "cisa_kev",
|
|
283
|
+
"enabled": True,
|
|
284
|
+
"explicit_only": True,
|
|
285
|
+
"cost_tier": "free",
|
|
286
|
+
"label": "CISA KEV 在野利用漏洞",
|
|
287
|
+
"family": "security",
|
|
288
|
+
"priority": 47,
|
|
289
|
+
"qps": 1,
|
|
290
|
+
"period": "second",
|
|
291
|
+
"cost_per_call": 0.0,
|
|
292
|
+
"cost_unit": "free",
|
|
293
|
+
"timeout": 30,
|
|
294
|
+
"canary_query": "chrome",
|
|
295
|
+
"desc": "CISA 已知被在野利用漏洞目录(官方确认已被实际攻击利用的 CVE;全量 1.35MB 本地过滤)",
|
|
296
|
+
"description": "CISA KEV 在野利用漏洞目录",
|
|
297
|
+
"coverage": ["security"],
|
|
298
|
+
"recommended": False,
|
|
299
|
+
"latency_ms": 4000,
|
|
300
|
+
"quality_queries": [
|
|
301
|
+
{"query": "chrome"},
|
|
302
|
+
{"query": "CVE-2021-44228"},
|
|
303
|
+
],
|
|
304
|
+
},
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def register(builders: dict[str, Any], engine_specs: dict[str, Any]) -> None:
|
|
309
|
+
"""将本地引擎注册到全局 builders 和 engine_specs 中。
|
|
310
|
+
|
|
311
|
+
在 engines.py 的 _load_registry() 中、构建 registry 之前调用。
|
|
312
|
+
文件不存在时 engines.py 跳过 import,不影响公开仓库。
|
|
313
|
+
外置 spec(engines/specs/*.yaml)已有的字段保留,不抹除 explicit_only/family 等元数据。
|
|
314
|
+
"""
|
|
315
|
+
builders["osv"] = _build_osv_engine
|
|
316
|
+
builders["cisa_kev"] = _build_cisa_kev_engine
|
|
317
|
+
for name, spec in _LOCAL_ENGINE_SPECS.items():
|
|
318
|
+
base = dict(engine_specs.get(name) or {})
|
|
319
|
+
for k, v in spec.items():
|
|
320
|
+
base.setdefault(k, v)
|
|
321
|
+
engine_specs[name] = base
|