argo-search 1.0.1 → 2.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +735 -0
- package/README.es.md +777 -0
- package/README.ja.md +777 -0
- package/README.ko.md +777 -0
- package/README.md +778 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2356 -96
- package/backends/engine_registry.yaml +3428 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2488 -160
- package/backends/source_types_cn.json +109 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +772 -0
- package/bin/argo.js +102 -39
- package/config.yaml +5139 -367
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +575 -0
- package/docs/MCP_SETUP.md +108 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +36 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +106 -0
- package/package.json +55 -12
- package/packages/dsh-plugin/README.md +152 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1533 -0
- package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
- package/packages/dsh-plugin/dsh/index.js +527 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +406 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +1382 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +294 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/article_meta.py +131 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1066 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +751 -0
- package/scripts/crawl.py +74 -35
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/dispatch_result.py +98 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +982 -0
- package/scripts/engine_env.py +539 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +738 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +799 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +310 -0
- package/scripts/engines_builders_batch10.py +204 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1359 -0
- package/scripts/evidence.py +620 -166
- package/scripts/evidence_loop.py +624 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1804 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +195 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +166 -0
- package/scripts/install.sh +221 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +402 -0
- package/scripts/macro_countries.py +97 -0
- package/scripts/matrix_search_eval.py +1187 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +492 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +124 -0
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +125 -419
- package/scripts/mcp_setup.py +431 -0
- package/scripts/mcp_tools.py +331 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/mcp_warm.py +127 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +633 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/recovery_runner.py +150 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +658 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +607 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +169 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/retry_policy.py +76 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +243 -0
- package/scripts/route_combo.py +917 -0
- package/scripts/route_domains.py +341 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +886 -406
- package/scripts/search_attribution.py +185 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +640 -0
- package/scripts/search_entry.py +395 -0
- package/scripts/search_output.py +690 -0
- package/scripts/search_pipeline.py +926 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +129 -16
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +391 -0
- package/scripts/serp_guard.py +336 -0
- package/scripts/serp_kill_log.py +74 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +107 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +447 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +186 -0
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +716 -242
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""digup_bridge.py — DigUp 本地语义搜索的 CLI 桥(声明式引擎用,无网络)。
|
|
3
|
+
|
|
4
|
+
DigUp(github.com/ARahim3/DigUp)在设备端用 EmbeddingGemma 2 给本地文件建
|
|
5
|
+
多模态向量索引(照片/截图/PDF/文档/音频/视频,命中精确到页/秒/行)。它的
|
|
6
|
+
`search --json` 输出的是 path/kind/score/excerpt 行,不是 argo 契约的
|
|
7
|
+
title/url——本桥做一次纯本地映射,让 engines/specs/digup.yaml 走通用 cli
|
|
8
|
+
builder,不新增 Python 特例分支(与 train/realtime_index 同形态)。
|
|
9
|
+
|
|
10
|
+
用法(spec 调用形如):
|
|
11
|
+
python3 scripts/digup_bridge.py "<query>" -n 5
|
|
12
|
+
|
|
13
|
+
digup 二进制定位(先到先得):
|
|
14
|
+
--bin 参数 > $DIGUP_BIN > PATH 里的 digup
|
|
15
|
+
> /Applications/DigUp.app/Contents/Helpers/digup(brew cask / DMG 装的 App)
|
|
16
|
+
|
|
17
|
+
索引定位:--index 参数 > $DIGUP_INDEX > digup 默认
|
|
18
|
+
(~/Library/Application Support/DigUp/index.noindex)。
|
|
19
|
+
|
|
20
|
+
输出:单个 JSON {"results": [...]}(argo cli builder 的自动 JSON 路径)。
|
|
21
|
+
任何失败都输出空 results 并 exit 0——引擎层语义:本源此刻无结果,而不是
|
|
22
|
+
整次检索报错。诊断信息走 stderr。
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import argparse
|
|
28
|
+
import datetime
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import shutil
|
|
32
|
+
import subprocess
|
|
33
|
+
import sys
|
|
34
|
+
|
|
35
|
+
# digup 自身首查要加载 ~250MB text-only 模型(冷启实测 0.4–0.8s),再留检索
|
|
36
|
+
# 与进程开销。引擎层 spec timeout=30 兜底,这里取小于它的值,保证被卡时
|
|
37
|
+
# 桥先返回结构化空结果而不是被引擎层整体杀掉。
|
|
38
|
+
_DIGUP_TIMEOUT_S = 25
|
|
39
|
+
|
|
40
|
+
_APP_HELPERS = (
|
|
41
|
+
"/Applications/DigUp.app/Contents/Helpers/digup",
|
|
42
|
+
os.path.expanduser("~/Applications/DigUp.app/Contents/Helpers/digup"),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
_KIND_ZH = {
|
|
46
|
+
"screenshot": "截图",
|
|
47
|
+
"image": "图片",
|
|
48
|
+
"pdf": "PDF",
|
|
49
|
+
"doc": "文档",
|
|
50
|
+
"audio": "音频",
|
|
51
|
+
"video": "视频",
|
|
52
|
+
"code": "代码",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_bin(explicit: str | None = None) -> str:
|
|
57
|
+
"""定位 digup 可执行文件(显式 > $DIGUP_BIN > PATH > App Helpers);找不到返回空串。"""
|
|
58
|
+
candidates = []
|
|
59
|
+
if explicit:
|
|
60
|
+
candidates.append(explicit)
|
|
61
|
+
env_bin = os.environ.get("DIGUP_BIN", "").strip()
|
|
62
|
+
if env_bin:
|
|
63
|
+
candidates.append(env_bin)
|
|
64
|
+
for path in candidates:
|
|
65
|
+
if path and os.path.isfile(path) and os.access(path, os.X_OK):
|
|
66
|
+
return path
|
|
67
|
+
found = shutil.which("digup")
|
|
68
|
+
if found:
|
|
69
|
+
return found
|
|
70
|
+
for path in _APP_HELPERS:
|
|
71
|
+
if os.path.isfile(path) and os.access(path, os.X_OK):
|
|
72
|
+
return path
|
|
73
|
+
return ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _clock(seconds: float) -> str:
|
|
77
|
+
total = int(seconds)
|
|
78
|
+
return f"{total // 60}:{total % 60:02d}"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _loc_hint(row: dict) -> str:
|
|
82
|
+
"""digup 的命中位置(页/秒/段/行)转中文后缀;无定位返回空。"""
|
|
83
|
+
segment = row.get("segment") or ""
|
|
84
|
+
loc = row.get("loc")
|
|
85
|
+
if loc is None:
|
|
86
|
+
return " · 图内文字" if segment == "ocr" else ""
|
|
87
|
+
if segment in ("page", "pageImage"):
|
|
88
|
+
return f" · 第{int(loc)}页"
|
|
89
|
+
if segment == "audio":
|
|
90
|
+
return f" · {_clock(loc)}(声音)"
|
|
91
|
+
if segment == "frame":
|
|
92
|
+
return f" · {_clock(loc)}"
|
|
93
|
+
if segment == "chunk":
|
|
94
|
+
return f" · 第{int(loc)}段"
|
|
95
|
+
if segment == "lines":
|
|
96
|
+
end = row.get("loc_end")
|
|
97
|
+
return f" · 第{int(loc)}–{int(end)}行" if end else f" · 第{int(loc)}行"
|
|
98
|
+
if segment == "cells":
|
|
99
|
+
end = row.get("loc_end")
|
|
100
|
+
return f" · 单元格{int(loc)}–{int(end)}" if end else f" · 单元格{int(loc)}"
|
|
101
|
+
return ""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _url(row: dict) -> str:
|
|
105
|
+
"""file:// 链接;PDF 带 #page=N、音视频带 #t=start,end(Preview/QuickTime
|
|
106
|
+
认这两个 fragment,也让同文件不同页/时刻的命中不被 argo 按 url 去重合并)。"""
|
|
107
|
+
path = row.get("path") or ""
|
|
108
|
+
if not path:
|
|
109
|
+
return ""
|
|
110
|
+
url = "file://" + path
|
|
111
|
+
segment = row.get("segment") or ""
|
|
112
|
+
loc = row.get("loc")
|
|
113
|
+
if loc is None:
|
|
114
|
+
return url
|
|
115
|
+
if segment in ("page", "pageImage"):
|
|
116
|
+
return f"{url}#page={int(loc)}"
|
|
117
|
+
if segment in ("audio", "frame"):
|
|
118
|
+
end = row.get("loc_end")
|
|
119
|
+
return f"{url}#t={int(loc)},{int(end if end is not None else loc)}"
|
|
120
|
+
return url
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _snippet(row: dict) -> str:
|
|
124
|
+
"""excerpt 优先(剥掉 digup 的高亮记号 «»);无 excerpt(图像/纯名命中)
|
|
125
|
+
给「类型 · 位置描述」或所在目录,保证 snippet 永远不为空。"""
|
|
126
|
+
excerpt = str(row.get("excerpt") or "").replace("«", "").replace("»", "")
|
|
127
|
+
excerpt = " ".join(excerpt.split())
|
|
128
|
+
if excerpt:
|
|
129
|
+
return excerpt[:300]
|
|
130
|
+
kind = _KIND_ZH.get(row.get("kind") or "", "文件")
|
|
131
|
+
hint = _loc_hint(row)
|
|
132
|
+
if hint:
|
|
133
|
+
return f"{kind}{hint.lstrip(' ·')} · 内容语义匹配"
|
|
134
|
+
folder = os.path.dirname(row.get("path") or "")
|
|
135
|
+
home = os.path.expanduser("~")
|
|
136
|
+
if folder.startswith(home):
|
|
137
|
+
folder = "~" + folder[len(home):]
|
|
138
|
+
return f"{kind} · 关键词命中 · {folder or '本地文件'}"[:300]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def map_row(row: dict, rank: int) -> dict:
|
|
142
|
+
"""digup 结果行 → argo 结果契约(title/url/snippet/source 必需)。"""
|
|
143
|
+
path = row.get("path") or ""
|
|
144
|
+
title = (os.path.basename(path) if path else "") + _loc_hint(row)
|
|
145
|
+
result = {
|
|
146
|
+
"title": title[:200],
|
|
147
|
+
"url": _url(row),
|
|
148
|
+
"snippet": _snippet(row),
|
|
149
|
+
"source": "digup",
|
|
150
|
+
# 与其他 cli builder 一致的展示分(rrf_merge 按名次融合,score 只作展示)
|
|
151
|
+
"score": round(max(1.0 - 0.1 * rank, 0.1), 4),
|
|
152
|
+
}
|
|
153
|
+
# digup --json 不带 modified(SearchHit 序列化面没有它),桥自己 stat:
|
|
154
|
+
# 文件被移动/删除时拿不到就算了,不硬造日期。
|
|
155
|
+
if path:
|
|
156
|
+
try:
|
|
157
|
+
mtime = os.stat(path).st_mtime
|
|
158
|
+
result["published_at"] = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d")
|
|
159
|
+
except OSError:
|
|
160
|
+
pass
|
|
161
|
+
return result
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def search(query: str, n: int, bin_path: str, index_dir: str | None) -> list[dict]:
|
|
165
|
+
cmd = [bin_path, "search", query, "--json", "-n", str(n)]
|
|
166
|
+
if index_dir:
|
|
167
|
+
cmd += ["--index", index_dir]
|
|
168
|
+
try:
|
|
169
|
+
proc = subprocess.run(cmd, capture_output=True, text=True,
|
|
170
|
+
encoding="utf-8", errors="replace",
|
|
171
|
+
timeout=_DIGUP_TIMEOUT_S)
|
|
172
|
+
except subprocess.TimeoutExpired:
|
|
173
|
+
print(f"digup_bridge: digup 超时(>{_DIGUP_TIMEOUT_S}s)", file=sys.stderr)
|
|
174
|
+
return []
|
|
175
|
+
except OSError as e:
|
|
176
|
+
print(f"digup_bridge: 无法执行 digup: {e}", file=sys.stderr)
|
|
177
|
+
return []
|
|
178
|
+
if proc.returncode != 0:
|
|
179
|
+
tail = (proc.stderr or "").strip()[-200:]
|
|
180
|
+
print(f"digup_bridge: digup rc={proc.returncode}: {tail}", file=sys.stderr)
|
|
181
|
+
return []
|
|
182
|
+
try:
|
|
183
|
+
rows = json.loads(proc.stdout)
|
|
184
|
+
except (json.JSONDecodeError, ValueError):
|
|
185
|
+
print("digup_bridge: digup 输出不是 JSON", file=sys.stderr)
|
|
186
|
+
return []
|
|
187
|
+
if not isinstance(rows, list):
|
|
188
|
+
print("digup_bridge: digup 输出不是结果数组", file=sys.stderr)
|
|
189
|
+
return []
|
|
190
|
+
return [map_row(r, i) for i, r in enumerate(rows) if isinstance(r, dict)]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def main(argv: list[str] | None = None) -> int:
|
|
194
|
+
parser = argparse.ArgumentParser(description="DigUp → argo 结果契约(本地映射,无网络)")
|
|
195
|
+
parser.add_argument("query", help="搜索词(自然语言描述文件内容)")
|
|
196
|
+
parser.add_argument("-n", type=int, default=5, help="返回条数")
|
|
197
|
+
parser.add_argument("--bin", default=None, help="digup 可执行文件路径(默认自动定位)")
|
|
198
|
+
parser.add_argument("--index", default=None, help="索引目录(默认 digup 自身默认值)")
|
|
199
|
+
args = parser.parse_args(argv)
|
|
200
|
+
|
|
201
|
+
bin_path = resolve_bin(args.bin)
|
|
202
|
+
if not bin_path:
|
|
203
|
+
print("digup_bridge: 找不到 digup 可执行文件(装 App、放进 PATH,"
|
|
204
|
+
"或设 DIGUP_BIN / --bin),返回空结果", file=sys.stderr)
|
|
205
|
+
print(json.dumps({"results": []}))
|
|
206
|
+
return 0
|
|
207
|
+
index_dir = args.index or os.environ.get("DIGUP_INDEX", "").strip() or None
|
|
208
|
+
print(json.dumps({"results": search(args.query, max(1, args.n), bin_path, index_dir)},
|
|
209
|
+
ensure_ascii=False))
|
|
210
|
+
return 0
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
if __name__ == "__main__":
|
|
214
|
+
sys.exit(main())
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""dispatch_result.py — 引擎编排的产出对象与墙钟记账(2026-10-21 从 engine_dispatch 拆出)。
|
|
3
|
+
|
|
4
|
+
拆出原因:module size 门禁要求 engine_dispatch 只能减不能增,而 2026-10-21 的
|
|
5
|
+
尾等待记账需要同时扩数据类与汇总逻辑。两者都是**纯数据**(无 IO、无调度依赖),
|
|
6
|
+
放在调度器里只会让「调度策略」和「记账口径」继续互相遮挡。
|
|
7
|
+
|
|
8
|
+
记账口径(为什么不是一个数):
|
|
9
|
+
- `useful_ms`:最后一个**贡献结果**的引擎完成时刻(答案从这一刻起已在手里)。
|
|
10
|
+
- `first_sufficient_ms`:早停判据首次成立(结果够用)的时刻;no_early 域没有
|
|
11
|
+
这个时刻,保持 0。
|
|
12
|
+
- `tail_wait_ms`:结果已可交付后仍在等的时间。有早停时刻用它算,没有就用
|
|
13
|
+
「首条结果时刻」算——后者正是 no_early 域白等的来源。
|
|
14
|
+
- `abandoned_engine_ms`:被早停弃置的引擎已花的墙钟**之和**。它是并行资源
|
|
15
|
+
浪费,不等于用户多等的时间(两个引擎并行跑到最后,赢家交结果、输家被弃置,
|
|
16
|
+
用户没有任何额外等待)。`wasted_ms` 保留为它的别名,兼容旧读法。
|
|
17
|
+
|
|
18
|
+
历史教训:旧 `wasted_ms` 口径换过三次,前两版分别是「非 ok 引擎 latency 之和」
|
|
19
|
+
(并行时必然大于墙钟)与 `wall - useful`(有界并发下恒为 0)。两版都让仪表看
|
|
20
|
+
起来正常、却无法指向任何动作。这次拆成两个互不重叠的量,各自对应一个旋钮:
|
|
21
|
+
tail 指向收工时机,abandoned 指向并发/对冲预算。
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
__all__ = ["DispatchResult", "summarize_dispatch"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DispatchResult:
|
|
32
|
+
"""编排产出 + 两个钩子(融合后仍要补搜的路径用)。
|
|
33
|
+
|
|
34
|
+
`run_one` / `ingest` 必须外露:macro_data 域的 D6 证据下限补搜在融合之后,
|
|
35
|
+
它要跑单个引擎并把结果并进同一份账(raw_results / engine_outcomes /
|
|
36
|
+
engine_latency / abandoned 计数),另起一份会让 engines_used 与配额记账对不上。
|
|
37
|
+
|
|
38
|
+
`budget_used_ms` / `budget_total_ms`:本次编排的实际墙钟消耗与总预算
|
|
39
|
+
(deep 等无预算模式 total 为 None)。搜索输出把它挂进 timing.budget,
|
|
40
|
+
让「这次离预算上限还有多远」可观测,而不必反推 process_ms。
|
|
41
|
+
|
|
42
|
+
记账字段的语义与历史见模块 docstring;`wasted_ms` 仅为兼容别名。
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
__slots__ = ("raw_results", "engine_outcomes", "engine_latency",
|
|
46
|
+
"wasted_ms", "early_stopped", "run_one", "ingest",
|
|
47
|
+
"budget_used_ms", "budget_total_ms", "useful_ms",
|
|
48
|
+
"first_sufficient_ms", "tail_wait_ms", "abandoned_engine_ms")
|
|
49
|
+
|
|
50
|
+
def __init__(self, raw_results: Any, engine_outcomes: Any,
|
|
51
|
+
engine_latency: Any, wasted_ms: int, early_stopped: bool,
|
|
52
|
+
run_one: Any, ingest: Any, budget_used_ms: int | None = None,
|
|
53
|
+
budget_total_ms: int | None = None, useful_ms: int = 0,
|
|
54
|
+
first_sufficient_ms: int = 0, tail_wait_ms: int = 0,
|
|
55
|
+
abandoned_engine_ms: int | None = None) -> None:
|
|
56
|
+
self.raw_results = raw_results
|
|
57
|
+
self.engine_outcomes = engine_outcomes
|
|
58
|
+
self.engine_latency = engine_latency
|
|
59
|
+
self.wasted_ms = wasted_ms
|
|
60
|
+
self.early_stopped = early_stopped
|
|
61
|
+
self.run_one = run_one
|
|
62
|
+
self.ingest = ingest
|
|
63
|
+
self.budget_used_ms = budget_used_ms
|
|
64
|
+
self.budget_total_ms = budget_total_ms
|
|
65
|
+
self.useful_ms = useful_ms
|
|
66
|
+
self.first_sufficient_ms = first_sufficient_ms
|
|
67
|
+
self.tail_wait_ms = tail_wait_ms
|
|
68
|
+
# 旧调用方只传 wasted_ms:别名取同一值,不引入第二个真源。
|
|
69
|
+
self.abandoned_engine_ms = (wasted_ms if abandoned_engine_ms is None
|
|
70
|
+
else abandoned_engine_ms)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def summarize_dispatch(*, wall_ms: int, contributed_ms: list[float],
|
|
74
|
+
first_result_at: float | None,
|
|
75
|
+
first_sufficient_at: float | None,
|
|
76
|
+
base: float, abandoned_ms: list[float]) -> dict[str, int]:
|
|
77
|
+
"""把调度器的事件时刻折算成对外记账字段(纯函数,便于单测)。
|
|
78
|
+
|
|
79
|
+
`base` 是本次编排的单调钟起点;两个 `*_at` 都是绝对时刻。没有可交付结果
|
|
80
|
+
(全失败)时尾等待记 0:没有东西可交付,那段等待不是「可省的」。
|
|
81
|
+
"""
|
|
82
|
+
useful_ms = int(min(max(contributed_ms), wall_ms)) if contributed_ms else 0
|
|
83
|
+
|
|
84
|
+
def _rel(at: float | None) -> int:
|
|
85
|
+
if at is None:
|
|
86
|
+
return 0
|
|
87
|
+
return int(min(max(at - base, 0.0), wall_ms / 1000.0) * 1000)
|
|
88
|
+
|
|
89
|
+
first_result_ms = _rel(first_result_at)
|
|
90
|
+
first_sufficient_ms = _rel(first_sufficient_at)
|
|
91
|
+
anchor = first_sufficient_ms or first_result_ms
|
|
92
|
+
return {
|
|
93
|
+
"useful_ms": useful_ms,
|
|
94
|
+
"first_result_ms": first_result_ms,
|
|
95
|
+
"first_sufficient_ms": first_sufficient_ms,
|
|
96
|
+
"tail_wait_ms": max(0, wall_ms - anchor) if anchor else 0,
|
|
97
|
+
"abandoned_engine_ms": int(sum(abandoned_ms)),
|
|
98
|
+
}
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""engine_admission.py — 引擎生产准入状态
|
|
3
|
+
|
|
4
|
+
状态文件目录(由 argo_paths 派生,ARGO_ADMISSION_DIR 可覆盖):
|
|
5
|
+
<状态目录>/admission/<engine_id>.json
|
|
6
|
+
|
|
7
|
+
字段:
|
|
8
|
+
engine_id, admitted_at, stages_passed, quality_score,
|
|
9
|
+
avg_latency_ms, blocked, reason, updated_at
|
|
10
|
+
|
|
11
|
+
路由规则:
|
|
12
|
+
- blocked=true → 自动路由跳过
|
|
13
|
+
- 用户 --engine 强制指定仍可调用(调试)
|
|
14
|
+
- 无 admission 文件 → 默认放行(向后兼容存量引擎)
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import time
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from cli_io import dumps_pretty
|
|
27
|
+
|
|
28
|
+
# 本地状态目录唯一来源(env ARGO_STATE_DIR → config cache.db_path 父目录 → 旧路径)
|
|
29
|
+
import argo_paths as _paths
|
|
30
|
+
|
|
31
|
+
# ARGO_ADMISSION_DIR 优先(测试隔离);未设置时由唯一来源派生目录
|
|
32
|
+
DEFAULT_ADMISSION_DIR = Path(
|
|
33
|
+
os.path.expanduser(os.environ.get(
|
|
34
|
+
"ARGO_ADMISSION_DIR",
|
|
35
|
+
str(_paths.state_path("admission")),
|
|
36
|
+
))
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def admission_dir() -> Path:
|
|
41
|
+
# 保持每次重复执行结果一致确保目录存在:状态目录会被测试用 ARGO_STATE_DIR 隔离,进程内也
|
|
42
|
+
# 可能切换,缓存目录反而容易写到已被清理的旧目录。exist_ok=True 在目录已存在
|
|
43
|
+
# 时只是一次很便宜的路径查询,真正的重复大头(读记录前先 exists 再 read)已在
|
|
44
|
+
# load_admission 里去掉。
|
|
45
|
+
d = DEFAULT_ADMISSION_DIR
|
|
46
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
return d
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _admission_stem(engine_id: str) -> str:
|
|
51
|
+
"""engine_id → 记录文件名(去 .json)。读写共用一处转义,不得各写各的。"""
|
|
52
|
+
return engine_id.replace("/", "_").replace("..", "_")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def admission_path(engine_id: str) -> Path:
|
|
56
|
+
return admission_dir() / f"{_admission_stem(engine_id)}.json"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _read_path(engine_id: str) -> Path:
|
|
60
|
+
"""读记录用的路径,不触发 mkdir。
|
|
61
|
+
|
|
62
|
+
读一个还不存在的记录根本不需要先建目录;此前 load_admission 经
|
|
63
|
+
admission_path 每次都 mkdir(parents=True),一次搜索因此多出约两百次
|
|
64
|
+
路径查询。写记录(save_admission 等)仍走 admission_path 确保目录存在。
|
|
65
|
+
"""
|
|
66
|
+
return DEFAULT_ADMISSION_DIR / f"{_admission_stem(engine_id)}.json"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _now_iso() -> str:
|
|
70
|
+
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# 读缓存:一次 routable 扫描会对每个引擎问一次 is_blocked(),此前每次都裸读盘
|
|
74
|
+
# (实测一次扫描 668 次 read_text / 41 ms;同一份记录被读 3 次——is_blocked、
|
|
75
|
+
# is_admitted、引擎详情各来一遍)。按文件路径记忆,**写入路径立即失效**:同
|
|
76
|
+
# 进程读己所写永远新鲜(record_validation → load_admission 的测试序列就靠这条
|
|
77
|
+
# 保证);别的进程改了记录则由 TTL 保底(默认 1 s,ARGO_ADMISSION_TTL_S 可调,
|
|
78
|
+
# 设 0 关闭记忆、回到逐次读盘)。
|
|
79
|
+
#
|
|
80
|
+
# 键用路径字符串而非 engine_id:状态目录可被 ARGO_STATE_DIR / ARGO_ADMISSION_DIR
|
|
81
|
+
# 在进程内切换(测试隔离的常规手段),按 engine_id 记忆会把「A 目录下没有记录」
|
|
82
|
+
# 错认成「B 目录下也没有」。
|
|
83
|
+
_admission_read_cache: dict[str, tuple[float, dict[str, Any] | None]] = {}
|
|
84
|
+
|
|
85
|
+
# env 文件回退路径的记忆化:(单调钟读数, 解析值)。
|
|
86
|
+
#
|
|
87
|
+
# 只记忆「env 文件」这一条,不碰 os.environ 直读——后者是字典查找、本就免费,
|
|
88
|
+
# 而且必须逐次生效(本旋钮的测试与 CI 都靠 setenv 控制,记忆化会把 setenv 吞掉,
|
|
89
|
+
# 例如 test_ttl_zero_disables_cache)。贵的是回退路径:engine_env.get_env →
|
|
90
|
+
# _envfile_load → 对候选密钥文件逐个 stat。
|
|
91
|
+
_TTL_MEMO_S = 1.0
|
|
92
|
+
_ttl_envfile_memo: tuple[float, float] | None = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _parse_ttl(raw: str) -> float:
|
|
96
|
+
"""TTL 字符串 → 秒。空值或非法值统一回落 1.0(唯一判据,两处共用)。"""
|
|
97
|
+
try:
|
|
98
|
+
return float(raw or 1.0)
|
|
99
|
+
except ValueError:
|
|
100
|
+
return 1.0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _envfile_ttl() -> float:
|
|
104
|
+
"""从 env 文件解析 TTL,按 _TTL_MEMO_S 秒记忆化。
|
|
105
|
+
|
|
106
|
+
本函数在**每个引擎**的准入判定里都会被调用(实测一次路由决策 215 次),
|
|
107
|
+
而回退路径每次都要穿透到 engine_env.get_env → _envfile_load → 对候选密钥
|
|
108
|
+
文件逐个 stat。同一个进程级常量被解 215 遍,实测占准入扫描的三分之一
|
|
109
|
+
(2.79ms → 1.49ms,1.87×)。窗口取 1 秒与「本模块的读缓存默认 TTL」同量级:
|
|
110
|
+
改 env 文件最多 1 秒后生效,热更语义仍成立。
|
|
111
|
+
"""
|
|
112
|
+
global _ttl_envfile_memo
|
|
113
|
+
now = time.monotonic()
|
|
114
|
+
hit = _ttl_envfile_memo
|
|
115
|
+
if hit is not None and now - hit[0] < _TTL_MEMO_S:
|
|
116
|
+
return hit[1]
|
|
117
|
+
try:
|
|
118
|
+
from engine_env import get_env
|
|
119
|
+
raw = get_env("ARGO_ADMISSION_TTL_S")
|
|
120
|
+
except Exception:
|
|
121
|
+
raw = ""
|
|
122
|
+
value = _parse_ttl(raw)
|
|
123
|
+
_ttl_envfile_memo = (now, value)
|
|
124
|
+
return value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _admission_ttl() -> float:
|
|
128
|
+
"""读缓存的 TTL(秒);未设置或非法值回落 1.0。
|
|
129
|
+
|
|
130
|
+
走 engine_env.get_env 而不是 os.environ 直读:开关写进 ~/.config/argo/env
|
|
131
|
+
也要生效——与 config._stamp_ttl 同一计算方式(同一个旋钮不该有两套可读位置)。
|
|
132
|
+
|
|
133
|
+
os.environ 命中时逐次直读(零成本、setenv 立即生效);只有回退到 env 文件的
|
|
134
|
+
那条路径走记忆化,见 _envfile_ttl。
|
|
135
|
+
"""
|
|
136
|
+
raw = os.environ.get("ARGO_ADMISSION_TTL_S", "")
|
|
137
|
+
if raw.strip():
|
|
138
|
+
return _parse_ttl(raw)
|
|
139
|
+
return _envfile_ttl()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _read_admission_file(path: Path) -> dict[str, Any] | None:
|
|
143
|
+
"""裸读一份准入记录(无缓存)。"""
|
|
144
|
+
# 直接读、靠异常判断文件是否存在:既不先 exists(),也不先建目录。
|
|
145
|
+
# 文件/目录不存在时 read_text 抛 FileNotFoundError(OSError 子类),一并保底。
|
|
146
|
+
try:
|
|
147
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
148
|
+
return data if isinstance(data, dict) else None
|
|
149
|
+
except (OSError, json.JSONDecodeError, ValueError):
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# 存在集合:一次 scandir 回答「哪些引擎有记录」,替代逐引擎 open 探测。
|
|
154
|
+
# 动机(2026-10-01 实测):261 引擎只有 99 份记录,全量清单/route 扫描的
|
|
155
|
+
# 261 次准入 open 里 162 次注定 ENOENT(62%)——本机 SSD 全扫描 2.8ms 无感,
|
|
156
|
+
# 但慢 IO 环境(沙箱/CI/网络盘)单次 syscall 可放大到毫秒级,同一清单 4s+。
|
|
157
|
+
# 集合只回答「有没有」,记录内容仍走读缓存逐条读:集合内引擎的读取行为与
|
|
158
|
+
# 旧实现逐位一致,减少的只有注定扑空的那部分 open。
|
|
159
|
+
#
|
|
160
|
+
# 语义与读缓存同一条 TTL(_admission_ttl,ARGO_ADMISSION_TTL_S=0 一并关闭、
|
|
161
|
+
# 回到逐次精确读):窗口内别的进程新建/删除记录最多晚 TTL 秒可见,与读缓存
|
|
162
|
+
# 的跨进程新鲜度同一口径。键按目录字符串——状态目录可被测试在进程内切换,
|
|
163
|
+
# A 目录扫出的「无记录」不得带进 B 目录。写路径 save_admission 写完即把本
|
|
164
|
+
# 引擎并入集合:同进程读己所写不等 TTL。load_admission_fresh 不走集合——
|
|
165
|
+
# 读-改-写必须看到磁盘真相,绕过一切记忆。
|
|
166
|
+
_admission_dir_set_cache: dict[str, tuple[float, frozenset[str]]] = {}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _admission_exists_set() -> frozenset[str]:
|
|
170
|
+
"""有记录的引擎文件名(去 .json)集合,按 TTL 重扫目录。"""
|
|
171
|
+
key = str(DEFAULT_ADMISSION_DIR)
|
|
172
|
+
now = time.monotonic()
|
|
173
|
+
hit = _admission_dir_set_cache.get(key)
|
|
174
|
+
if hit is not None and now - hit[0] < _admission_ttl():
|
|
175
|
+
return hit[1]
|
|
176
|
+
try:
|
|
177
|
+
stems = frozenset(
|
|
178
|
+
entry.name[: -len(".json")]
|
|
179
|
+
for entry in os.scandir(DEFAULT_ADMISSION_DIR)
|
|
180
|
+
if entry.name.endswith(".json")
|
|
181
|
+
)
|
|
182
|
+
except OSError:
|
|
183
|
+
stems = frozenset()
|
|
184
|
+
_admission_dir_set_cache[key] = (now, stems)
|
|
185
|
+
return stems
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _admission_dir_set_add(dir_key: str, stem: str) -> None:
|
|
189
|
+
"""把刚写入的记录并入已缓存的集合;无缓存则不动(下次扫描自然包含)。"""
|
|
190
|
+
hit = _admission_dir_set_cache.get(dir_key)
|
|
191
|
+
if hit is not None:
|
|
192
|
+
_admission_dir_set_cache[dir_key] = (hit[0], hit[1] | {stem})
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def load_admission(engine_id: str) -> dict[str, Any] | None:
|
|
196
|
+
"""读一份准入记录(带进程内 TTL 缓存,见 _admission_read_cache)。
|
|
197
|
+
|
|
198
|
+
**只用于读路径**(路由判定、状态展示)。读-改-写的调用方要用
|
|
199
|
+
load_admission_fresh:TTL 窗口里的旧值会被原样写回,吞掉别的进程刚写入的
|
|
200
|
+
字段(审计实测:子进程写入 stages_passed 的 quality,父进程在 TTL 内读-改-写
|
|
201
|
+
后只剩 health,quality 丢失)。
|
|
202
|
+
"""
|
|
203
|
+
path = _read_path(engine_id)
|
|
204
|
+
ttl = _admission_ttl()
|
|
205
|
+
now = time.monotonic()
|
|
206
|
+
if ttl > 0:
|
|
207
|
+
hit = _admission_read_cache.get(str(path))
|
|
208
|
+
if hit is not None and now - hit[0] < ttl:
|
|
209
|
+
return hit[1]
|
|
210
|
+
# 存在集合短路:目录里没有这个引擎的记录,连 open 都不做(62% 注定
|
|
211
|
+
# 扑空的那部分)。扑空结果与旧实现一样进读缓存,同 TTL 新鲜度。
|
|
212
|
+
if _admission_stem(engine_id) not in _admission_exists_set():
|
|
213
|
+
_admission_read_cache[str(path)] = (now, None)
|
|
214
|
+
return None
|
|
215
|
+
data = _read_admission_file(path)
|
|
216
|
+
if ttl > 0:
|
|
217
|
+
_admission_read_cache[str(path)] = (now, data)
|
|
218
|
+
return data
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def load_admission_fresh(engine_id: str) -> dict[str, Any] | None:
|
|
222
|
+
"""读一份准入记录并刷新缓存——读-改-写序列的读端专用。
|
|
223
|
+
|
|
224
|
+
写路径必须看到磁盘上的最新状态:缓存的旧值会让「读旧 → 合并 → 写回」丢掉
|
|
225
|
+
别的进程刚写进去的字段,而准入记录是路由的输入,丢字段会直接改变路由结果。
|
|
226
|
+
"""
|
|
227
|
+
path = _read_path(engine_id)
|
|
228
|
+
data = _read_admission_file(path)
|
|
229
|
+
if _admission_ttl() > 0:
|
|
230
|
+
_admission_read_cache[str(path)] = (time.monotonic(), data)
|
|
231
|
+
return data
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def save_admission(engine_id: str, record: dict[str, Any]) -> dict[str, Any]:
|
|
235
|
+
path = admission_path(engine_id)
|
|
236
|
+
out = {
|
|
237
|
+
"engine_id": engine_id,
|
|
238
|
+
"admitted_at": record.get("admitted_at"),
|
|
239
|
+
"stages_passed": list(record.get("stages_passed") or []),
|
|
240
|
+
"quality_score": record.get("quality_score"),
|
|
241
|
+
"avg_latency_ms": record.get("avg_latency_ms"),
|
|
242
|
+
"blocked": bool(record.get("blocked", False)),
|
|
243
|
+
"reason": record.get("reason") or "",
|
|
244
|
+
"updated_at": _now_iso(),
|
|
245
|
+
"health": record.get("health"),
|
|
246
|
+
"quality": record.get("quality"),
|
|
247
|
+
}
|
|
248
|
+
path.write_text(dumps_pretty(out), encoding="utf-8")
|
|
249
|
+
# 写完即失效:缓存与文件是一对状态,只更新一半会让本进程读到自己没写的旧值
|
|
250
|
+
_admission_read_cache.pop(str(path), None)
|
|
251
|
+
# 存在集合同步并入:新记录不等 TTL 就可读(读己所写)
|
|
252
|
+
_admission_dir_set_add(str(path.parent), path.stem)
|
|
253
|
+
return out
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def set_blocked(engine_id: str, blocked: bool, reason: str = "") -> dict[str, Any]:
|
|
257
|
+
# 读端用 fresh:这是读-改-写序列,缓存里的旧值会被原样写回,吞掉别的进程
|
|
258
|
+
# 刚写进去的字段(TTL 窗口内的丢更新)
|
|
259
|
+
current = load_admission_fresh(engine_id) or {"engine_id": engine_id, "stages_passed": []}
|
|
260
|
+
current["blocked"] = blocked
|
|
261
|
+
current["reason"] = reason
|
|
262
|
+
if not blocked and not current.get("admitted_at"):
|
|
263
|
+
current["admitted_at"] = _now_iso()
|
|
264
|
+
return save_admission(engine_id, current)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def record_validation(
|
|
268
|
+
engine_id: str,
|
|
269
|
+
*,
|
|
270
|
+
stages_passed: list[str],
|
|
271
|
+
quality_score: float | None = None,
|
|
272
|
+
avg_latency_ms: float | None = None,
|
|
273
|
+
blocked: bool | None = None,
|
|
274
|
+
reason: str = "",
|
|
275
|
+
health: dict[str, Any] | None = None,
|
|
276
|
+
quality: dict[str, Any] | None = None,
|
|
277
|
+
admit: bool = False,
|
|
278
|
+
) -> dict[str, Any]:
|
|
279
|
+
"""写入验证结果;admit=True 且未 blocked 时标记准入时间。"""
|
|
280
|
+
# 读-改-写:读端必须 fresh,否则 TTL 内的旧值会把并发写入的字段吞掉
|
|
281
|
+
current = load_admission_fresh(engine_id) or {}
|
|
282
|
+
merged_stages = list(dict.fromkeys(
|
|
283
|
+
list(current.get("stages_passed") or []) + list(stages_passed or [])
|
|
284
|
+
))
|
|
285
|
+
if blocked is None:
|
|
286
|
+
# 默认:health 失败则 block;通过则 unblock
|
|
287
|
+
blocked = "health" not in stages_passed and bool(stages_passed)
|
|
288
|
+
if "health" in (stages_passed or []):
|
|
289
|
+
blocked = False
|
|
290
|
+
if health is not None and not health.get("ok", False) and health.get("status") != "skipped":
|
|
291
|
+
blocked = True
|
|
292
|
+
|
|
293
|
+
admitted_at = current.get("admitted_at")
|
|
294
|
+
if admit and not blocked:
|
|
295
|
+
admitted_at = _now_iso()
|
|
296
|
+
elif blocked:
|
|
297
|
+
# 保持历史 admitted_at,但 blocked 生效
|
|
298
|
+
pass
|
|
299
|
+
|
|
300
|
+
# reason 必须反映**本次**判定,不得粘滞历史原因。
|
|
301
|
+
# 此前写 `reason or current.get("reason")`:先跑 `--stage health` 失败会留下
|
|
302
|
+
# `health_failed`;之后补跑 `--stage quality`(health 已并入 merged_stages、
|
|
303
|
+
# 本次 health.ok=true 且 blocked=False)时本次 reason 为空 → 回退到历史
|
|
304
|
+
# `health_failed`,与 blocked=False 自相矛盾。引擎详情页据此显示
|
|
305
|
+
# 「blocked=true / reason=health_failed」而 health.status=pass,导致
|
|
306
|
+
# routable=False——实测 24 个引擎(批次九全部 + realtime_index)因此被
|
|
307
|
+
# 错误拉黑、装了没通电。
|
|
308
|
+
if reason:
|
|
309
|
+
eff_reason = reason
|
|
310
|
+
elif blocked:
|
|
311
|
+
eff_reason = "validation_failed"
|
|
312
|
+
else:
|
|
313
|
+
# 本次未 block:清掉历史失败原因,避免与 blocked=False 矛盾
|
|
314
|
+
eff_reason = ""
|
|
315
|
+
|
|
316
|
+
record = {
|
|
317
|
+
"admitted_at": admitted_at,
|
|
318
|
+
"stages_passed": merged_stages,
|
|
319
|
+
"quality_score": quality_score if quality_score is not None else current.get("quality_score"),
|
|
320
|
+
"avg_latency_ms": avg_latency_ms if avg_latency_ms is not None else current.get("avg_latency_ms"),
|
|
321
|
+
"blocked": blocked,
|
|
322
|
+
"reason": eff_reason,
|
|
323
|
+
"health": health if health is not None else current.get("health"),
|
|
324
|
+
"quality": quality if quality is not None else current.get("quality"),
|
|
325
|
+
}
|
|
326
|
+
return save_admission(engine_id, record)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def is_blocked(engine_id: str, default: bool = False) -> bool:
|
|
330
|
+
"""是否被准入系统拉黑。无记录时 default=False(兼容存量)。"""
|
|
331
|
+
rec = load_admission(engine_id)
|
|
332
|
+
if rec is None:
|
|
333
|
+
return default
|
|
334
|
+
return bool(rec.get("blocked", False))
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def is_admitted(engine_id: str) -> bool:
|
|
338
|
+
rec = load_admission(engine_id)
|
|
339
|
+
if rec is None:
|
|
340
|
+
return False
|
|
341
|
+
return bool(rec.get("admitted_at")) and not bool(rec.get("blocked"))
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def list_admissions() -> dict[str, dict[str, Any]]:
|
|
345
|
+
result: dict[str, dict[str, Any]] = {}
|
|
346
|
+
d = admission_dir()
|
|
347
|
+
for path in sorted(d.glob("*.json")):
|
|
348
|
+
try:
|
|
349
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
350
|
+
if isinstance(data, dict) and data.get("engine_id"):
|
|
351
|
+
result[data["engine_id"]] = data
|
|
352
|
+
else:
|
|
353
|
+
result[path.stem] = data if isinstance(data, dict) else {}
|
|
354
|
+
except Exception:
|
|
355
|
+
continue
|
|
356
|
+
return result
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def filter_routable(engine_ids: list[str] | set[str]) -> list[str]:
|
|
360
|
+
"""过滤掉 blocked 引擎,保持原顺序。"""
|
|
361
|
+
out = []
|
|
362
|
+
for e in engine_ids:
|
|
363
|
+
if is_blocked(e):
|
|
364
|
+
continue
|
|
365
|
+
out.append(e)
|
|
366
|
+
return out
|