argo-search 1.0.1 → 2.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +735 -0
- package/README.es.md +777 -0
- package/README.ja.md +777 -0
- package/README.ko.md +777 -0
- package/README.md +778 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2356 -96
- package/backends/engine_registry.yaml +3428 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2488 -160
- package/backends/source_types_cn.json +109 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +772 -0
- package/bin/argo.js +102 -39
- package/config.yaml +5139 -367
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +575 -0
- package/docs/MCP_SETUP.md +108 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +36 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +106 -0
- package/package.json +55 -12
- package/packages/dsh-plugin/README.md +152 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1533 -0
- package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
- package/packages/dsh-plugin/dsh/index.js +527 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +406 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +1382 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +294 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/article_meta.py +131 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1066 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +751 -0
- package/scripts/crawl.py +74 -35
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/dispatch_result.py +98 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +982 -0
- package/scripts/engine_env.py +539 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +738 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +799 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +310 -0
- package/scripts/engines_builders_batch10.py +204 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1359 -0
- package/scripts/evidence.py +620 -166
- package/scripts/evidence_loop.py +624 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1804 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +195 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +166 -0
- package/scripts/install.sh +221 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +402 -0
- package/scripts/macro_countries.py +97 -0
- package/scripts/matrix_search_eval.py +1187 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +492 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +124 -0
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +125 -419
- package/scripts/mcp_setup.py +431 -0
- package/scripts/mcp_tools.py +331 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/mcp_warm.py +127 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +633 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/recovery_runner.py +150 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +658 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +607 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +169 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/retry_policy.py +76 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +243 -0
- package/scripts/route_combo.py +917 -0
- package/scripts/route_domains.py +341 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +886 -406
- package/scripts/search_attribution.py +185 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +640 -0
- package/scripts/search_entry.py +395 -0
- package/scripts/search_output.py +690 -0
- package/scripts/search_pipeline.py +926 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +129 -16
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +391 -0
- package/scripts/serp_guard.py +336 -0
- package/scripts/serp_kill_log.py +74 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +107 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +447 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +186 -0
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +716 -242
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""recovery_l0 — 恢复链 L0:early-stop 误判时补跑被跳过的同域引擎。
|
|
3
|
+
|
|
4
|
+
early-stop 触发后 merged 归零(壳/噪声被下游过滤清零),combo 里
|
|
5
|
+
routed 但未调用的引擎是最小成本恢复对象——它们与已跑引擎同域,
|
|
6
|
+
比 L3 换跨域引擎更可能交付本域结果(2026-10-11 热榜实测:
|
|
7
|
+
wave-1 首引擎交壳骗过早停、同域真榜单引擎被 cancelled,merged
|
|
8
|
+
归零触发 2.3s 跨域恢复链)。
|
|
9
|
+
|
|
10
|
+
L0 是恢复决策树的第 0 级:补跑本 combo 的 skipped 引擎,非空
|
|
11
|
+
即采用;失败才落 L3 换引擎。执行器由调用方注入(search_pipeline
|
|
12
|
+
的 _run_one/_ingest 闭包),本模块不依赖编排状态,与 recovery.py
|
|
13
|
+
的 L1–L4 决策树职责分离。
|
|
14
|
+
|
|
15
|
+
两条硬约束(2026-10-11 甲轮收口):
|
|
16
|
+
|
|
17
|
+
- **壳不算「交付」**:补跑结果与充分性判定(query_signals)、
|
|
18
|
+
漏斗壳过滤共用 serp_guard.drop_shell_urls 同一口径——被跳过的
|
|
19
|
+
引擎若也交壳,不算恢复成功,落 L3。否则壳经 L0 绕过主链壳过滤
|
|
20
|
+
直送最终输出(主链过滤跑在恢复链之前)。
|
|
21
|
+
- **死线**:deadline(monotonic 秒)之后不再点火新引擎;剩余不足
|
|
22
|
+
_MIN_USEFUL_SLICE_S(口径:近 72h 真实 P50=1430ms,低于它新起
|
|
23
|
+
的引擎大概率白烧一次调用)也不点火。补跑是串行的,无此约束时
|
|
24
|
+
最坏 |skipped| × 单引擎超时,把已失败的查询再拖出几十秒。
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import time
|
|
30
|
+
from typing import Any, Callable
|
|
31
|
+
|
|
32
|
+
from search_rank import deduplicate_by_url, rrf_merge
|
|
33
|
+
|
|
34
|
+
# 点火下限:剩余窗口低于一个真实 P50 时,新引擎大概率来不及交付。
|
|
35
|
+
_MIN_USEFUL_SLICE_S = 1.5
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _deliverable(res: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
39
|
+
"""无 error 且非壳的结果才算「交付」(壳口径唯一出口:serp_guard)。"""
|
|
40
|
+
goods = [r for r in res if isinstance(r, dict) and "error" not in r]
|
|
41
|
+
try:
|
|
42
|
+
from serp_guard import drop_shell_urls
|
|
43
|
+
except ImportError:
|
|
44
|
+
return goods # 守卫不可用 → 壳不过滤(fail-open,与模块容错一致)
|
|
45
|
+
return drop_shell_urls(goods)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def finalize_l3_results(rec_results: list[dict[str, Any]],
|
|
49
|
+
max_results: int) -> list[dict[str, Any]]:
|
|
50
|
+
"""L3 恢复结果收口:去重 → 壳过滤 → 截断。
|
|
51
|
+
|
|
52
|
+
主链的壳过滤跑在恢复链之前,恢复产出(L0/L3)是唯一可能把壳带回
|
|
53
|
+
最终输出的入口——L0 在 _deliverable 判定处过滤,L3 在此统一收口。
|
|
54
|
+
"""
|
|
55
|
+
merged = deduplicate_by_url(rec_results)
|
|
56
|
+
try:
|
|
57
|
+
from serp_guard import drop_shell_urls
|
|
58
|
+
except ImportError:
|
|
59
|
+
pass
|
|
60
|
+
else:
|
|
61
|
+
merged = drop_shell_urls(merged)
|
|
62
|
+
return merged[:max_results]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def maybe_retry_skipped(
|
|
66
|
+
early_stopped: bool,
|
|
67
|
+
engines_combo: list[str],
|
|
68
|
+
raw_results: dict[str, list[dict[str, Any]]],
|
|
69
|
+
available_engines: Callable[[], list[str] | set[str]],
|
|
70
|
+
query: str,
|
|
71
|
+
run_one: Callable[..., tuple[str, list[dict[str, Any]], Any, float]],
|
|
72
|
+
ingest: Callable[..., None],
|
|
73
|
+
max_results: int,
|
|
74
|
+
emit_usage: Callable[[str, dict[str, Any]], bool] | None,
|
|
75
|
+
deadline: float | None = None,
|
|
76
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
77
|
+
"""early-stop 误判后的 L0 恢复:补跑被跳过的同域引擎。
|
|
78
|
+
|
|
79
|
+
返回 (结果, 恢复结论)。结果非空时调用方将其作为 merged、
|
|
80
|
+
结论随顶层 recovery 字段输出;为空返回 ([], None),调用方
|
|
81
|
+
落 L3 换引擎。补跑结果按引擎记回 raw_results(provenance
|
|
82
|
+
与自适应学习可见),并按恢复流发射使用日志。
|
|
83
|
+
|
|
84
|
+
deadline:monotonic 秒(与 time.monotonic() 同钟);None = 不限
|
|
85
|
+
(仅测试用,生产调用方必须给——见模块 docstring 死线约束)。
|
|
86
|
+
"""
|
|
87
|
+
if not early_stopped:
|
|
88
|
+
return [], None
|
|
89
|
+
skipped = [e for e in engines_combo if e not in raw_results]
|
|
90
|
+
if not skipped:
|
|
91
|
+
return [], None
|
|
92
|
+
try:
|
|
93
|
+
available = set(available_engines())
|
|
94
|
+
except Exception:
|
|
95
|
+
available = None
|
|
96
|
+
lists: list[list[dict[str, Any]]] = []
|
|
97
|
+
for eng in skipped:
|
|
98
|
+
if deadline is not None:
|
|
99
|
+
remain = deadline - time.monotonic()
|
|
100
|
+
if remain <= 0 or remain < _MIN_USEFUL_SLICE_S:
|
|
101
|
+
break
|
|
102
|
+
if available is not None and eng not in available:
|
|
103
|
+
continue
|
|
104
|
+
try:
|
|
105
|
+
_e, res, _out, _lat = run_one(eng)
|
|
106
|
+
ingest(_e, res, _out, _lat)
|
|
107
|
+
goods = _deliverable(res)
|
|
108
|
+
if goods:
|
|
109
|
+
# _engine 戳记与恢复执行器同约定:provenance 记回
|
|
110
|
+
# raw_results(by_eng 分组)依赖它,缺失即静默丢账。
|
|
111
|
+
for r in goods:
|
|
112
|
+
r.setdefault("_engine", eng)
|
|
113
|
+
lists.append(goods)
|
|
114
|
+
except Exception:
|
|
115
|
+
continue
|
|
116
|
+
if not lists:
|
|
117
|
+
return [], None
|
|
118
|
+
merged = (rrf_merge(lists) if len(lists) > 1
|
|
119
|
+
else deduplicate_by_url(lists[0]))
|
|
120
|
+
merged = merged[:max_results]
|
|
121
|
+
if not merged:
|
|
122
|
+
return [], None
|
|
123
|
+
by_eng: dict[str, list[dict[str, Any]]] = {}
|
|
124
|
+
for r in merged:
|
|
125
|
+
eng = r.get("_engine") or r.get("source") or ""
|
|
126
|
+
if eng:
|
|
127
|
+
r.setdefault("consensus_engines", [eng])
|
|
128
|
+
by_eng.setdefault(eng, []).append(r)
|
|
129
|
+
for eng, lst in by_eng.items():
|
|
130
|
+
raw_results.setdefault(eng, lst)
|
|
131
|
+
info = {
|
|
132
|
+
"triggered": True,
|
|
133
|
+
"recovered": True,
|
|
134
|
+
"level_used": "L0",
|
|
135
|
+
"strategy_used": "retry_skipped",
|
|
136
|
+
"steps_tried": [{
|
|
137
|
+
"level": "L0",
|
|
138
|
+
"strategy": "retry_skipped",
|
|
139
|
+
"query": query,
|
|
140
|
+
"engines": skipped,
|
|
141
|
+
"outcome": f"{len(merged)} results",
|
|
142
|
+
}],
|
|
143
|
+
"final_query": query,
|
|
144
|
+
"note": "early-stop 误判后补跑 skipped 引擎成功",
|
|
145
|
+
}
|
|
146
|
+
if emit_usage is not None:
|
|
147
|
+
try:
|
|
148
|
+
emit_usage("recovery", {
|
|
149
|
+
"query": (query[:60] if query else query),
|
|
150
|
+
"triggered": True,
|
|
151
|
+
"recovered": True,
|
|
152
|
+
"level_used": "L0",
|
|
153
|
+
"strategy_used": "retry_skipped",
|
|
154
|
+
"steps_tried": 1,
|
|
155
|
+
"final_query": (query[:60] if query else query),
|
|
156
|
+
"note": info["note"],
|
|
157
|
+
})
|
|
158
|
+
except Exception:
|
|
159
|
+
pass
|
|
160
|
+
return merged, info
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""recovery_runner.py — 搜索主管线错误恢复执行器与 L3 策略。
|
|
3
|
+
|
|
4
|
+
拆分自 search_pipeline.py:将零结果/异常退化恢复链与有界并发对冲执行器从主管线剥离。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from bounded_run import run_bounded
|
|
13
|
+
from except_sets import OPT_IMPORT, SHAPE_BENIGN
|
|
14
|
+
from search_rank import deduplicate_by_url
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run_pipeline_recovery(
|
|
18
|
+
*,
|
|
19
|
+
query: str,
|
|
20
|
+
merged: list[dict[str, Any]],
|
|
21
|
+
raw_results: dict[str, list],
|
|
22
|
+
decision: dict[str, Any],
|
|
23
|
+
engines: list[str],
|
|
24
|
+
available_engines_fn: Any,
|
|
25
|
+
max_results: int,
|
|
26
|
+
max_rec_level: str | None,
|
|
27
|
+
echo_triggered: bool,
|
|
28
|
+
primary_grace_s: float,
|
|
29
|
+
timeout: float,
|
|
30
|
+
depth: str,
|
|
31
|
+
mode: str,
|
|
32
|
+
since_iso: str | None,
|
|
33
|
+
until_iso: str | None,
|
|
34
|
+
skip_cache: bool,
|
|
35
|
+
quota_batch: Any,
|
|
36
|
+
engine_search_fn: Any,
|
|
37
|
+
emit_usage_fn: Any = None,
|
|
38
|
+
recovery_engines_set: set[str] | None = None,
|
|
39
|
+
engine_domain_extra: dict[str, Any] | None = None,
|
|
40
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
41
|
+
"""执行恢复链(L1 放宽 / L2 变体 / L3 换引擎):有界并发对冲跑候选引擎并复检壳。"""
|
|
42
|
+
recovery_info = None
|
|
43
|
+
try:
|
|
44
|
+
from recovery import run_recovery
|
|
45
|
+
tried = list(raw_results.keys()) or list(engines)
|
|
46
|
+
fallback_engines = decision.get("engines_fallback") or []
|
|
47
|
+
|
|
48
|
+
rec_level = max_rec_level
|
|
49
|
+
if echo_triggered:
|
|
50
|
+
rec_level = "L3"
|
|
51
|
+
elif fallback_engines and decision.get("domain") not in (
|
|
52
|
+
None, "", "general", "general_search") \
|
|
53
|
+
and (rec_level is None or rec_level < "L3"):
|
|
54
|
+
rec_level = "L3"
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
enabled_set = set(available_engines_fn())
|
|
58
|
+
except Exception:
|
|
59
|
+
enabled_set = None
|
|
60
|
+
|
|
61
|
+
dk = engine_domain_extra or {}
|
|
62
|
+
|
|
63
|
+
def _fire(eng: str) -> list[dict[str, Any]]:
|
|
64
|
+
if recovery_engines_set is not None:
|
|
65
|
+
recovery_engines_set.add(eng)
|
|
66
|
+
try:
|
|
67
|
+
res = engine_search_fn(
|
|
68
|
+
query, eng, n=max_results,
|
|
69
|
+
timeout=timeout, depth=depth, mode=mode,
|
|
70
|
+
since=since_iso, until=until_iso,
|
|
71
|
+
skip_cache=skip_cache, **dk,
|
|
72
|
+
)
|
|
73
|
+
except Exception as e:
|
|
74
|
+
sys.stderr.write(f"[recovery] 恢复链异常({type(e).__name__}): {e}\n")
|
|
75
|
+
res = []
|
|
76
|
+
goods = [r for r in (res or []) if isinstance(r, dict) and "error" not in r]
|
|
77
|
+
quota_batch.add(eng, bool(goods))
|
|
78
|
+
if goods:
|
|
79
|
+
for r in goods:
|
|
80
|
+
r.setdefault("_engine", eng)
|
|
81
|
+
r.setdefault("_recovered", True)
|
|
82
|
+
try:
|
|
83
|
+
from postprocess_signals import drop_echo_items as _drop_echo
|
|
84
|
+
goods = _drop_echo(goods, query)
|
|
85
|
+
except OPT_IMPORT + SHAPE_BENIGN:
|
|
86
|
+
pass
|
|
87
|
+
return goods
|
|
88
|
+
|
|
89
|
+
def _recovery_executor(rq: str, rengines: list[str]) -> list[dict[str, Any]]:
|
|
90
|
+
if len(rengines) <= 1:
|
|
91
|
+
return _fire(rengines[0]) if rengines else []
|
|
92
|
+
grace = max(0.0, float(primary_grace_s or 0.0))
|
|
93
|
+
wait_s = grace * (len(rengines) - 1) + max(1.0, float(timeout)) + 1.0
|
|
94
|
+
|
|
95
|
+
def _enough(finished: list[tuple[str, Any]]) -> bool:
|
|
96
|
+
return any(isinstance(v, list) and v for _i, v in finished)
|
|
97
|
+
|
|
98
|
+
finished, _unfinished = run_bounded(
|
|
99
|
+
rengines, _fire, wait_s, max_workers=3,
|
|
100
|
+
enough=_enough, stagger_s=grace,
|
|
101
|
+
)
|
|
102
|
+
for _item, value in finished:
|
|
103
|
+
if isinstance(value, list) and value:
|
|
104
|
+
return value
|
|
105
|
+
return []
|
|
106
|
+
|
|
107
|
+
rec_results, rec_result = run_recovery(
|
|
108
|
+
query, tried, _recovery_executor,
|
|
109
|
+
engines_fallback=fallback_engines, enabled=enabled_set, mode=mode,
|
|
110
|
+
max_level=rec_level,
|
|
111
|
+
)
|
|
112
|
+
recovery_info = rec_result.to_dict()
|
|
113
|
+
|
|
114
|
+
if emit_usage_fn is not None:
|
|
115
|
+
try:
|
|
116
|
+
emit_usage_fn("recovery", {
|
|
117
|
+
"query": (query[:60] if query else query),
|
|
118
|
+
"triggered": recovery_info.get("triggered"),
|
|
119
|
+
"recovered": recovery_info.get("recovered"),
|
|
120
|
+
"level_used": recovery_info.get("level_used"),
|
|
121
|
+
"strategy_used": recovery_info.get("strategy_used"),
|
|
122
|
+
"steps_tried": len(recovery_info.get("steps_tried") or []),
|
|
123
|
+
"final_query": (recovery_info.get("final_query") or "")[:60],
|
|
124
|
+
"note": recovery_info.get("note", ""),
|
|
125
|
+
})
|
|
126
|
+
except SHAPE_BENIGN:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
if rec_results:
|
|
130
|
+
try:
|
|
131
|
+
from recovery_l0 import finalize_l3_results
|
|
132
|
+
merged = finalize_l3_results(rec_results, max_results)
|
|
133
|
+
except ImportError:
|
|
134
|
+
merged = deduplicate_by_url(rec_results)[:max_results]
|
|
135
|
+
|
|
136
|
+
rec_by_eng: dict[str, list] = {}
|
|
137
|
+
for r in merged:
|
|
138
|
+
eng = r.get("_engine") or r.get("source") or ""
|
|
139
|
+
if eng:
|
|
140
|
+
r.setdefault("consensus_engines", [eng])
|
|
141
|
+
rec_by_eng.setdefault(eng, []).append(r)
|
|
142
|
+
for eng_name, lst in rec_by_eng.items():
|
|
143
|
+
raw_results.setdefault(eng_name, lst)
|
|
144
|
+
|
|
145
|
+
except ImportError:
|
|
146
|
+
pass
|
|
147
|
+
except Exception as e:
|
|
148
|
+
sys.stderr.write(f"[recovery] 错误恢复跳过: {type(e).__name__}\n")
|
|
149
|
+
|
|
150
|
+
return merged, recovery_info
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""redskill_engine.py — 小红书 REDSkill 排行榜与全量技能检索(argo 引擎,type=cli)
|
|
3
|
+
|
|
4
|
+
数据源: https://cowork.xiaohongshu.com/s/redskill-rank/data.json
|
|
5
|
+
- 公开静态 JSON(14.6MB),每日 08:05 更新,无鉴权
|
|
6
|
+
- 结构: dataDate / genTime / useList(使用榜100) / newList(7天新增100)
|
|
7
|
+
/ todayList(当日新增100) / authorList(作者发布榜100) / allSkills(全量47650)
|
|
8
|
+
- 注意: 页面搜索框为纯前端过滤(无 ?q= 深链),榜单技能才带 note_id
|
|
9
|
+
|
|
10
|
+
缓存: ~/.cache/argo-redskill/data.json(按 dataDate 判断过期,过期自动刷新)
|
|
11
|
+
|
|
12
|
+
用法:
|
|
13
|
+
python3 redskill_engine.py search "小红书封面" -n 5
|
|
14
|
+
python3 redskill_engine.py rank [use|new|today|author] -n 10
|
|
15
|
+
python3 redskill_engine.py update
|
|
16
|
+
python3 redskill_engine.py stats
|
|
17
|
+
|
|
18
|
+
输出: JSON {"results": [{title,url,snippet,score,source,...}]}(argo cli builder 兼容)
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
import time
|
|
29
|
+
import urllib.error
|
|
30
|
+
import urllib.parse
|
|
31
|
+
import urllib.request
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
# 复用 scripts/ 的原子写唯一来源(本文件在 scripts/redskill/ 子目录,
|
|
36
|
+
# 父目录不在 sys.path 上,需显式加)。
|
|
37
|
+
_SCRIPTS_DIR = str(Path(__file__).resolve().parent.parent)
|
|
38
|
+
if _SCRIPTS_DIR not in sys.path:
|
|
39
|
+
sys.path.insert(0, _SCRIPTS_DIR)
|
|
40
|
+
import argo_paths # noqa: E402
|
|
41
|
+
from net_proxy import open_url # noqa: E402 出口调度唯一入口(issue #13 同类修复)
|
|
42
|
+
|
|
43
|
+
DATA_URL = "https://cowork.xiaohongshu.com/s/redskill-rank/data.json"
|
|
44
|
+
PAGE_URL = "https://cowork.xiaohongshu.com/s/redskill-rank/"
|
|
45
|
+
EXPLORE_URL = "https://www.xiaohongshu.com/explore/{}"
|
|
46
|
+
SEARCH_URL = "https://www.xiaohongshu.com/search_result?keyword={}"
|
|
47
|
+
PROFILE_URL = "https://www.xiaohongshu.com/user/profile/{}"
|
|
48
|
+
CACHE_DIR = Path(os.environ.get("ARGO_REDSKILL_CACHE")
|
|
49
|
+
or (argo_paths.platform_cache_default() / "argo-redskill"))
|
|
50
|
+
CACHE_FILE = CACHE_DIR / "data.json"
|
|
51
|
+
USER_AGENT = "argo-search/1.0 (+https://github.com/taxueseek/argo; redskill)"
|
|
52
|
+
DOWNLOAD_TIMEOUT = 120 # 14.6MB 全量,首次下载需要更长时间
|
|
53
|
+
|
|
54
|
+
RANK_LISTS = {
|
|
55
|
+
"use": "useList",
|
|
56
|
+
"new": "newList",
|
|
57
|
+
"today": "todayList",
|
|
58
|
+
"author": "authorList",
|
|
59
|
+
}
|
|
60
|
+
RANK_LABELS = {
|
|
61
|
+
"use": "使用人数榜",
|
|
62
|
+
"new": "7天新增榜",
|
|
63
|
+
"today": "当日新增榜",
|
|
64
|
+
"author": "作者发布榜",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
# 中文技能名可能带 .skill 后缀等,检索时归一化
|
|
68
|
+
_NAME_NORM_RE = re.compile(r"[\s._\-]+")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _log(msg: str) -> None:
|
|
72
|
+
print(f"[redskill] {msg}", file=sys.stderr)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _http_get(url: str, timeout: int = DOWNLOAD_TIMEOUT) -> bytes:
|
|
76
|
+
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"})
|
|
77
|
+
with open_url(req, timeout=timeout) as resp:
|
|
78
|
+
return resp.read()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _load_data(force: bool = False) -> dict[str, Any]:
|
|
82
|
+
"""加载数据:缓存新鲜则直接用,过期或缺失则下载。
|
|
83
|
+
|
|
84
|
+
新鲜判定:dataDate 距今 <= 2 天(genTime 每日 08:05,允许跨一天缓冲)。
|
|
85
|
+
"""
|
|
86
|
+
cached = _read_cache()
|
|
87
|
+
if not force and cached:
|
|
88
|
+
data_date = cached.get("dataDate", "")
|
|
89
|
+
if _is_fresh(data_date):
|
|
90
|
+
return cached
|
|
91
|
+
_log(f"缓存过期(dataDate={data_date}),刷新中…")
|
|
92
|
+
return _download(force=force)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _is_fresh(data_date: str) -> bool:
|
|
96
|
+
if not data_date:
|
|
97
|
+
return False
|
|
98
|
+
try:
|
|
99
|
+
y, m, d = (int(x) for x in data_date.split("-"))
|
|
100
|
+
from datetime import date
|
|
101
|
+
|
|
102
|
+
gap = (date.today() - date(y, m, d)).days
|
|
103
|
+
return gap <= 2
|
|
104
|
+
except Exception:
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _read_cache() -> dict[str, Any] | None:
|
|
109
|
+
try:
|
|
110
|
+
if CACHE_FILE.exists() and CACHE_FILE.stat().st_size > 0:
|
|
111
|
+
with open(CACHE_FILE, encoding="utf-8") as f:
|
|
112
|
+
return json.load(f)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
_log(f"缓存读取失败: {e}")
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _download(force: bool = False) -> dict[str, Any]:
|
|
119
|
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
120
|
+
raw = _http_get(DATA_URL)
|
|
121
|
+
data = json.loads(raw.decode("utf-8"))
|
|
122
|
+
if "allSkills" not in data:
|
|
123
|
+
raise ValueError("data.json 结构异常:缺少 allSkills")
|
|
124
|
+
# 原子写走 argo_paths 唯一来源(唯一 tmp 名);旧实现固定 `.tmp` 名,
|
|
125
|
+
# 并发刷新(手动 update 撞上搜索自动刷新)会互相搬走临时文件而写失败。
|
|
126
|
+
# 14.6MB 缓存用 indent=None 省掉格式化的时间与体积。
|
|
127
|
+
argo_paths.atomic_write_json(CACHE_FILE, data, indent=None)
|
|
128
|
+
if force:
|
|
129
|
+
_log(f"已强制刷新缓存: {CACHE_FILE} ({CACHE_FILE.stat().st_size / 1e6:.1f}MB, dataDate={data.get('dataDate')})")
|
|
130
|
+
return data
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _note_map(data: dict[str, Any]) -> dict[int, str]:
|
|
134
|
+
"""从三个榜单合并 skill_id -> note_id 映射(allSkills 无 note_id)。"""
|
|
135
|
+
m: dict[int, str] = {}
|
|
136
|
+
for key in ("useList", "newList", "todayList"):
|
|
137
|
+
for item in data.get(key, []):
|
|
138
|
+
sid = item.get("skill_id")
|
|
139
|
+
note = item.get("note_id")
|
|
140
|
+
if sid and note:
|
|
141
|
+
m[int(sid)] = note
|
|
142
|
+
return m
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _tokenize(query: str) -> list[str]:
|
|
146
|
+
"""切词:英文/数字按空白与连字符,中文保留整串 + 逐字(用于子串覆盖)。"""
|
|
147
|
+
q = query.strip().lower()
|
|
148
|
+
tokens = re.split(r"[\s,,。;;、/\\|::().()]+", q)
|
|
149
|
+
tokens = [t for t in tokens if t]
|
|
150
|
+
return tokens
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _search_all(data: dict[str, Any], query: str, n: int) -> list[dict[str, Any]]:
|
|
154
|
+
"""全量技能关键词检索(对比 allSkills 的 skill_name + skill_description)。"""
|
|
155
|
+
all_skills = data.get("allSkills", [])
|
|
156
|
+
note_map = _note_map(data)
|
|
157
|
+
# 榜单热力榜:useList skill_id -> 排名(用于热门加成)。
|
|
158
|
+
# 单条记录缺 skill_id 不致命,跳过而非让整个搜索抛错
|
|
159
|
+
hot_rank: dict[int, int] = {}
|
|
160
|
+
for i, item in enumerate(data.get("useList", [])):
|
|
161
|
+
sid = item.get("skill_id")
|
|
162
|
+
if sid:
|
|
163
|
+
hot_rank[int(sid)] = i
|
|
164
|
+
|
|
165
|
+
q = query.strip().lower()
|
|
166
|
+
tokens = _tokenize(q) or [q]
|
|
167
|
+
no_space_q = re.sub(r"[\s]+", "", q)
|
|
168
|
+
|
|
169
|
+
scored: list[tuple[float, dict[str, Any]]] = []
|
|
170
|
+
for s in all_skills:
|
|
171
|
+
name = str(s.get("skill_name", ""))
|
|
172
|
+
desc = str(s.get("skill_description", ""))
|
|
173
|
+
name_l = name.lower()
|
|
174
|
+
desc_l = desc.lower()
|
|
175
|
+
score = 0.0
|
|
176
|
+
# 整串命中
|
|
177
|
+
if q in name_l:
|
|
178
|
+
score += 10.0
|
|
179
|
+
elif q in desc_l:
|
|
180
|
+
score += 6.0
|
|
181
|
+
# 去空白串命中(中文查询常见「小红书封面」)
|
|
182
|
+
if no_space_q and no_space_q != q and (no_space_q in name_l or no_space_q in desc_l):
|
|
183
|
+
score += 6.0
|
|
184
|
+
# 子词命中
|
|
185
|
+
hit_terms = 0
|
|
186
|
+
for t in tokens:
|
|
187
|
+
if t in name_l:
|
|
188
|
+
score += 3.0
|
|
189
|
+
hit_terms += 1
|
|
190
|
+
elif t in desc_l:
|
|
191
|
+
score += 1.5
|
|
192
|
+
hit_terms += 1
|
|
193
|
+
if hit_terms == 0:
|
|
194
|
+
continue
|
|
195
|
+
# 词覆盖率加成
|
|
196
|
+
score += 2.0 * hit_terms / max(len(tokens), 1)
|
|
197
|
+
# 热门加成(榜单技能优先)
|
|
198
|
+
sid = s.get("skill_id")
|
|
199
|
+
if sid and int(sid) in hot_rank:
|
|
200
|
+
score += 1.0
|
|
201
|
+
# 描述越短、名称越匹配,越像「直接答案」
|
|
202
|
+
if score > 0:
|
|
203
|
+
scored.append((score, s))
|
|
204
|
+
|
|
205
|
+
scored.sort(key=lambda x: -x[0])
|
|
206
|
+
results: list[dict[str, Any]] = []
|
|
207
|
+
for score, s in scored[:n]:
|
|
208
|
+
sid = s.get("skill_id")
|
|
209
|
+
note = note_map.get(int(sid)) if sid else None
|
|
210
|
+
name = s.get("skill_name", "")
|
|
211
|
+
if note:
|
|
212
|
+
url = EXPLORE_URL.format(note)
|
|
213
|
+
else:
|
|
214
|
+
# 无笔记链接时给站内搜索(URL 随技能名唯一,避免同 URL 去重误伤)
|
|
215
|
+
url = SEARCH_URL.format(urllib.parse.quote(name))
|
|
216
|
+
results.append({
|
|
217
|
+
"title": name,
|
|
218
|
+
"url": url,
|
|
219
|
+
"snippet": str(s.get("skill_description", ""))[:500],
|
|
220
|
+
"score": round(score, 3),
|
|
221
|
+
"source": "redskill",
|
|
222
|
+
"skill_id": sid,
|
|
223
|
+
})
|
|
224
|
+
return results
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _rank_list(data: dict[str, Any], rank_type: str, n: int) -> list[dict[str, Any]]:
|
|
228
|
+
items = data.get(RANK_LISTS[rank_type], [])
|
|
229
|
+
results = []
|
|
230
|
+
for i, item in enumerate(items[:n]):
|
|
231
|
+
if rank_type == "author":
|
|
232
|
+
results.append({
|
|
233
|
+
"title": item.get("nickname", ""),
|
|
234
|
+
"url": PROFILE_URL.format(item.get("author_id", "")),
|
|
235
|
+
"snippet": f"发布技能 {item.get('skill_cnt', 0)} 个",
|
|
236
|
+
"score": round(1.0 - i * 0.01, 3),
|
|
237
|
+
"source": "redskill",
|
|
238
|
+
"rank": i + 1,
|
|
239
|
+
"author_id": item.get("author_id"),
|
|
240
|
+
})
|
|
241
|
+
else:
|
|
242
|
+
note = item.get("note_id")
|
|
243
|
+
url = EXPLORE_URL.format(note) if note else PAGE_URL
|
|
244
|
+
|
|
245
|
+
snippet = str(item.get("skill_description", ""))[:500]
|
|
246
|
+
results.append({
|
|
247
|
+
"title": item.get("skill_name", ""),
|
|
248
|
+
"url": url,
|
|
249
|
+
"snippet": snippet,
|
|
250
|
+
"score": round(1.0 - i * 0.01, 3),
|
|
251
|
+
"source": "redskill",
|
|
252
|
+
"rank": i + 1,
|
|
253
|
+
"skill_id": item.get("skill_id"),
|
|
254
|
+
"use_cnt": item.get("use_cnt") if rank_type in ("use", "today") else item.get("new7_cnt"),
|
|
255
|
+
"use_users": item.get("use_users") if rank_type in ("use", "today") else None,
|
|
256
|
+
})
|
|
257
|
+
return results
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _stats(data: dict[str, Any]) -> dict[str, Any]:
|
|
261
|
+
return {
|
|
262
|
+
"dataDate": data.get("dataDate"),
|
|
263
|
+
"genTime": data.get("genTime"),
|
|
264
|
+
"counts": {
|
|
265
|
+
"useList": len(data.get("useList", [])),
|
|
266
|
+
"newList": len(data.get("newList", [])),
|
|
267
|
+
"todayList": len(data.get("todayList", [])),
|
|
268
|
+
"authorList": len(data.get("authorList", [])),
|
|
269
|
+
"allSkills": len(data.get("allSkills", [])),
|
|
270
|
+
},
|
|
271
|
+
"cache_file": str(CACHE_FILE),
|
|
272
|
+
"cache_mb": round(CACHE_FILE.stat().st_size / 1e6, 1) if CACHE_FILE.exists() else 0,
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def main() -> int:
|
|
277
|
+
parser = argparse.ArgumentParser(description="小红书 REDSkill 引擎(排行榜 + 全量技能检索)")
|
|
278
|
+
parser.add_argument("action", nargs="?", default="search", help="search | rank | update | stats")
|
|
279
|
+
parser.add_argument("query", nargs="?")
|
|
280
|
+
parser.add_argument("-n", type=int, default=5)
|
|
281
|
+
parser.add_argument("--force-update", action="store_true")
|
|
282
|
+
args = parser.parse_args()
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
if args.action == "update":
|
|
286
|
+
data = _download(force=True)
|
|
287
|
+
print(json.dumps({"results": [{"title": f"{RANK_LABELS[k]} {len(data.get(v, []))} 条",
|
|
288
|
+
"url": PAGE_URL,
|
|
289
|
+
"snippet": f"dataDate={data.get('dataDate')} allSkills={len(data.get('allSkills', []))}",
|
|
290
|
+
"score": 1.0, "source": "redskill"} for k, v in RANK_LISTS.items()]},
|
|
291
|
+
ensure_ascii=False))
|
|
292
|
+
return 0
|
|
293
|
+
|
|
294
|
+
if args.action == "stats":
|
|
295
|
+
data = _read_cache()
|
|
296
|
+
if not data:
|
|
297
|
+
print(json.dumps({"results": [], "error": "无缓存,先运行 update"}), ensure_ascii=False)
|
|
298
|
+
return 0
|
|
299
|
+
s = _stats(data)
|
|
300
|
+
print(json.dumps({"results": [{"title": f"{s['dataDate']} 全量 {s['counts']['allSkills']}",
|
|
301
|
+
"url": PAGE_URL,
|
|
302
|
+
"snippet": json.dumps(s, ensure_ascii=False)[:300],
|
|
303
|
+
"score": 1.0, "source": "redskill"}]}, ensure_ascii=False))
|
|
304
|
+
return 0
|
|
305
|
+
|
|
306
|
+
data = _load_data(force=args.force_update)
|
|
307
|
+
|
|
308
|
+
if args.action == "rank" or (args.action in RANK_LISTS and not args.query):
|
|
309
|
+
rank_type = args.action if args.action in RANK_LISTS else (args.query or "use")
|
|
310
|
+
if rank_type not in RANK_LISTS:
|
|
311
|
+
rank_type = "use"
|
|
312
|
+
results = _rank_list(data, rank_type, args.n)
|
|
313
|
+
else:
|
|
314
|
+
q = args.query or args.action
|
|
315
|
+
if not q or q in ("search", "s"):
|
|
316
|
+
print(json.dumps({"results": [], "error": "缺少查询词(用法: search <query> -n N)"}), ensure_ascii=False)
|
|
317
|
+
return 0
|
|
318
|
+
results = _search_all(data, q, args.n)
|
|
319
|
+
|
|
320
|
+
print(json.dumps({"results": results}, ensure_ascii=False))
|
|
321
|
+
return 0
|
|
322
|
+
except urllib.error.URLError as e:
|
|
323
|
+
print(json.dumps({"results": [], "error": f"下载失败: {e}(可稍后重试或先运行 update)"}),
|
|
324
|
+
ensure_ascii=False)
|
|
325
|
+
return 1
|
|
326
|
+
except Exception as e: # noqa: BLE001 - 引擎出口统一 JSON 化
|
|
327
|
+
print(json.dumps({"results": [], "error": str(e)}), ensure_ascii=False)
|
|
328
|
+
return 1
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
if __name__ == "__main__":
|
|
332
|
+
sys.exit(main())
|