argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
package/scripts/tweet.py
ADDED
|
@@ -0,0 +1,596 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""tweet.py — X 帖子完整打包(`argo tweet`)。
|
|
3
|
+
|
|
4
|
+
与 `argo search` 的分工:搜索**发现**推文(把推文压成一条结果参与 RRF 融合),
|
|
5
|
+
本命令**吸收**推文(给定链接或 ID,把整份内容原样落地)。同一条推文,搜索要
|
|
6
|
+
的是「一行」,打包要的是「整份」——所以它是独立命令,不是搜索结果的字段。
|
|
7
|
+
|
|
8
|
+
按需采用(默认轻、重活要显式要):
|
|
9
|
+
argo tweet "<url|id>" 正文打到 stdout
|
|
10
|
+
argo tweet "<url|id>" --json 结构化 JSON(全文+媒体 URL+互动+串/引用)
|
|
11
|
+
argo tweet "<url|id>" --out DIR 落盘四件套(post.md/raw.json/manifest/CHECKLIST)
|
|
12
|
+
argo tweet "<url|id>" --out DIR --media 再下载图片/视频(SSRF 守卫 + 代理感知)
|
|
13
|
+
argo tweet "<url|id>" --mode thread|conversation|quotes|reposts
|
|
14
|
+
argo tweet "<url|id>" --with-quotes --with-reposts --limit 50
|
|
15
|
+
|
|
16
|
+
零 Key、不碰 Cookie/登录态;媒体下载走 net_proxy.open_url(认 config.yaml 代理)
|
|
17
|
+
且每跳过 url_safety.check_url(拒内网/保留地址)。
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sys
|
|
26
|
+
import time
|
|
27
|
+
import urllib.request
|
|
28
|
+
from datetime import datetime, timezone
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
33
|
+
import fxtwitter_api as fx # noqa: E402
|
|
34
|
+
from cli_io import dumps, dumps_pretty # noqa: E402
|
|
35
|
+
from net_proxy import open_url # noqa: E402
|
|
36
|
+
from url_safety import check_url # noqa: E402
|
|
37
|
+
|
|
38
|
+
MODES = ("status", "thread", "conversation", "quotes", "reposts")
|
|
39
|
+
DEFAULT_OUT_PARENT = "argo-tweet"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _now_iso() -> str:
|
|
43
|
+
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _safe_name(s: str, max_len: int = 80) -> str:
|
|
47
|
+
s = re.sub(r"[^\w.\-]+", "_", s or "", flags=re.U)
|
|
48
|
+
return (s or "item")[:max_len]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def download_file(url: str, dest: Path, *, timeout: float = 60) -> dict[str, Any]:
|
|
52
|
+
"""下载一个媒体文件到 dest。返回 {ok, path, bytes, url, error}。
|
|
53
|
+
|
|
54
|
+
两道关与全仓出口纪律一致:先 check_url 拒掉非 http(s)/内网目标,再经
|
|
55
|
+
net_proxy.open_url 出网(否则「必须走代理」的环境里会静默连不上)。
|
|
56
|
+
"""
|
|
57
|
+
ok, reason = check_url(url)
|
|
58
|
+
if not ok:
|
|
59
|
+
return {"ok": False, "path": str(dest), "bytes": 0, "url": url,
|
|
60
|
+
"error": f"URL 被安全策略拒绝: {reason}"}
|
|
61
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
try:
|
|
63
|
+
req = urllib.request.Request(
|
|
64
|
+
url, headers={"User-Agent": fx.USER_AGENT, "Accept": "*/*"})
|
|
65
|
+
with open_url(req, timeout=timeout) as resp:
|
|
66
|
+
data = resp.read()
|
|
67
|
+
dest.write_bytes(data)
|
|
68
|
+
return {"ok": True, "path": str(dest), "bytes": len(data),
|
|
69
|
+
"url": url, "error": None}
|
|
70
|
+
except Exception as e: # 单张图失败不该拖垮整次打包
|
|
71
|
+
return {"ok": False, "path": str(dest), "bytes": 0, "url": url,
|
|
72
|
+
"error": f"{type(e).__name__}: {e}"}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _author_handle(status: dict[str, Any], author: Any = None) -> str:
|
|
76
|
+
for src in (author, status.get("author")):
|
|
77
|
+
if isinstance(src, dict) and src.get("screen_name"):
|
|
78
|
+
return str(src["screen_name"])
|
|
79
|
+
return str(status.get("screen_name") or "unknown")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def status_to_md_block(status: dict[str, Any], *, image_rel_paths=None,
|
|
83
|
+
video_rel_paths=None, heading_level: int = 3) -> str:
|
|
84
|
+
"""一条推文 → markdown 区块(含本地媒体相对路径)。"""
|
|
85
|
+
handle = _author_handle(status, status.get("author"))
|
|
86
|
+
author = status.get("author")
|
|
87
|
+
name = author.get("name") if isinstance(author, dict) else ""
|
|
88
|
+
h = "#" * max(2, min(heading_level, 5))
|
|
89
|
+
lines = [
|
|
90
|
+
f"{h} @{handle}" + (f" ({name})" if name else ""),
|
|
91
|
+
"",
|
|
92
|
+
f"- **URL**: {status.get('url') or ''}",
|
|
93
|
+
f"- **ID**: {status.get('id') or ''}",
|
|
94
|
+
f"- **时间**: {status.get('created_at') or ''}",
|
|
95
|
+
f"- **互动**: likes={status.get('likes')} "
|
|
96
|
+
f"reposts={status.get('reposts') or status.get('retweets')} "
|
|
97
|
+
f"replies={status.get('replies')} views={status.get('views')}",
|
|
98
|
+
"",
|
|
99
|
+
fx.status_text(status) or "_(无文本)_",
|
|
100
|
+
"",
|
|
101
|
+
]
|
|
102
|
+
if image_rel_paths:
|
|
103
|
+
lines += [f"{h}# 图片", ""] + [f"" for p in image_rel_paths] + [""]
|
|
104
|
+
if video_rel_paths:
|
|
105
|
+
lines += [f"{h}# 视频", ""] + [f"- [{Path(p).name}]({p})"
|
|
106
|
+
for p in video_rel_paths] + [""]
|
|
107
|
+
return "\n".join(lines)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def user_to_md_block(user: dict[str, Any], *, heading_level: int = 3) -> str:
|
|
111
|
+
"""转发列表里是 profile 对象,不是推文。"""
|
|
112
|
+
handle = user.get("screen_name") or user.get("username") or "unknown"
|
|
113
|
+
name = user.get("name") or ""
|
|
114
|
+
h = "#" * max(2, min(heading_level, 5))
|
|
115
|
+
lines = [
|
|
116
|
+
f"{h} @{handle}" + (f" ({name})" if name else ""),
|
|
117
|
+
"",
|
|
118
|
+
f"- **ID**: {user.get('id') or ''}",
|
|
119
|
+
f"- **粉丝**: {user.get('followers') or user.get('followers_count')}",
|
|
120
|
+
f"- **关注**: {user.get('following') or user.get('friends_count')}",
|
|
121
|
+
f"- **认证**: {user.get('verified') or user.get('is_blue_verified')}",
|
|
122
|
+
f"- **主页**: https://x.com/{handle}",
|
|
123
|
+
"",
|
|
124
|
+
]
|
|
125
|
+
bio = (user.get("description") or user.get("bio") or "").strip()
|
|
126
|
+
return "\n".join(lines + ([bio, ""] if bio else []))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def collect_media_jobs(status: dict[str, Any], *, video_quality: str,
|
|
130
|
+
prefix: str) -> tuple[list[dict], list[dict]]:
|
|
131
|
+
"""返回 (photo_jobs, video_jobs),每项含 url / dest_rel / meta。"""
|
|
132
|
+
photos = []
|
|
133
|
+
for i, p in enumerate(fx.iter_photos(status), 1):
|
|
134
|
+
url = p.get("url") or ""
|
|
135
|
+
if not url:
|
|
136
|
+
continue
|
|
137
|
+
ext = ".png" if ".png" in url else ".webp" if ".webp" in url else ".jpg"
|
|
138
|
+
photos.append({"url": url, "meta": p, "kind": "image",
|
|
139
|
+
"dest_rel": f"media/images/{prefix}_{i:02d}{ext}"})
|
|
140
|
+
videos = []
|
|
141
|
+
for i, v in enumerate(fx.iter_videos(status), 1):
|
|
142
|
+
url, chosen = fx.pick_video_url(v, quality=video_quality)
|
|
143
|
+
videos.append({"url": url, "meta": v, "chosen": chosen, "kind": "video",
|
|
144
|
+
"dest_rel": f"media/videos/{prefix}_{i:02d}.mp4",
|
|
145
|
+
"skip_reason": None if url else "no_downloadable_mp4"})
|
|
146
|
+
return photos, videos
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _dedupe_posts(posts: list[Any]) -> list[dict]:
|
|
150
|
+
"""按 id 去重(thread/replies/quotes 交叉重叠是常态)。"""
|
|
151
|
+
seen: set[str] = set()
|
|
152
|
+
out: list[dict] = []
|
|
153
|
+
for p in posts:
|
|
154
|
+
if not isinstance(p, dict):
|
|
155
|
+
continue
|
|
156
|
+
pid = str(p.get("id") or "")
|
|
157
|
+
if pid and pid in seen:
|
|
158
|
+
continue
|
|
159
|
+
if pid:
|
|
160
|
+
seen.add(pid)
|
|
161
|
+
out.append(p)
|
|
162
|
+
return out
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _fetch_bundle(status_id: str, mode: str, *, timeout: float,
|
|
166
|
+
limit: int) -> dict[str, Any]:
|
|
167
|
+
"""按 mode 拉取,返回统一 bundle(raw/author/root_status/thread/replies/quotes/repost_users/posts)。"""
|
|
168
|
+
bundle: dict[str, Any] = {
|
|
169
|
+
"raw": {}, "author": None, "root_status": None, "thread": [],
|
|
170
|
+
"replies": [], "quotes": [], "repost_users": [], "posts": [],
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
root_error: list[str] = []
|
|
174
|
+
|
|
175
|
+
def _root() -> tuple[dict | None, Any]:
|
|
176
|
+
"""主帖拉不到不致命(quotes/reposts 仍可交付),但**不静默**:原因记进
|
|
177
|
+
root_error,随 raw 落进产物,否则「为什么没有主帖」只能靠猜。"""
|
|
178
|
+
try:
|
|
179
|
+
b = fx.get_status(status_id, timeout=timeout)
|
|
180
|
+
return (b.get("status") if isinstance(b.get("status"), dict) else None,
|
|
181
|
+
b.get("raw"))
|
|
182
|
+
except fx.FxTwitterError as e:
|
|
183
|
+
root_error.append(str(e))
|
|
184
|
+
return None, None
|
|
185
|
+
|
|
186
|
+
def _raw(raw: dict) -> dict:
|
|
187
|
+
return {**raw, "status_error": root_error[0]} if root_error else raw
|
|
188
|
+
|
|
189
|
+
if mode == "quotes":
|
|
190
|
+
q = fx.get_quotes(status_id, limit=limit, timeout=timeout)
|
|
191
|
+
root, root_raw = _root()
|
|
192
|
+
quotes = [r for r in (q.get("results") or []) if isinstance(r, dict)]
|
|
193
|
+
return {**bundle, "raw": _raw({"quotes": q.get("raw"), "status": root_raw}),
|
|
194
|
+
"author": (root or {}).get("author"), "root_status": root,
|
|
195
|
+
"quotes": quotes,
|
|
196
|
+
"posts": _dedupe_posts(([root] if root else []) + quotes)}
|
|
197
|
+
|
|
198
|
+
if mode == "reposts":
|
|
199
|
+
r = fx.get_reposts(status_id, limit=limit, timeout=timeout)
|
|
200
|
+
root, root_raw = _root()
|
|
201
|
+
users = [u for u in (r.get("results") or []) if isinstance(u, dict)]
|
|
202
|
+
return {**bundle, "raw": _raw({"reposts": r.get("raw"), "status": root_raw}),
|
|
203
|
+
"author": (root or {}).get("author"), "root_status": root,
|
|
204
|
+
"repost_users": users, "posts": [root] if root else []}
|
|
205
|
+
|
|
206
|
+
if mode == "thread":
|
|
207
|
+
b = fx.get_thread(status_id, timeout=timeout)
|
|
208
|
+
root = b.get("status") if isinstance(b.get("status"), dict) else None
|
|
209
|
+
thread = [i for i in (b.get("thread") or []) if isinstance(i, dict)]
|
|
210
|
+
root_id = str((root or {}).get("id") or "")
|
|
211
|
+
posts = ([root] if root else []) + [
|
|
212
|
+
t for t in thread if str(t.get("id") or "") != root_id]
|
|
213
|
+
return {**bundle, "raw": b["raw"], "author": b.get("author"),
|
|
214
|
+
"root_status": root or (posts[0] if posts else None),
|
|
215
|
+
"thread": thread, "posts": _dedupe_posts(posts)}
|
|
216
|
+
|
|
217
|
+
if mode == "conversation":
|
|
218
|
+
b = fx.get_conversation(status_id, timeout=timeout)
|
|
219
|
+
root = b.get("status") if isinstance(b.get("status"), dict) else None
|
|
220
|
+
thread = [i for i in (b.get("thread") or []) if isinstance(i, dict)]
|
|
221
|
+
replies = [i for i in (b.get("replies") or []) if isinstance(i, dict)]
|
|
222
|
+
return {**bundle, "raw": b["raw"], "author": b.get("author"),
|
|
223
|
+
"root_status": root, "thread": thread, "replies": replies,
|
|
224
|
+
"posts": _dedupe_posts(([root] if root else []) + thread + replies)}
|
|
225
|
+
|
|
226
|
+
# status(默认):单条;响应里带串时尽量并入
|
|
227
|
+
b = fx.get_status(status_id, timeout=timeout)
|
|
228
|
+
root = b.get("status") if isinstance(b.get("status"), dict) else None
|
|
229
|
+
thr = b.get("thread")
|
|
230
|
+
thread = [i for i in thr if isinstance(i, dict)] if isinstance(thr, list) and len(thr) > 1 else []
|
|
231
|
+
root_id = str((root or {}).get("id") or "")
|
|
232
|
+
posts = ([root] if root else []) + [
|
|
233
|
+
t for t in thread if str(t.get("id") or "") != root_id]
|
|
234
|
+
return {**bundle, "raw": b["raw"], "author": b.get("author"),
|
|
235
|
+
"root_status": root, "thread": thread,
|
|
236
|
+
"posts": _dedupe_posts(posts)}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def build_bundle(target: str, *, mode: str = "status", timeout: float = 15,
|
|
240
|
+
with_quotes: bool = False, with_reposts: bool = False,
|
|
241
|
+
limit: int = 20) -> tuple[str, dict[str, Any]]:
|
|
242
|
+
"""拉取并叠加 flags,返回 (status_id, bundle)。"""
|
|
243
|
+
status_id = fx.extract_status_id(target)
|
|
244
|
+
if not status_id:
|
|
245
|
+
raise ValueError(f"无法解析 status id: {target!r}(给推文 URL 或纯数字 ID)")
|
|
246
|
+
mode = (mode or "status").lower()
|
|
247
|
+
if mode not in MODES:
|
|
248
|
+
raise ValueError(f"未知 mode: {mode}(可选 {', '.join(MODES)})")
|
|
249
|
+
|
|
250
|
+
limit = max(1, min(int(limit), 100))
|
|
251
|
+
bundle = _fetch_bundle(status_id, mode, timeout=timeout, limit=limit)
|
|
252
|
+
raw_extra: dict[str, Any] = {}
|
|
253
|
+
|
|
254
|
+
if with_quotes and mode != "quotes":
|
|
255
|
+
try:
|
|
256
|
+
q = fx.get_quotes(status_id, limit=limit, timeout=timeout)
|
|
257
|
+
bundle["quotes"] = [r for r in (q.get("results") or []) if isinstance(r, dict)]
|
|
258
|
+
raw_extra["quotes"] = q.get("raw")
|
|
259
|
+
except fx.FxTwitterError as e:
|
|
260
|
+
raw_extra["quotes_error"] = str(e)
|
|
261
|
+
|
|
262
|
+
if with_reposts and mode != "reposts":
|
|
263
|
+
try:
|
|
264
|
+
r = fx.get_reposts(status_id, limit=limit, timeout=timeout)
|
|
265
|
+
bundle["repost_users"] = [u for u in (r.get("results") or []) if isinstance(u, dict)]
|
|
266
|
+
raw_extra["reposts"] = r.get("raw")
|
|
267
|
+
except fx.FxTwitterError as e:
|
|
268
|
+
raw_extra["reposts_error"] = str(e)
|
|
269
|
+
|
|
270
|
+
if raw_extra:
|
|
271
|
+
base = bundle.get("raw")
|
|
272
|
+
bundle["raw"] = ({**base, **raw_extra} if isinstance(base, dict)
|
|
273
|
+
else {"primary": base, **raw_extra})
|
|
274
|
+
bundle["_extra_quotes"] = bundle.get("quotes") or []
|
|
275
|
+
bundle["_extra_repost_users"] = bundle.get("repost_users") or []
|
|
276
|
+
return status_id, bundle
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _render_text(status_id: str, mode: str, bundle: dict[str, Any]) -> str:
|
|
280
|
+
"""默认 stdout:可读正文(全文,不截断)。"""
|
|
281
|
+
root = bundle.get("root_status") or {}
|
|
282
|
+
handle = _author_handle(root, bundle.get("author"))
|
|
283
|
+
out = [f"@{handle} · {root.get('created_at') or '?'} · mode={mode}",
|
|
284
|
+
f"{root.get('url') or 'https://x.com/i/status/' + status_id}",
|
|
285
|
+
f"互动: likes={root.get('likes')} reposts={root.get('reposts') or root.get('retweets')} "
|
|
286
|
+
f"replies={root.get('replies')} views={root.get('views')}",
|
|
287
|
+
""]
|
|
288
|
+
posts = bundle.get("posts") or []
|
|
289
|
+
for i, p in enumerate(posts, 1):
|
|
290
|
+
if len(posts) > 1:
|
|
291
|
+
out.append(f"--- [{i}/{len(posts)}] ---")
|
|
292
|
+
out.append(status_to_md_block(p, heading_level=2))
|
|
293
|
+
users = bundle.get("_extra_repost_users") or []
|
|
294
|
+
if users:
|
|
295
|
+
out.append(f"--- 转发用户 {len(users)} ---")
|
|
296
|
+
for u in users[:50]:
|
|
297
|
+
out.append(user_to_md_block(u, heading_level=3))
|
|
298
|
+
photos = sum(len(fx.iter_photos(p)) for p in posts)
|
|
299
|
+
videos = sum(len(fx.iter_videos(p)) for p in posts)
|
|
300
|
+
if photos or videos:
|
|
301
|
+
out.append("")
|
|
302
|
+
out.append(f"[媒体] images={photos} videos={videos}"
|
|
303
|
+
f"(用 --out DIR --media 下载到本地)")
|
|
304
|
+
return "\n".join(out)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def to_json(status_id: str, mode: str, bundle: dict[str, Any]) -> dict[str, Any]:
|
|
308
|
+
"""结构化输出:全文 + 媒体 URL + 互动 + 串/引用/转发,原样可消费。"""
|
|
309
|
+
def _brief(st: dict[str, Any]) -> dict[str, Any]:
|
|
310
|
+
return {
|
|
311
|
+
"id": str(st.get("id") or ""),
|
|
312
|
+
"url": st.get("url"),
|
|
313
|
+
"author": (st.get("author") or {}).get("screen_name") if isinstance(st.get("author"), dict) else None,
|
|
314
|
+
"created_at": st.get("created_at"),
|
|
315
|
+
"text": fx.status_text(st),
|
|
316
|
+
"likes": st.get("likes"), "reposts": st.get("reposts") or st.get("retweets"),
|
|
317
|
+
"replies": st.get("replies"), "views": st.get("views"),
|
|
318
|
+
"images": [p.get("url") for p in fx.iter_photos(st) if p.get("url")],
|
|
319
|
+
"videos": [v.get("url") for v in fx.iter_videos(st) if v.get("url")],
|
|
320
|
+
"is_note_tweet": st.get("is_note_tweet"),
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
root = bundle.get("root_status") or {}
|
|
324
|
+
return {
|
|
325
|
+
"schema_version": "argo-tweet/1.0",
|
|
326
|
+
"status_id": status_id,
|
|
327
|
+
"mode": mode,
|
|
328
|
+
"entry_url": root.get("url"),
|
|
329
|
+
"author": bundle.get("author") if isinstance(bundle.get("author"), dict) else None,
|
|
330
|
+
"post_count": len(bundle.get("posts") or []),
|
|
331
|
+
"posts": [_brief(p) for p in (bundle.get("posts") or [])],
|
|
332
|
+
"thread": [_brief(t) for t in (bundle.get("thread") or [])],
|
|
333
|
+
"replies": [_brief(r) for r in (bundle.get("replies") or [])],
|
|
334
|
+
"quotes": [_brief(q) for q in (bundle.get("quotes") or [])],
|
|
335
|
+
"repost_users": [
|
|
336
|
+
{"screen_name": u.get("screen_name") or u.get("username"),
|
|
337
|
+
"name": u.get("name"), "followers": u.get("followers")}
|
|
338
|
+
for u in (bundle.get("repost_users") or []) if isinstance(u, dict)
|
|
339
|
+
],
|
|
340
|
+
"provider": "fxtwitter",
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def pack(target: str, *, mode: str = "status", out_dir: Path | None = None,
|
|
345
|
+
download_media: bool = False, video_quality: str = "max",
|
|
346
|
+
timeout: float = 15, with_quotes: bool = False,
|
|
347
|
+
with_reposts: bool = False, limit: int = 20) -> dict[str, Any]:
|
|
348
|
+
"""完整打包:落盘 post.md / raw.json / manifest.json / CHECKLIST.md(可选媒体)。"""
|
|
349
|
+
t0 = time.time()
|
|
350
|
+
status_id, bundle = build_bundle(
|
|
351
|
+
target, mode=mode, timeout=timeout, with_quotes=with_quotes,
|
|
352
|
+
with_reposts=with_reposts, limit=limit)
|
|
353
|
+
|
|
354
|
+
posts = bundle.get("posts") or []
|
|
355
|
+
users = bundle.get("_extra_repost_users") or []
|
|
356
|
+
if not posts and not users:
|
|
357
|
+
raise ValueError("API 返回空(无帖子、无转发用户)")
|
|
358
|
+
|
|
359
|
+
root = (Path(out_dir).expanduser() if out_dir
|
|
360
|
+
else Path.cwd() / DEFAULT_OUT_PARENT / status_id).resolve()
|
|
361
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
362
|
+
|
|
363
|
+
media_results: list[dict[str, Any]] = []
|
|
364
|
+
md_parts: list[str] = []
|
|
365
|
+
root_status = bundle.get("root_status") or (posts[0] if posts else None)
|
|
366
|
+
handle = _author_handle(root_status or {}, bundle.get("author"))
|
|
367
|
+
summary = fx.status_text(root_status or {})[:60].replace("\n", " ")
|
|
368
|
+
|
|
369
|
+
md_parts += [f"# X 打包 · @{handle}", "",
|
|
370
|
+
f"- **入口 ID**: `{status_id}`", f"- **模式**: `{mode}`",
|
|
371
|
+
f"- **帖子条数**: {len(posts)}"]
|
|
372
|
+
if bundle.get("quotes"):
|
|
373
|
+
md_parts.append(f"- **引用条数**: {len(bundle['quotes'])}")
|
|
374
|
+
if users:
|
|
375
|
+
md_parts.append(f"- **转发用户数**: {len(users)}")
|
|
376
|
+
md_parts += [f"- **打包时间**: {_now_iso()}"]
|
|
377
|
+
if summary:
|
|
378
|
+
md_parts.append(f"- **摘要**: {summary}")
|
|
379
|
+
md_parts += ["", "---", ""]
|
|
380
|
+
|
|
381
|
+
def render_one(st: dict[str, Any]) -> str:
|
|
382
|
+
sid = str(st.get("id") or "unknown")
|
|
383
|
+
photos, videos = collect_media_jobs(st, video_quality=video_quality,
|
|
384
|
+
prefix=_safe_name(sid))
|
|
385
|
+
img_rels, vid_rels = [], []
|
|
386
|
+
for job in photos + videos:
|
|
387
|
+
if download_media and job.get("url"):
|
|
388
|
+
dest = root / job["dest_rel"]
|
|
389
|
+
res = download_file(job["url"], dest,
|
|
390
|
+
timeout=max(timeout * 4, 120))
|
|
391
|
+
res.update(kind=job["kind"], status_id=sid)
|
|
392
|
+
media_results.append(res)
|
|
393
|
+
if res["ok"]:
|
|
394
|
+
(img_rels if job["kind"] == "image" else vid_rels).append(job["dest_rel"])
|
|
395
|
+
else:
|
|
396
|
+
# 未下载:ok=None 表示「没尝试」,不是成功也不是失败——否则
|
|
397
|
+
# 「有 URL」会被记成 images_ok,把「按需」的账算成「已下好」。
|
|
398
|
+
media_results.append({
|
|
399
|
+
"ok": None, "path": None, "bytes": 0,
|
|
400
|
+
"url": job.get("url"), "kind": job["kind"], "status_id": sid,
|
|
401
|
+
"skipped_download": True, "has_url": bool(job.get("url")),
|
|
402
|
+
"error": job.get("skip_reason"), "chosen": job.get("chosen"),
|
|
403
|
+
})
|
|
404
|
+
return status_to_md_block(st, image_rel_paths=img_rels,
|
|
405
|
+
video_rel_paths=vid_rels, heading_level=3)
|
|
406
|
+
|
|
407
|
+
def section(title: str, items: list[dict], *, as_users: bool = False) -> None:
|
|
408
|
+
if not items:
|
|
409
|
+
return
|
|
410
|
+
md_parts.append(f"## {title}")
|
|
411
|
+
md_parts.append("")
|
|
412
|
+
for it in items:
|
|
413
|
+
md_parts.append(user_to_md_block(it, heading_level=3) if as_users
|
|
414
|
+
else render_one(it))
|
|
415
|
+
# 用 extend 不用 +=:嵌套函数里 `md_parts += [...]` 会把 md_parts
|
|
416
|
+
# 变成 section 的**局部名**(列表的 += 也是赋值),前面的 append
|
|
417
|
+
# 随即 UnboundLocalError——所有走 section 的 mode 会当场崩。
|
|
418
|
+
md_parts.extend(["---", ""])
|
|
419
|
+
|
|
420
|
+
if mode == "conversation":
|
|
421
|
+
root_id = str((root_status or {}).get("id") or "")
|
|
422
|
+
thread = [t for t in (bundle.get("thread") or [])
|
|
423
|
+
if str(t.get("id") or "") != root_id]
|
|
424
|
+
tids = {str(t.get("id") or "") for t in thread}
|
|
425
|
+
replies = [r for r in (bundle.get("replies") or [])
|
|
426
|
+
if str(r.get("id") or "") not in tids]
|
|
427
|
+
if root_status:
|
|
428
|
+
section("主帖", [root_status])
|
|
429
|
+
section("Thread", thread)
|
|
430
|
+
section("回复", replies)
|
|
431
|
+
elif mode == "quotes":
|
|
432
|
+
if root_status:
|
|
433
|
+
section("主帖", [root_status])
|
|
434
|
+
section("引用", bundle.get("quotes") or [])
|
|
435
|
+
elif mode == "reposts":
|
|
436
|
+
if root_status:
|
|
437
|
+
section("主帖", [root_status])
|
|
438
|
+
section("转发用户", users, as_users=True)
|
|
439
|
+
elif mode == "thread":
|
|
440
|
+
section("Thread", posts)
|
|
441
|
+
else:
|
|
442
|
+
for st in posts:
|
|
443
|
+
md_parts.append(render_one(st))
|
|
444
|
+
md_parts += ["---", ""]
|
|
445
|
+
|
|
446
|
+
if with_quotes and mode != "quotes" and bundle.get("quotes"):
|
|
447
|
+
section("引用", bundle["quotes"])
|
|
448
|
+
if with_reposts and mode != "reposts" and users:
|
|
449
|
+
section("转发用户", users, as_users=True)
|
|
450
|
+
|
|
451
|
+
post_md = root / "post.md"
|
|
452
|
+
post_md.write_text("\n".join(md_parts), encoding="utf-8")
|
|
453
|
+
raw_path = root / "raw.json"
|
|
454
|
+
raw_path.write_text(dumps_pretty(bundle.get("raw")), encoding="utf-8")
|
|
455
|
+
|
|
456
|
+
def _ok(kind: str, flag: bool) -> list[dict]:
|
|
457
|
+
"""只统计**真尝试过下载**的条目(skipped 的 ok=None,两侧都不算)。"""
|
|
458
|
+
return [m for m in media_results
|
|
459
|
+
if m.get("kind") == kind and not m.get("skipped_download")
|
|
460
|
+
and bool(m.get("ok")) is flag]
|
|
461
|
+
|
|
462
|
+
images_ok, images_fail = _ok("image", True), _ok("image", False)
|
|
463
|
+
videos_ok, videos_fail = _ok("video", True), _ok("video", False)
|
|
464
|
+
text_ok = any(fx.status_text(p) for p in posts) or bool(users)
|
|
465
|
+
quotes = bundle.get("quotes") or []
|
|
466
|
+
|
|
467
|
+
manifest = {
|
|
468
|
+
"schema_version": "argo-tweet/1.0",
|
|
469
|
+
"status_id": status_id, "mode": mode, "packed_at": _now_iso(),
|
|
470
|
+
"elapsed_ms": round((time.time() - t0) * 1000, 1),
|
|
471
|
+
"entry_url": (root_status or {}).get("url"),
|
|
472
|
+
"author": bundle.get("author") if isinstance(bundle.get("author"), dict) else None,
|
|
473
|
+
"post_count": len(posts),
|
|
474
|
+
"post_ids": [str(p.get("id")) for p in posts],
|
|
475
|
+
"quote_count": len(quotes),
|
|
476
|
+
"quote_ids": [str(q.get("id")) for q in quotes],
|
|
477
|
+
"repost_user_count": len(users),
|
|
478
|
+
"repost_handles": [str(u.get("screen_name") or u.get("username") or "")
|
|
479
|
+
for u in users],
|
|
480
|
+
"with_quotes": with_quotes or mode == "quotes",
|
|
481
|
+
"with_reposts": with_reposts or mode == "reposts",
|
|
482
|
+
"limit": limit, "download_media": download_media,
|
|
483
|
+
"video_quality": video_quality, "media": media_results,
|
|
484
|
+
"counts": {
|
|
485
|
+
"posts": len(posts), "quotes": len(quotes), "repost_users": len(users),
|
|
486
|
+
"images_ok": len(images_ok), "images_fail": len(images_fail),
|
|
487
|
+
"videos_ok": len(videos_ok), "videos_fail": len(videos_fail),
|
|
488
|
+
"text_ok": text_ok,
|
|
489
|
+
},
|
|
490
|
+
"paths": {"root": str(root), "post_md": str(post_md),
|
|
491
|
+
"raw_json": str(raw_path)},
|
|
492
|
+
"provider": "fxtwitter",
|
|
493
|
+
}
|
|
494
|
+
manifest_path = root / "manifest.json"
|
|
495
|
+
manifest_path.write_text(dumps_pretty(manifest), encoding="utf-8")
|
|
496
|
+
|
|
497
|
+
n_images = sum(1 for m in media_results if m.get("kind") == "image")
|
|
498
|
+
n_videos = sum(1 for m in media_results if m.get("kind") == "video")
|
|
499
|
+
|
|
500
|
+
def _media_check(label: str, n: int, ok: list, fail: list) -> tuple:
|
|
501
|
+
"""「没下载」与「没媒体」要分开报——前者是用户选择,后者是客观事实。"""
|
|
502
|
+
if not download_media:
|
|
503
|
+
return (label, True,
|
|
504
|
+
f"未下载({n} 待下,--media 开启)" if n else "无媒体(skip)")
|
|
505
|
+
return (label, not fail,
|
|
506
|
+
"无媒体(skip)" if n == 0 else f"ok={len(ok)} fail={len(fail)}")
|
|
507
|
+
|
|
508
|
+
checks = [
|
|
509
|
+
("正文/内容非空", text_ok, "至少一条有 text 或有效列表"),
|
|
510
|
+
("raw.json 已写", raw_path.is_file() and raw_path.stat().st_size > 2, str(raw_path)),
|
|
511
|
+
("post.md 已写", post_md.is_file() and post_md.stat().st_size > 10, str(post_md)),
|
|
512
|
+
_media_check("图片下载", n_images, images_ok, images_fail),
|
|
513
|
+
_media_check("视频下载", n_videos, videos_ok, videos_fail),
|
|
514
|
+
]
|
|
515
|
+
all_pass = all(c[1] for c in checks)
|
|
516
|
+
cl = [f"# CHECKLIST · {status_id}", "",
|
|
517
|
+
f"- 打包时间: {_now_iso()}", f"- 模式: {mode}",
|
|
518
|
+
f"- 结果: **{'PASS' if all_pass else 'PARTIAL'}**", "",
|
|
519
|
+
"| 项 | 状态 | 说明 |", "|----|------|------|"]
|
|
520
|
+
cl += [f"| {n} | {'✅' if ok else '❌'} | {d} |" for n, ok, d in checks]
|
|
521
|
+
if images_fail or videos_fail:
|
|
522
|
+
cl += ["", "## 失败明细", ""]
|
|
523
|
+
cl += [f"- `{m.get('kind')}` {m.get('url') or m.get('path')}: {m.get('error')}"
|
|
524
|
+
for m in images_fail + videos_fail]
|
|
525
|
+
checklist_path = root / "CHECKLIST.md"
|
|
526
|
+
checklist_path.write_text("\n".join(cl + [""]), encoding="utf-8")
|
|
527
|
+
manifest["paths"]["checklist"] = str(checklist_path)
|
|
528
|
+
manifest["checklist_pass"] = all_pass
|
|
529
|
+
manifest_path.write_text(dumps_pretty(manifest), encoding="utf-8")
|
|
530
|
+
return manifest
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def main(argv: list[str] | None = None) -> int:
|
|
534
|
+
ap = argparse.ArgumentParser(
|
|
535
|
+
prog="argo tweet",
|
|
536
|
+
description="X 帖子完整打包(FxTwitter,零 Key;搜索用 argo search)")
|
|
537
|
+
ap.add_argument("target", help="推文 URL 或纯数字 status id")
|
|
538
|
+
ap.add_argument("--mode", choices=list(MODES), default="status",
|
|
539
|
+
help="status(默认,带串时并入)| thread | conversation | quotes | reposts")
|
|
540
|
+
ap.add_argument("--out", type=Path, default=None,
|
|
541
|
+
help=f"落盘目录(默认 ./{DEFAULT_OUT_PARENT}/<id>);给了就写四件套")
|
|
542
|
+
ap.add_argument("--media", action="store_true",
|
|
543
|
+
help="下载图片/视频(需 --out;默认只记 URL 不下载)")
|
|
544
|
+
ap.add_argument("--video-quality", choices=["max", "min", "720"], default="max")
|
|
545
|
+
ap.add_argument("--with-quotes", action="store_true", help="额外附加「引用」分区")
|
|
546
|
+
ap.add_argument("--with-reposts", action="store_true", help="额外附加「转发用户」分区")
|
|
547
|
+
ap.add_argument("--limit", type=int, default=20, help="quotes/reposts 上限(默认 20,上限 100)")
|
|
548
|
+
ap.add_argument("--timeout", type=float, default=15)
|
|
549
|
+
ap.add_argument("--json", action="store_true", help="输出结构化 JSON(全文+媒体 URL+互动)")
|
|
550
|
+
args = ap.parse_args(argv)
|
|
551
|
+
|
|
552
|
+
try:
|
|
553
|
+
if args.out is not None or args.media:
|
|
554
|
+
manifest = pack(
|
|
555
|
+
args.target, mode=args.mode, out_dir=args.out,
|
|
556
|
+
download_media=args.media, video_quality=args.video_quality,
|
|
557
|
+
timeout=args.timeout, with_quotes=args.with_quotes,
|
|
558
|
+
with_reposts=args.with_reposts, limit=args.limit)
|
|
559
|
+
if args.json:
|
|
560
|
+
print(dumps(manifest))
|
|
561
|
+
else:
|
|
562
|
+
c = manifest["counts"]
|
|
563
|
+
extra = "".join(
|
|
564
|
+
f" {k}={c[k]}" for k in ("quotes", "repost_users") if c.get(k))
|
|
565
|
+
print(f"packed → {manifest['paths']['root']}")
|
|
566
|
+
print(f"posts={c['posts']}{extra} "
|
|
567
|
+
f"images={c['images_ok']}/{c['images_ok'] + c['images_fail']} "
|
|
568
|
+
f"videos={c['videos_ok']}/{c['videos_ok'] + c['videos_fail']} "
|
|
569
|
+
f"checklist={'PASS' if manifest.get('checklist_pass') else 'PARTIAL'} "
|
|
570
|
+
f"({manifest['elapsed_ms']}ms)")
|
|
571
|
+
for key in ("post_md", "raw_json", "checklist"):
|
|
572
|
+
print(f" {key:<10} {manifest['paths'][key]}")
|
|
573
|
+
return 0
|
|
574
|
+
|
|
575
|
+
status_id, bundle = build_bundle(
|
|
576
|
+
args.target, mode=args.mode, timeout=args.timeout,
|
|
577
|
+
with_quotes=args.with_quotes, with_reposts=args.with_reposts,
|
|
578
|
+
limit=args.limit)
|
|
579
|
+
if not bundle.get("posts") and not bundle.get("_extra_repost_users"):
|
|
580
|
+
print("API 返回空(无帖子、无转发用户)", file=sys.stderr)
|
|
581
|
+
return 1
|
|
582
|
+
if args.json:
|
|
583
|
+
print(dumps(to_json(status_id, args.mode, bundle)))
|
|
584
|
+
else:
|
|
585
|
+
print(_render_text(status_id, args.mode, bundle))
|
|
586
|
+
return 0
|
|
587
|
+
except ValueError as e:
|
|
588
|
+
print(f"argo tweet: {e}", file=sys.stderr)
|
|
589
|
+
return 1
|
|
590
|
+
except fx.FxTwitterError as e:
|
|
591
|
+
print(f"argo tweet: FxTwitter 错误: {e}", file=sys.stderr)
|
|
592
|
+
return 2
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
if __name__ == "__main__":
|
|
596
|
+
raise SystemExit(main())
|