argo-search 1.0.1 → 2.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (472) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +735 -0
  3. package/README.es.md +777 -0
  4. package/README.ja.md +777 -0
  5. package/README.ko.md +777 -0
  6. package/README.md +778 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/why-better.svg +104 -0
  12. package/assets/readme/workflow.svg +83 -0
  13. package/backends/domain_profiles.json +2356 -96
  14. package/backends/engine_registry.yaml +3428 -503
  15. package/backends/query_synonyms_cn.json +35 -0
  16. package/backends/quota_profiles.json +2488 -160
  17. package/backends/source_types_cn.json +109 -0
  18. package/backends/text_lexicon.json +44 -0
  19. package/bin/argo +772 -0
  20. package/bin/argo.js +102 -39
  21. package/config.yaml +5139 -367
  22. package/cordis.patch.yml +21 -0
  23. package/data/countries.json +40 -0
  24. package/data/lang_matrix/lang_capability.json +842 -0
  25. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  26. package/data/regions_cn.json +1 -0
  27. package/docs/ADDING_NEW_ENGINE.md +184 -0
  28. package/docs/ARGO_INTRO.md +124 -0
  29. package/docs/DATA_SOURCE_GAPS.md +81 -0
  30. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  31. package/docs/ENGINE_CATALOG.md +575 -0
  32. package/docs/MCP_SETUP.md +108 -0
  33. package/docs/PLATFORM_PATHS.md +111 -0
  34. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  35. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  36. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  37. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  38. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  39. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  40. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  41. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  42. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  43. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  44. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  45. package/docs/SEARCH_ARCHIVE.md +115 -0
  46. package/docs/assets/architecture.svg +104 -0
  47. package/docs/assets/hero.svg +98 -0
  48. package/docs/engines/bangumi.md +27 -0
  49. package/docs/engines/biorxiv.md +27 -0
  50. package/docs/engines/clawhub.md +27 -0
  51. package/docs/engines/crt_sh.md +27 -0
  52. package/docs/engines/datacite.md +27 -0
  53. package/docs/engines/deps_dev.md +27 -0
  54. package/docs/engines/douban_movie.md +27 -0
  55. package/docs/engines/endoflife.md +27 -0
  56. package/docs/engines/flk_law.md +27 -0
  57. package/docs/engines/fxtwitter.md +27 -0
  58. package/docs/engines/google_news.md +27 -0
  59. package/docs/engines/gov_regulations.md +27 -0
  60. package/docs/engines/guqian.md +27 -0
  61. package/docs/engines/hackernews.md +27 -0
  62. package/docs/engines/iplant.md +27 -0
  63. package/docs/engines/keenable.md +27 -0
  64. package/docs/engines/met_museum.md +27 -0
  65. package/docs/engines/nasa_images.md +27 -0
  66. package/docs/engines/netease_music.md +27 -0
  67. package/docs/engines/npm.md +27 -0
  68. package/docs/engines/nvd.md +27 -0
  69. package/docs/engines/openreview.md +27 -0
  70. package/docs/engines/openstd.md +27 -0
  71. package/docs/engines/parallel_free.md +27 -0
  72. package/docs/engines/people_daily.md +27 -0
  73. package/docs/engines/realtime_index.md +27 -0
  74. package/docs/engines/sec_edgar.md +27 -0
  75. package/docs/engines/seltz.md +27 -0
  76. package/docs/engines/skillsmp.md +27 -0
  77. package/docs/engines/sspai.md +27 -0
  78. package/docs/engines/std_samr.md +27 -0
  79. package/docs/engines/train.md +27 -0
  80. package/docs/engines/un_comtrade.md +27 -0
  81. package/docs/engines/wikisource.md +27 -0
  82. package/docs/engines/zdic.md +27 -0
  83. package/docs/engines/zenodo.md +27 -0
  84. package/docs/engines/zhihu_hot_app.md +27 -0
  85. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  86. package/engines/README.md +10 -0
  87. package/engines/_template_cli.yaml +39 -0
  88. package/engines/_template_http.yaml +37 -0
  89. package/engines/plugins/_template_plugin.py +50 -0
  90. package/engines/specs/_cisa_kev.yaml +54 -0
  91. package/engines/specs/_osv.yaml +63 -0
  92. package/engines/specs/aljazeera.yaml +43 -0
  93. package/engines/specs/antara.yaml +43 -0
  94. package/engines/specs/artic.yaml +20 -0
  95. package/engines/specs/aviation_weather.yaml +39 -0
  96. package/engines/specs/bangumi.yaml +21 -0
  97. package/engines/specs/biorxiv.yaml +24 -0
  98. package/engines/specs/carbon_intensity.yaml +32 -0
  99. package/engines/specs/cfets.yaml +41 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/defillama.yaml +38 -0
  113. package/engines/specs/deps_dev.yaml +21 -0
  114. package/engines/specs/digup.yaml +36 -0
  115. package/engines/specs/doi.yaml +39 -0
  116. package/engines/specs/douban_movie.yaml +22 -0
  117. package/engines/specs/dw_news.yaml +47 -0
  118. package/engines/specs/egov_law.yaml +21 -0
  119. package/engines/specs/elpais.yaml +43 -0
  120. package/engines/specs/endoflife.yaml +21 -0
  121. package/engines/specs/energy_charts.yaml +21 -0
  122. package/engines/specs/factcheck_org.yaml +52 -0
  123. package/engines/specs/faz.yaml +43 -0
  124. package/engines/specs/federal_register.yaml +57 -0
  125. package/engines/specs/firecrawl.yaml +53 -0
  126. package/engines/specs/flk_law.yaml +24 -0
  127. package/engines/specs/folha.yaml +43 -0
  128. package/engines/specs/france24.yaml +44 -0
  129. package/engines/specs/full_fact.yaml +47 -0
  130. package/engines/specs/fxtwitter.yaml +36 -0
  131. package/engines/specs/gbfs_nyc.yaml +32 -0
  132. package/engines/specs/gdacs.yaml +21 -0
  133. package/engines/specs/gdelt.yaml +64 -0
  134. package/engines/specs/gleif.yaml +55 -0
  135. package/engines/specs/google_news.yaml +22 -0
  136. package/engines/specs/gov_regulations.yaml +38 -0
  137. package/engines/specs/guardian_rss.yaml +59 -0
  138. package/engines/specs/hkex.yaml +40 -0
  139. package/engines/specs/hurriyet.yaml +43 -0
  140. package/engines/specs/iplant.yaml +25 -0
  141. package/engines/specs/jikan.yaml +24 -0
  142. package/engines/specs/jolpica.yaml +20 -0
  143. package/engines/specs/k10plus.yaml +20 -0
  144. package/engines/specs/keenable.yaml +44 -0
  145. package/engines/specs/keenable_free.yaml +36 -0
  146. package/engines/specs/lefigaro.yaml +43 -0
  147. package/engines/specs/listenbrainz.yaml +19 -0
  148. package/engines/specs/matichon.yaml +43 -0
  149. package/engines/specs/met_museum.yaml +21 -0
  150. package/engines/specs/nasa_images.yaml +45 -0
  151. package/engines/specs/netease_music.yaml +38 -0
  152. package/engines/specs/nhk.yaml +43 -0
  153. package/engines/specs/nhtsa_vpic.yaml +20 -0
  154. package/engines/specs/noaa_swpc.yaml +20 -0
  155. package/engines/specs/nvd.yaml +38 -0
  156. package/engines/specs/obis.yaml +20 -0
  157. package/engines/specs/opencitations.yaml +52 -0
  158. package/engines/specs/openf1.yaml +21 -0
  159. package/engines/specs/openfoodfacts.yaml +45 -0
  160. package/engines/specs/openligadb.yaml +20 -0
  161. package/engines/specs/openreview.yaml +37 -0
  162. package/engines/specs/openstd.yaml +21 -0
  163. package/engines/specs/people_daily.yaml +22 -0
  164. package/engines/specs/realtime_index.yaml +34 -0
  165. package/engines/specs/ror.yaml +20 -0
  166. package/engines/specs/satnogs.yaml +20 -0
  167. package/engines/specs/sec_edgar.yaml +46 -0
  168. package/engines/specs/shfe.yaml +39 -0
  169. package/engines/specs/skillsmp.yaml +50 -0
  170. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  171. package/engines/specs/soilgrids.yaml +25 -0
  172. package/engines/specs/sspai.yaml +36 -0
  173. package/engines/specs/stackexchange.yaml +46 -0
  174. package/engines/specs/std_samr.yaml +21 -0
  175. package/engines/specs/tass.yaml +43 -0
  176. package/engines/specs/tavily_free.yaml +56 -0
  177. package/engines/specs/tle_mirror.yaml +21 -0
  178. package/engines/specs/train.yaml +37 -0
  179. package/engines/specs/tvmaze.yaml +20 -0
  180. package/engines/specs/twitter_syndication.yaml +31 -0
  181. package/engines/specs/un_comtrade.yaml +23 -0
  182. package/engines/specs/unpaywall.yaml +54 -0
  183. package/engines/specs/vnpress.yaml +43 -0
  184. package/engines/specs/weather.yaml +32 -0
  185. package/engines/specs/who_don.yaml +21 -0
  186. package/engines/specs/who_gho.yaml +20 -0
  187. package/engines/specs/wikimedia_commons.yaml +34 -0
  188. package/engines/specs/wikisource.yaml +22 -0
  189. package/engines/specs/worms.yaml +19 -0
  190. package/engines/specs/x_trending.yaml +34 -0
  191. package/engines/specs/yna.yaml +43 -0
  192. package/engines/specs/ynet.yaml +43 -0
  193. package/engines/specs/yuanbao.yaml +93 -0
  194. package/engines/specs/zbmath.yaml +40 -0
  195. package/engines/specs/zdic.yaml +22 -0
  196. package/engines/specs/zenodo.yaml +27 -0
  197. package/engines/specs/zhihu_hot_app.yaml +35 -0
  198. package/mcp/clients.yaml +106 -0
  199. package/package.json +55 -12
  200. package/packages/dsh-plugin/README.md +152 -0
  201. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  202. package/packages/dsh-plugin/dsh/client.js +1533 -0
  203. package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
  204. package/packages/dsh-plugin/dsh/index.js +527 -0
  205. package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
  206. package/packages/dsh-plugin/dsh/runner.js +666 -0
  207. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  208. package/packages/dsh-plugin/package.json +38 -0
  209. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  210. package/packages/dsh-plugin/test/client.test.js +406 -0
  211. package/packages/dsh-plugin/test/config.test.js +30 -0
  212. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  213. package/packages/dsh-plugin/test/index.test.js +1382 -0
  214. package/references/query-templates.md +96 -0
  215. package/references/query-templates.yaml +118 -0
  216. package/scripts/ab_eval.py +219 -0
  217. package/scripts/ab_eval_p0p1.py +266 -0
  218. package/scripts/adaptive.py +294 -70
  219. package/scripts/answer.py +154 -0
  220. package/scripts/archive_run.py +591 -0
  221. package/scripts/argo_engine_registry.py +172 -0
  222. package/scripts/argo_paths.py +728 -0
  223. package/scripts/article.py +136 -0
  224. package/scripts/article_meta.py +131 -0
  225. package/scripts/arxiv_source.py +201 -0
  226. package/scripts/badcase_ledger.py +252 -0
  227. package/scripts/batch_probe.py +241 -0
  228. package/scripts/benchmark.py +19 -0
  229. package/scripts/bounded_run.py +144 -0
  230. package/scripts/browser_auth.py +323 -0
  231. package/scripts/budget.py +247 -0
  232. package/scripts/budget_cli.py +93 -0
  233. package/scripts/build_release_assets.sh +65 -0
  234. package/scripts/burstiness_detector.py +147 -0
  235. package/scripts/cache.py +1066 -67
  236. package/scripts/cache_guard.py +271 -0
  237. package/scripts/cache_key_vdom.py +56 -0
  238. package/scripts/cache_similarity.py +170 -0
  239. package/scripts/caliber_isolation.py +195 -0
  240. package/scripts/calibrate_lowquality.py +210 -0
  241. package/scripts/candidate_envelope.py +366 -0
  242. package/scripts/chrome_cdp.py +871 -0
  243. package/scripts/ci_drift_gate.py +90 -0
  244. package/scripts/circuit_breaker.py +389 -0
  245. package/scripts/citations.py +334 -0
  246. package/scripts/cite_cli.py +59 -0
  247. package/scripts/cjk_tokens.py +95 -0
  248. package/scripts/clarify.py +47 -24
  249. package/scripts/cli_io.py +240 -0
  250. package/scripts/config.py +683 -45
  251. package/scripts/config_cache.py +402 -0
  252. package/scripts/conn_pool.py +220 -0
  253. package/scripts/content_boundary.py +46 -0
  254. package/scripts/content_security.py +685 -0
  255. package/scripts/content_signals.py +751 -0
  256. package/scripts/crawl.py +74 -35
  257. package/scripts/cross_lingual_trace.py +293 -0
  258. package/scripts/daemon.py +866 -0
  259. package/scripts/digup_bridge.py +214 -0
  260. package/scripts/dispatch_result.py +98 -0
  261. package/scripts/engine_admission.py +366 -0
  262. package/scripts/engine_dispatch.py +982 -0
  263. package/scripts/engine_env.py +539 -0
  264. package/scripts/engine_failure.py +411 -0
  265. package/scripts/engine_families.py +738 -0
  266. package/scripts/engine_policy.py +218 -0
  267. package/scripts/engine_requires.py +190 -0
  268. package/scripts/engine_status.py +322 -0
  269. package/scripts/engine_validate.py +638 -0
  270. package/scripts/engines.py +799 -294
  271. package/scripts/engines_base.py +1521 -0
  272. package/scripts/engines_builders.py +310 -0
  273. package/scripts/engines_builders_batch10.py +204 -0
  274. package/scripts/engines_builders_batch11.py +174 -0
  275. package/scripts/engines_builders_batch12.py +103 -0
  276. package/scripts/engines_builders_batch13.py +340 -0
  277. package/scripts/engines_builders_batch14.py +656 -0
  278. package/scripts/engines_builders_batch15.py +249 -0
  279. package/scripts/engines_builders_batch9.py +1468 -0
  280. package/scripts/engines_builders_cn.py +2283 -0
  281. package/scripts/engines_builders_data.py +2923 -0
  282. package/scripts/engines_builders_data_macro.py +775 -0
  283. package/scripts/engines_builders_feeds.py +255 -0
  284. package/scripts/engines_builders_intl.py +1239 -0
  285. package/scripts/engines_builders_keyless.py +111 -0
  286. package/scripts/engines_builders_search.py +440 -0
  287. package/scripts/engines_builders_tech.py +1359 -0
  288. package/scripts/evidence.py +620 -166
  289. package/scripts/evidence_loop.py +624 -0
  290. package/scripts/evidence_tier.py +298 -0
  291. package/scripts/except_sets.py +25 -0
  292. package/scripts/extract.py +55 -16
  293. package/scripts/fact_align.py +207 -0
  294. package/scripts/fact_crosscheck.py +230 -0
  295. package/scripts/fetch.py +25 -12
  296. package/scripts/fetch_browser.py +87 -0
  297. package/scripts/fetch_jsrun.py +122 -0
  298. package/scripts/fetch_quality.py +225 -0
  299. package/scripts/fetch_render_tinyfish.py +117 -0
  300. package/scripts/fetch_v3.py +1804 -0
  301. package/scripts/focus_extract.py +350 -0
  302. package/scripts/fulltext_store.py +169 -0
  303. package/scripts/fxtwitter_api.py +325 -0
  304. package/scripts/gen_engine_catalog.py +364 -0
  305. package/scripts/gen_native_tools.py +91 -0
  306. package/scripts/gen_routing_snapshot.py +195 -0
  307. package/scripts/health_check.py +140 -402
  308. package/scripts/health_probe.py +23 -8
  309. package/scripts/hot_state.py +134 -0
  310. package/scripts/http_client.py +1053 -0
  311. package/scripts/image_ops.py +430 -0
  312. package/scripts/image_vision/vision_probe.swift +126 -0
  313. package/scripts/install.ps1 +166 -0
  314. package/scripts/install.sh +221 -0
  315. package/scripts/job.py +1186 -0
  316. package/scripts/jump_resolver.py +235 -0
  317. package/scripts/lang_capability.py +150 -0
  318. package/scripts/lang_detect.py +289 -0
  319. package/scripts/lang_matrix_probe.py +329 -0
  320. package/scripts/lang_pref.py +311 -0
  321. package/scripts/link_source.py +340 -0
  322. package/scripts/local_image.py +970 -0
  323. package/scripts/local_seek.py +402 -0
  324. package/scripts/macro_countries.py +97 -0
  325. package/scripts/matrix_search_eval.py +1187 -0
  326. package/scripts/mcp_compact.py +234 -0
  327. package/scripts/mcp_diag.py +8 -3
  328. package/scripts/mcp_handlers.py +492 -0
  329. package/scripts/mcp_handlers_surface.py +157 -0
  330. package/scripts/mcp_launch.sh +124 -0
  331. package/scripts/mcp_local.py +241 -0
  332. package/scripts/mcp_server.py +125 -419
  333. package/scripts/mcp_setup.py +431 -0
  334. package/scripts/mcp_tools.py +331 -0
  335. package/scripts/mcp_transport.py +191 -0
  336. package/scripts/mcp_warm.py +127 -0
  337. package/scripts/min_version_scan.py +335 -0
  338. package/scripts/net_proxy.py +258 -0
  339. package/scripts/network_aware.py +152 -0
  340. package/scripts/output_transparency.py +200 -0
  341. package/scripts/paid_policy.py +98 -0
  342. package/scripts/paper.py +274 -0
  343. package/scripts/pdf_extract.py +518 -0
  344. package/scripts/phrase_filter.py +66 -0
  345. package/scripts/plan.py +422 -0
  346. package/scripts/postprocess_signals.py +150 -0
  347. package/scripts/pre-commit +48 -0
  348. package/scripts/query_enhance.py +157 -0
  349. package/scripts/query_operators.py +196 -0
  350. package/scripts/query_rewriter.py +116 -7
  351. package/scripts/query_signals.py +300 -0
  352. package/scripts/query_templates.py +167 -0
  353. package/scripts/query_understanding.py +411 -0
  354. package/scripts/query_variants.py +298 -0
  355. package/scripts/quota.py +633 -76
  356. package/scripts/rank_signals.py +775 -0
  357. package/scripts/ranking_eval.py +319 -0
  358. package/scripts/readability_extract.py +433 -0
  359. package/scripts/recompute.py +415 -0
  360. package/scripts/recovery.py +616 -0
  361. package/scripts/recovery_l0.py +160 -0
  362. package/scripts/recovery_runner.py +150 -0
  363. package/scripts/redskill/redskill_engine.py +332 -0
  364. package/scripts/regression_p0p1.py +318 -0
  365. package/scripts/replay_eval.py +658 -0
  366. package/scripts/rerank_tfidf.py +225 -0
  367. package/scripts/research.py +607 -403
  368. package/scripts/research_cli.py +361 -0
  369. package/scripts/research_dossier.py +492 -0
  370. package/scripts/research_expand.py +169 -0
  371. package/scripts/research_gates.py +272 -0
  372. package/scripts/research_report.py +209 -0
  373. package/scripts/research_strategy.py +37 -0
  374. package/scripts/research_work_packages.py +195 -0
  375. package/scripts/result_lang.py +311 -0
  376. package/scripts/retry_policy.py +76 -0
  377. package/scripts/robots_guard.py +259 -0
  378. package/scripts/route.py +921 -254
  379. package/scripts/route_bangs.py +40 -0
  380. package/scripts/route_cache.py +243 -0
  381. package/scripts/route_combo.py +917 -0
  382. package/scripts/route_domains.py +341 -0
  383. package/scripts/route_lang.py +605 -0
  384. package/scripts/route_log.py +63 -0
  385. package/scripts/route_policy.py +250 -0
  386. package/scripts/rrf_classic.py +66 -0
  387. package/scripts/screenshot.py +73 -0
  388. package/scripts/search.py +886 -406
  389. package/scripts/search_attribution.py +185 -0
  390. package/scripts/search_benchmark.py +407 -0
  391. package/scripts/search_cli.py +640 -0
  392. package/scripts/search_entry.py +395 -0
  393. package/scripts/search_output.py +690 -0
  394. package/scripts/search_pipeline.py +926 -0
  395. package/scripts/search_rank.py +991 -0
  396. package/scripts/search_types.py +129 -16
  397. package/scripts/seek_locator.py +68 -0
  398. package/scripts/semantic_evidence.py +391 -0
  399. package/scripts/serp_guard.py +336 -0
  400. package/scripts/serp_kill_log.py +74 -0
  401. package/scripts/single_flight.py +120 -0
  402. package/scripts/social_engines/__init__.py +10 -0
  403. package/scripts/social_engines/bilibili_engine.py +131 -0
  404. package/scripts/social_engines/hackernews_engine.py +113 -0
  405. package/scripts/social_engines/reddit_engine.py +172 -0
  406. package/scripts/social_engines/twitter_engine.py +397 -0
  407. package/scripts/social_engines/v2ex_engine.py +101 -0
  408. package/scripts/social_engines/weibo_engine.py +128 -0
  409. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  410. package/scripts/social_engines/zhihu_engine.py +139 -0
  411. package/scripts/social_research.py +201 -0
  412. package/scripts/sogou_weixin_browser.py +318 -0
  413. package/scripts/source_ledger.py +249 -0
  414. package/scripts/stage_timing.py +58 -0
  415. package/scripts/stats_cli.py +107 -0
  416. package/scripts/straggler_policy.py +60 -0
  417. package/scripts/stylometry_detector.py +231 -0
  418. package/scripts/sync_backends.py +353 -0
  419. package/scripts/sync_installs.py +29 -0
  420. package/scripts/template_cli.py +127 -0
  421. package/scripts/text_lexicon.py +72 -0
  422. package/scripts/tfidf_router.py +109 -37
  423. package/scripts/time_utils.py +233 -0
  424. package/scripts/topic_research_profiles.py +643 -0
  425. package/scripts/train.py +737 -0
  426. package/scripts/tweet.py +596 -0
  427. package/scripts/url_canon.py +217 -0
  428. package/scripts/url_optimize.py +56 -0
  429. package/scripts/url_safety.py +236 -0
  430. package/scripts/usage_log.py +116 -0
  431. package/scripts/usage_log_clean.py +113 -0
  432. package/scripts/v2ex_nodes.py +311 -0
  433. package/scripts/variant_recall.py +188 -0
  434. package/scripts/watch.py +205 -0
  435. package/scripts/wechat_article.py +258 -0
  436. package/scripts/wide_research.py +447 -0
  437. package/scripts/wx.py +349 -0
  438. package/scripts/x_article_fetch.py +141 -0
  439. package/scripts/x_search.py +234 -0
  440. package/scripts/x_trending.py +189 -0
  441. package/scripts/yaml_load.py +67 -0
  442. package/scripts/zhihu_quota.py +139 -0
  443. package/sub-skills/ego-search/SKILL.md +253 -0
  444. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  445. package/sub-skills/ego-search/scripts/merge.py +226 -0
  446. package/sub-skills/ego-search/scripts/quality.py +64 -0
  447. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  448. package/sub-skills/ego-search/scripts/safety.py +109 -0
  449. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  450. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  451. package/sub-skills/js-run/SKILL.md +67 -0
  452. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  453. package/sub-skills/js-run/scripts/shim.js +188 -0
  454. package/sub-skills/local-search/SKILL.md +50 -53
  455. package/sub-skills/local-search/bm25.py +186 -0
  456. package/sub-skills/local-search/cli_runner.py +331 -0
  457. package/sub-skills/local-search/config.yaml +152 -55
  458. package/sub-skills/local-search/engine_registry.py +91 -26
  459. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  460. package/sub-skills/local-search/local_search_adapter.py +13 -2
  461. package/sub-skills/local-search/parse_maps.yaml +40 -42
  462. package/sub-skills/local-search/parsers.py +328 -0
  463. package/sub-skills/local-search/search_v3.py +716 -242
  464. package/sub-skills/local-search/smart_router.py +47 -7
  465. package/sub-skills/local-seek/SKILL.md +110 -0
  466. package/sub-skills/local-seek/config/domains.yaml +20 -0
  467. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  468. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  469. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  470. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  471. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  472. package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
@@ -0,0 +1,252 @@
1
+ #!/usr/bin/env python3
2
+ """badcase_ledger — 坏例账本:把分散三处的失败统一入账、可查、可做发布门禁。
3
+
4
+ 定位(2026-10-11 方案丙):argo 的失败证据分散在三处、形态各异——
5
+ 1. usage_log/query.jsonl:errors[] 非空或 count=0 的真实查询(含 serp_guard
6
+ 整页误杀外露,见 search.py drain 处);
7
+ 2. usage_log/recovery.jsonl:触发过恢复链的查询(triggered=true);
8
+ 3. circuit_breaker.json:state != closed 的引擎(点状快照,引擎级坏例)。
9
+ 本工具把它们归一进 SQLite 账本(<state>/badcase.db),支持 ingest(幂等去重)、
10
+ list、stats 与 check(发布前门禁:近 N 天新增坏例超阈值即非零退出)。
11
+
12
+ 与 replay_eval.py 的分工:replay_eval 用录制重放整条管线评打分质量(离线
13
+ 仲裁);本账本只收纳「真实发生的失败」并让它们可归因、可回归——对应评测
14
+ 体系的 badcase 管理环节:发现 → 入账 → 归因 → 修复后 check 不复发。
15
+
16
+ 数据卫生:engines_used 含表外引擎名(夹具/拼写错)的记录不入账——账本只收
17
+ 真实坏例,与 usage_log_clean 的隔离口径同源(engines.available_engines)。
18
+
19
+ 用法:
20
+ python3 scripts/badcase_ledger.py ingest # 三源归一入账(幂等)
21
+ python3 scripts/badcase_ledger.py list --kind query_error --limit 10
22
+ python3 scripts/badcase_ledger.py stats # 按 kind/引擎聚合
23
+ python3 scripts/badcase_ledger.py check --since 7d --max 50 # 发布门禁
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import argparse
28
+ import hashlib
29
+ import json
30
+ import sqlite3
31
+ import sys
32
+ from collections import Counter
33
+ from datetime import datetime, timedelta
34
+ from pathlib import Path
35
+ from typing import Any
36
+
37
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
38
+
39
+ import argo_paths # noqa: E402
40
+ from engines import available_engines # noqa: E402
41
+ from usage_log import usage_log_dir # noqa: E402
42
+
43
+ _CANON = frozenset(available_engines())
44
+
45
+ _SCHEMA = (
46
+ "CREATE TABLE IF NOT EXISTS badcases ("
47
+ " id INTEGER PRIMARY KEY,"
48
+ " kind TEXT NOT NULL," # query_error | empty_result | recovery | breaker
49
+ " ts TEXT NOT NULL," # 原始 ISO 时间串;breaker 快照用入账时刻
50
+ " query TEXT NOT NULL DEFAULT '',"
51
+ " engine TEXT NOT NULL DEFAULT '',"
52
+ " detail TEXT NOT NULL DEFAULT '',"
53
+ " fingerprint TEXT NOT NULL UNIQUE)"
54
+ )
55
+
56
+
57
+ def db_path() -> Path:
58
+ return argo_paths.state_path("badcase.db")
59
+
60
+
61
+ def _connect() -> sqlite3.Connection:
62
+ con = sqlite3.connect(str(db_path()), timeout=8)
63
+ con.execute(_SCHEMA)
64
+ return con
65
+
66
+
67
+ def _rows_from_query_log(path: Path) -> list[dict[str, Any]]:
68
+ if not path.exists():
69
+ return []
70
+ out = []
71
+ for line in path.read_text(encoding="utf-8").splitlines():
72
+ if not line.strip():
73
+ continue
74
+ try:
75
+ r = json.loads(line)
76
+ except json.JSONDecodeError:
77
+ continue
78
+ raw_used = r.get("engines_used") or []
79
+ used = [e for e in raw_used if e in _CANON]
80
+ if not used and raw_used:
81
+ continue # 表外名占满 → 夹具行,不入账(口径同 usage_log_clean)
82
+ raw_errors = r.get("errors")
83
+ # 生产日志里 errors 有两种形态:字符串列表(主流)与 int 计数(部分
84
+ # 旧条目)——统一成字符串列表再取前几条
85
+ errors = (raw_errors if isinstance(raw_errors, list)
86
+ else ([str(raw_errors)] if raw_errors else []))
87
+ if not errors and r.get("count"):
88
+ continue # 正常条目不是坏例
89
+ if errors:
90
+ out.append({"kind": "query_error", "ts": r.get("ts", ""),
91
+ "query": r.get("query", ""),
92
+ "engine": ",".join(used[:3]),
93
+ "detail": " | ".join(str(e)[:160] for e in errors[:3])})
94
+ else:
95
+ out.append({"kind": "empty_result", "ts": r.get("ts", ""),
96
+ "query": r.get("query", ""),
97
+ "engine": ",".join(used[:3]),
98
+ "detail": f"count=0, mode={r.get('mode')}"})
99
+ return out
100
+
101
+
102
+ def _rows_from_recovery_log(path: Path) -> list[dict[str, Any]]:
103
+ if not path.exists():
104
+ return []
105
+ out = []
106
+ for line in path.read_text(encoding="utf-8").splitlines():
107
+ if not line.strip():
108
+ continue
109
+ try:
110
+ r = json.loads(line)
111
+ except json.JSONDecodeError:
112
+ continue
113
+ if not r.get("triggered"):
114
+ continue
115
+ out.append({"kind": "recovery", "ts": r.get("ts", ""),
116
+ "query": r.get("query", ""),
117
+ "engine": "",
118
+ "detail": (f"level={r.get('level_used')} "
119
+ f"strategy={r.get('strategy_used')} "
120
+ f"recovered={r.get('recovered')}")})
121
+ return out
122
+
123
+
124
+ def _rows_from_breaker(path: Path) -> list[dict[str, Any]]:
125
+ if not path.exists():
126
+ return []
127
+ try:
128
+ engines = json.loads(path.read_text(encoding="utf-8")).get("engines", {})
129
+ except (json.JSONDecodeError, OSError):
130
+ return []
131
+ out = []
132
+ now = datetime.now().astimezone().isoformat(timespec="seconds")
133
+ for name, st in sorted(engines.items()):
134
+ if st.get("state") == "closed":
135
+ continue
136
+ out.append({"kind": "breaker", "ts": now, "query": "", "engine": name,
137
+ "detail": (f"state={st.get('state')} "
138
+ f"failures={st.get('failures')} "
139
+ f"opens={st.get('opens')}")})
140
+ return out
141
+
142
+
143
+ def ingest() -> dict:
144
+ log_dir = usage_log_dir()
145
+ rows = (_rows_from_query_log(log_dir / "query.jsonl")
146
+ + _rows_from_recovery_log(log_dir / "recovery.jsonl")
147
+ + _rows_from_breaker(argo_paths.state_path("circuit_breaker.json")))
148
+ con = _connect()
149
+ added = 0
150
+ try:
151
+ for r in rows:
152
+ fp = hashlib.sha1("|".join(
153
+ [str(r["kind"]), str(r["ts"]), str(r["query"]),
154
+ str(r["engine"]), str(r["detail"])[:120]])
155
+ .encode("utf-8")).hexdigest()
156
+ cur = con.execute(
157
+ "INSERT OR IGNORE INTO badcases(ts, query, engine, detail,"
158
+ " kind, fingerprint) VALUES (?,?,?,?,?,?)",
159
+ (r["ts"], r["query"], r["engine"], r["detail"],
160
+ r["kind"], fp))
161
+ added += cur.rowcount
162
+ total = con.execute("SELECT COUNT(*) FROM badcases").fetchone()[0]
163
+ con.commit()
164
+ finally:
165
+ con.close()
166
+ return {"sources": len(rows), "added": added, "total": total,
167
+ "db": str(db_path())}
168
+
169
+
170
+ def _parse_since(since: str) -> datetime:
171
+ """7d / 24h / 2026-10-01 → datetime(本地时区)。"""
172
+ since = since.strip().lower()
173
+ now = datetime.now().astimezone()
174
+ if since.endswith("d") and since[:-1].isdigit():
175
+ return now - timedelta(days=int(since[:-1]))
176
+ if since.endswith("h") and since[:-1].isdigit():
177
+ return now - timedelta(hours=int(since[:-1]))
178
+ return datetime.fromisoformat(since).astimezone()
179
+
180
+
181
+ def _fetch(kind: str | None, since: datetime | None,
182
+ limit: int) -> list[dict[str, Any]]:
183
+ con = _connect()
184
+ try:
185
+ sql = ("SELECT ts, kind, query, engine, detail FROM badcases"
186
+ " WHERE 1=1")
187
+ params: list = []
188
+ if kind:
189
+ sql += " AND kind = ?"
190
+ params.append(kind)
191
+ if since is not None:
192
+ sql += " AND ts >= ?"
193
+ params.append(since.isoformat(timespec="seconds"))
194
+ sql += " ORDER BY ts DESC LIMIT ?"
195
+ params.append(limit)
196
+ keys = ["ts", "kind", "query", "engine", "detail"]
197
+ return [dict(zip(keys, row)) for row in con.execute(sql, params)]
198
+ finally:
199
+ con.close()
200
+
201
+
202
+ def check(since: str, max_new: int) -> dict:
203
+ rows = _fetch(None, _parse_since(since), limit=10_000)
204
+ ok = len(rows) <= max_new
205
+ return {"since": since, "new_badcases": len(rows), "max": max_new,
206
+ "ok": ok}
207
+
208
+
209
+ def stats() -> dict:
210
+ con = _connect()
211
+ try:
212
+ by_kind = dict(con.execute(
213
+ "SELECT kind, COUNT(*) FROM badcases GROUP BY kind").fetchall())
214
+ by_engine = dict(con.execute(
215
+ "SELECT engine, COUNT(*) FROM badcases WHERE engine != ''"
216
+ " GROUP BY engine ORDER BY COUNT(*) DESC LIMIT 8").fetchall())
217
+ total = con.execute("SELECT COUNT(*) FROM badcases").fetchone()[0]
218
+ finally:
219
+ con.close()
220
+ return {"total": total, "by_kind": by_kind, "top_engines": by_engine}
221
+
222
+
223
+ def main(argv: list[str] | None = None) -> int:
224
+ ap = argparse.ArgumentParser(description="坏例账本(badcase 管理)")
225
+ sub = ap.add_subparsers(dest="cmd", required=True)
226
+ sub.add_parser("ingest")
227
+ p_list = sub.add_parser("list")
228
+ p_list.add_argument("--kind", default=None,
229
+ choices=["query_error", "empty_result",
230
+ "recovery", "breaker"])
231
+ p_list.add_argument("--limit", type=int, default=10)
232
+ sub.add_parser("stats")
233
+ p_check = sub.add_parser("check")
234
+ p_check.add_argument("--since", default="7d")
235
+ p_check.add_argument("--max", type=int, default=50)
236
+ args = ap.parse_args(argv)
237
+ if args.cmd == "ingest":
238
+ print(json.dumps(ingest(), ensure_ascii=False))
239
+ elif args.cmd == "list":
240
+ for row in _fetch(args.kind, None, args.limit):
241
+ print(json.dumps(row, ensure_ascii=False))
242
+ elif args.cmd == "stats":
243
+ print(json.dumps(stats(), ensure_ascii=False))
244
+ elif args.cmd == "check":
245
+ report = check(args.since, args.max)
246
+ print(json.dumps(report, ensure_ascii=False))
247
+ return 0 if report["ok"] else 1
248
+ return 0
249
+
250
+
251
+ if __name__ == "__main__":
252
+ sys.exit(main())
@@ -0,0 +1,241 @@
1
+ #!/usr/bin/env python3
2
+ """batch_probe.py — 批量 URL 预检报告。
3
+
4
+ 第一性原理:批量抓取最贵的失败方式是「跑到一半才知道一半拿不到」。
5
+ 整批直接开工,失败会在中途以各种形态冒出来(登录墙、链接已死、
6
+ 内容早就有),此时预算已经花掉一半。预检把可预判的失败挪到开工前,
7
+ 先给一张「什么能拿、什么拿不到、为什么」的清单,再决定怎么执行。
8
+
9
+ 条目分四类:
10
+ unsupported 无法处理:不是合法的 http(s) URL——执行前就该剔除
11
+ needs_auth 存在但需要登录态:用户能解决(先补凭证),提前列出
12
+ already_have 本地归档已有该 URL 的结果:跳过,不重复抓
13
+ not_found 探测发现已死(404/410):从批次里剔除或换源
14
+ (仅在 --probe 开启时判定;不探测不臆测)
15
+
16
+ 结论三档:
17
+ go 全部可执行
18
+ go_with_skips 部分可执行(清单随报告给出)
19
+ stop 没有可执行条目
20
+
21
+ 两条纪律:
22
+ - needs_auth 是唯一「用户可行动」的问题,报告里单独一节;
23
+ unsupported/not_found 是平台侧事实,列出即可。
24
+ - 网络探测失败(连不上)不产生判定——探测不到不等于不存在,
25
+ 不能把「可能的问题」换成「确定的误杀」。未探测的条目类别为 None,
26
+ 只记 unknown,不影响 go 判定。
27
+
28
+ 本模块只做判定,不做抓取:网络探测是可选步骤,抓取由 article/crawl
29
+ 等执行器负责。判定与 I/O 分离,保证纯函数部分可单测。
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import argparse
35
+ import sys
36
+ import urllib.parse
37
+ from concurrent.futures import ThreadPoolExecutor
38
+ from pathlib import Path
39
+ from typing import Any
40
+ from cli_io import dumps
41
+
42
+ UNSUPPORTED = "unsupported"
43
+ NEEDS_AUTH = "needs_auth"
44
+ ALREADY_HAVE = "already_have"
45
+ NOT_FOUND = "not_found"
46
+ UNKNOWN = "unknown"
47
+
48
+ # 用户可自行解决的问题(报告分节与文案依据)
49
+ ACTIONABLE = {NEEDS_AUTH}
50
+
51
+ # 常年登录墙的站点(公开内容也要登录才能看)。这里只列「正文必须登录」
52
+ # 的域——「有登录墙但首页可读」的站不列,避免误杀。
53
+ _KNOWN_LOGIN_WALLS = (
54
+ "x.com", "twitter.com", "instagram.com", "threads.net",
55
+ "facebook.com", "patreon.com", "medium.com",
56
+ "linux.do", "bbs.zhihu.com",
57
+ )
58
+
59
+ # 登录墙里的例外:单条推文有免登录通道(twitter_syndication 引擎走
60
+ # cdn.syndication.twimg.com)。判 needs_auth 会把用户支去「补凭证」,
61
+ # 而这条 URL 其实现在就能抓——同一批次里两个功能互相打架。
62
+ _LOGIN_WALL_EXEMPT_HOSTS = ("x.com", "twitter.com")
63
+
64
+
65
+ def _has_login_free_channel(host: str, url: str) -> bool:
66
+ """该 URL 是否走 argo 已有的免登录通道(目前仅单条推文)。"""
67
+ if not any(host == d or host.endswith("." + d)
68
+ for d in _LOGIN_WALL_EXEMPT_HOSTS):
69
+ return False
70
+ try:
71
+ from engines_builders_intl import extract_tweet_id
72
+ except ImportError:
73
+ return False
74
+ return extract_tweet_id(url) is not None
75
+
76
+
77
+ _TIER_LIMITS = {"probe": 8}
78
+
79
+
80
+ def classify_url(url: str, known_urls: set[str] | None = None) -> str:
81
+ """本地规则分类(无网络)。返回四类之一或 UNKNOWN(合法但无本地证据)。"""
82
+ u = (url or "").strip()
83
+ if not u:
84
+ return UNSUPPORTED
85
+ parsed = urllib.parse.urlparse(u)
86
+ if parsed.scheme not in ("http", "https") or not parsed.hostname:
87
+ return UNSUPPORTED
88
+ host = parsed.hostname.lower()
89
+ if known_urls and u in known_urls:
90
+ return ALREADY_HAVE
91
+ if any(host == d or host.endswith("." + d) for d in _KNOWN_LOGIN_WALLS):
92
+ # 例外的判定要用原始 URL(推文 ID 在路径/查询串里,不是 host 的一部分)
93
+ if _has_login_free_channel(host, u):
94
+ return UNKNOWN
95
+ return NEEDS_AUTH
96
+ return UNKNOWN
97
+
98
+
99
+ def probe_liveness(url: str, timeout: float = 6.0) -> str | None:
100
+ """网络探测单条 URL 的存活。
101
+
102
+ 返回 NOT_FOUND(404/410,确定已死)、UNKNOWN(任何其他结局——
103
+ 包括连接失败:连不上不是「不存在」的证据)。
104
+ """
105
+ try:
106
+ from http_client import fetch_url
107
+ resp = fetch_url(url, timeout=timeout)
108
+ status = resp.get("status") or 0
109
+ if status in (404, 410):
110
+ return NOT_FOUND
111
+ return UNKNOWN
112
+ except Exception:
113
+ return UNKNOWN
114
+
115
+
116
+ def probe_batch(
117
+ urls: list[str],
118
+ *,
119
+ known_urls: set[str] | None = None,
120
+ do_probe: bool = False,
121
+ ) -> dict[str, Any]:
122
+ """对一批 URL 生成预检报告。
123
+
124
+ 返回 {
125
+ total, ready, verdict,
126
+ items: [{url, problem(或 None), actionable}],
127
+ ready_urls, # 可执行清单
128
+ by_problem: {类别: [url]},
129
+ }
130
+ """
131
+ known = known_urls or set()
132
+ items: list[dict[str, Any]] = []
133
+
134
+ # 网络探测只对「本地无证据」的条目做——unsupported/auth/已归档的
135
+ # 判定不需要网络;探测并发受 TIER_LIMITS 约束,不放大源站压力
136
+ local_kinds = [classify_url(u, known) for u in urls]
137
+ net_kinds: dict[int, str] = {}
138
+ if do_probe:
139
+ candidates = [(i, u) for i, (u, k) in enumerate(zip(urls, local_kinds))
140
+ if k == UNKNOWN]
141
+ if candidates:
142
+ with ThreadPoolExecutor(
143
+ max_workers=min(len(candidates), _TIER_LIMITS["probe"])) as ex:
144
+ for (i, _u), kind in zip(candidates,
145
+ ex.map(lambda t: probe_liveness(t[1]),
146
+ candidates)):
147
+ net_kinds[i] = kind or UNKNOWN
148
+
149
+ for i, u in enumerate(urls):
150
+ kind = local_kinds[i]
151
+ if kind == UNKNOWN and i in net_kinds:
152
+ kind = net_kinds[i]
153
+ items.append({
154
+ "url": u,
155
+ "problem": kind if kind != UNKNOWN else None,
156
+ "actionable": kind in ACTIONABLE,
157
+ })
158
+
159
+ ready = [it for it in items if it["problem"] is None]
160
+ by_problem: dict[str, list[str]] = {}
161
+ for it in items:
162
+ if it["problem"]:
163
+ by_problem.setdefault(it["problem"], []).append(it["url"])
164
+
165
+ if not ready:
166
+ verdict = "stop"
167
+ elif len(ready) < len(items):
168
+ verdict = "go_with_skips"
169
+ else:
170
+ verdict = "go"
171
+
172
+ return {
173
+ "total": len(items),
174
+ "ready": len(ready),
175
+ "verdict": verdict,
176
+ "items": items,
177
+ "ready_urls": [it["url"] for it in ready],
178
+ "by_problem": by_problem,
179
+ }
180
+
181
+
182
+ def _format_report(report: dict[str, Any]) -> str:
183
+ """纯文本报告(给人看)。"""
184
+ lines = [
185
+ f"预检报告:共 {report['total']} 条,可执行 {report['ready']} 条"
186
+ f"(verdict={report['verdict']})",
187
+ "",
188
+ ]
189
+ labels = {
190
+ NEEDS_AUTH: "需要登录态(可行动:先补凭证再抓)",
191
+ UNSUPPORTED: "无法处理(非 http(s) URL)",
192
+ ALREADY_HAVE: "本地归档已有(跳过不重抓)",
193
+ NOT_FOUND: "探测已死(404/410)",
194
+ }
195
+ for kind in (NEEDS_AUTH, UNSUPPORTED, ALREADY_HAVE, NOT_FOUND):
196
+ urls = report["by_problem"].get(kind) or []
197
+ if urls:
198
+ lines.append(f"■ {labels[kind]}({len(urls)})")
199
+ lines.extend(f" - {u}" for u in urls)
200
+ lines.append("")
201
+ lines.append(f"■ 可执行清单({len(report['ready_urls'])})")
202
+ lines.extend(f" - {u}" for u in report["ready_urls"])
203
+ return "\n".join(lines)
204
+
205
+
206
+ def main() -> int:
207
+ ap = argparse.ArgumentParser(description="批量 URL 预检")
208
+ ap.add_argument("urls", nargs="*", help="URL 列表(空格分隔)")
209
+ ap.add_argument("--file", "-f", help="从文件读取(每行一条 URL)")
210
+ ap.add_argument("--json", action="store_true", help="JSON 输出")
211
+ ap.add_argument("--probe", action="store_true",
212
+ help="联网探测存活(404/410 判死;默认纯本地规则)")
213
+ args = ap.parse_args()
214
+
215
+ urls = list(args.urls)
216
+ if args.file:
217
+ p = Path(args.file).expanduser()
218
+ if not p.exists():
219
+ print(f"文件不存在: {p}", file=sys.stderr)
220
+ return 1
221
+ urls.extend(
222
+ line.strip() for line in p.read_text(encoding="utf-8").splitlines()
223
+ if line.strip() and not line.strip().startswith("#"))
224
+ if not urls:
225
+ from cli_io import read_stdin_if_piped
226
+ urls = [line.strip() for line in read_stdin_if_piped().splitlines()
227
+ if line.strip()]
228
+ if not urls:
229
+ ap.print_help()
230
+ return 1
231
+
232
+ report = probe_batch(urls, do_probe=args.probe)
233
+ if args.json:
234
+ print(dumps(report))
235
+ else:
236
+ print(_format_report(report))
237
+ return 0
238
+
239
+
240
+ if __name__ == "__main__":
241
+ sys.exit(main())
@@ -10,7 +10,16 @@ benchmark.py — 性能测试
10
10
  """
11
11
  from __future__ import annotations
12
12
 
13
+ # 状态隔离(2026-10-10 方案甲):走真实搜索路径的脚本会写生产状态(实测夹具
14
+ # 引擎已混入 circuit_breaker/usage_log)。须先于 argo 状态模块导入重定向
15
+ # (机制见 argo_paths.isolate_state_dir docstring);被测试导入时不生效。
16
+ if __name__ == "__main__":
17
+ from argo_paths import isolate_state_dir
18
+ isolate_state_dir("argo-bench")
19
+
20
+ import atexit
13
21
  import json
22
+ import shutil
14
23
  import sys
15
24
  import time
16
25
  import os
@@ -18,6 +27,16 @@ import os
18
27
  SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
19
28
  sys.path.insert(0, SCRIPT_DIR)
20
29
 
30
+ # 状态隔离必须早于 cache / route 的 import:两者在模块级就按当时的
31
+ # ARGO_STATE_DIR 定下路径常量(cache.db / adaptive.db),import 之后再设环境
32
+ # 变量对它们无效。本脚本会真读写缓存与学习库——不隔离就会把 query_0..9 这类
33
+ # 夹具写进生产 cache.db,且缓存命中率是拿生产热缓存量出来的(数字失真)。
34
+ # 无法像 search_benchmark.py 那样放进 main():本文件的 cache/route 是模块级
35
+ # import,main() 时路径早已钉死。
36
+ import argo_paths
37
+ _STATE_DIR = argo_paths.isolate_state_dir("argo-bench")
38
+ atexit.register(shutil.rmtree, _STATE_DIR, ignore_errors=True)
39
+
21
40
  from tfidf_router import get_router, semantic_route
22
41
  from cache import SearchCache
23
42
  from route import route_query
@@ -0,0 +1,144 @@
1
+ #!/usr/bin/env python3
2
+ """bounded_run.py — 有界并发执行:总墙钟时间一到就返回,不被慢任务拖住。
3
+
4
+ 背景
5
+ ----
6
+ 仓库里多处用 ``with ThreadPoolExecutor(...) as ex:`` 配合
7
+ ``as_completed(futures, timeout=T)`` 想实现「最多等 T 秒」。但 ``with`` 退出时
8
+ 会执行 ``shutdown(wait=True)``,把所有还在跑的任务 join 完才真正返回——即使
9
+ ``as_completed`` 已经超时,调用方仍会被一个卡住的网络读拖住,T 秒预算形同虚设
10
+ (实测单查询被拖到 76s);而且超时抛出的 TimeoutError 若没人接,会直接中断整次
11
+ 调用,已经拿到的结果也丢了。
12
+
13
+ 做法
14
+ ----
15
+ 改用 daemon 线程加轮询:
16
+
17
+ * 任务跑在 daemon 线程上,到点就返回已完成的结果,**不去 join 还没完成的任务**
18
+ (daemon 线程不会阻止进程退出,各任务内部仍有自己的超时保底);
19
+ * 单个任务抛异常只记成该任务失败,不影响其它任务;
20
+ * 总墙钟时间被 ``wait_s`` 硬性约束,不会超出。
21
+
22
+ search.py 的引擎并发已经是这套做法,这里抽成通用实现,供 research / social /
23
+ crawl / fetch 等复用,避免同一类问题在各处重复出现。
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import threading
29
+ import time
30
+ from typing import Any, Callable, Generic, TypeVar
31
+
32
+ I = TypeVar("I")
33
+ O = TypeVar("O")
34
+
35
+
36
+ class TaskError(Generic[I]):
37
+ """单个任务执行时抛了异常。用对象而非魔法字符串承载,避免和任务正常返回值混淆。"""
38
+
39
+ __slots__ = ("exc", "item")
40
+
41
+ def __init__(self, item: I, exc: BaseException):
42
+ self.item = item
43
+ self.exc = exc
44
+
45
+ def __repr__(self) -> str: # pragma: no cover - 调试辅助
46
+ return f"TaskError(item={self.item!r}, exc={self.exc!r})"
47
+
48
+
49
+ def run_bounded(
50
+ items: list[I],
51
+ worker: Callable[[I], O],
52
+ wait_s: float,
53
+ max_workers: int = 3,
54
+ *,
55
+ enough: Callable[[list[tuple[I, Any]]], bool] | None = None,
56
+ stagger_s: float = 0.0,
57
+ poll: float = 0.01,
58
+ ) -> tuple[list[tuple[I, Any]], list[I]]:
59
+ """并发执行 ``worker(item)``,总等待不超过 ``wait_s`` 秒。
60
+
61
+ 返回 ``(finished, unfinished)``:
62
+
63
+ * ``finished``:已完成任务的 ``(item, value)`` 列表,按完成先后排列;
64
+ 任务抛异常时 ``value`` 是 :class:`TaskError`,调用方按需转成失败占位。
65
+ * ``unfinished``:到点仍没完成(或被 ``enough`` 提前收尾时还在跑)的 item 列表,
66
+ 调用方可据此补超时提示。这些任务不会被 join,本函数因此能准时返回。
67
+
68
+ ``enough(finished)`` 返回 True 时提前收尾:不再启动排队中的剩余任务,已在跑的
69
+ 任务归入 unfinished。``max_workers`` 限制同时在跑的任务数。
70
+
71
+ ``stagger_s`` 起步间隔(秒),单调:0 = 不节流(立即填满并发位);值越大越接近
72
+ 严格串行(大于单任务超时上限即完全退回串行)。仅在**有任务在跑**时节制——
73
+ 对冲语义(首任务先跑,宽限窗内交付就只付一次调用),而非无条件串行化;在跑
74
+ 任务数为 0 时立即补发(前一个已返回,没有可对冲的对象,等满窗纯属浪费)。
75
+ engine_dispatch 的 hedged race 与 search_pipeline 的恢复链是同一哲学的两个
76
+ 配置:改这里的起步语义,两边同时生效。
77
+ """
78
+ queue = list(items)
79
+ running: list[tuple[threading.Thread, I, dict]] = []
80
+ finished: list[tuple[I, Any]] = []
81
+ deadline = time.monotonic() + max(0.0, float(wait_s))
82
+ last_start: float | None = None
83
+ done_evt = threading.Event()
84
+
85
+ def _start(item: I) -> None:
86
+ holder: dict = {}
87
+
88
+ def _work() -> None:
89
+ try:
90
+ holder["v"] = worker(item)
91
+ except BaseException as exc: # noqa: BLE001 - 任意任务异常都要隔离,不能让一个任务炸掉整组
92
+ holder["v"] = TaskError(item, exc)
93
+ finally:
94
+ done_evt.set()
95
+
96
+ th = threading.Thread(target=_work, daemon=True)
97
+ th.start()
98
+ running.append((th, item, holder))
99
+
100
+ while queue or running:
101
+ done_evt.clear()
102
+ if time.monotonic() >= deadline:
103
+ break
104
+ # 按并发上限补启动排队任务
105
+ while queue and len(running) < max(1, int(max_workers)):
106
+ if (stagger_s > 0 and running and last_start is not None
107
+ and (time.monotonic() - last_start) < stagger_s):
108
+ break # 起步节流:等满 stagger 再补发(对冲)
109
+ _start(queue.pop(0))
110
+ last_start = time.monotonic()
111
+ progressed = False
112
+ for entry in list(running):
113
+ th, item, holder = entry
114
+ if th.is_alive():
115
+ continue
116
+ running.remove(entry)
117
+ value = holder.get("v", TaskError(item, RuntimeError("任务结束但没有返回值")))
118
+ finished.append((item, value))
119
+ progressed = True
120
+ if enough is not None and enough(finished):
121
+ queue.clear()
122
+ break
123
+ if enough is not None and enough(finished):
124
+ break
125
+ if not progressed:
126
+ remain = deadline - time.monotonic()
127
+ if remain <= 0:
128
+ break
129
+ wait_step = remain
130
+ if queue and stagger_s > 0 and last_start is not None:
131
+ wait_step = min(wait_step, max(0.0, stagger_s - (time.monotonic() - last_start)))
132
+ done_evt.wait(min(poll, wait_step) if poll > 0 else wait_step)
133
+
134
+ # 收尾:还活着的不等待(关键:不 join),计入 unfinished;恰在末次轮询后完成的
135
+ # 照常收进来,避免结果悄悄丢失。因并发上限到点仍没排上起跑的任务同样算未完成,
136
+ # 不能让它既不在 finished、也不在 unfinished 里被悄悄丢掉。
137
+ unfinished: list[I] = list(queue)
138
+ for th, item, holder in running:
139
+ if th.is_alive():
140
+ unfinished.append(item)
141
+ else:
142
+ value = holder.get("v", TaskError(item, RuntimeError("任务结束但没有返回值")))
143
+ finished.append((item, value))
144
+ return finished, unfinished