argo-search 1.0.1 → 2.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (472) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +735 -0
  3. package/README.es.md +777 -0
  4. package/README.ja.md +777 -0
  5. package/README.ko.md +777 -0
  6. package/README.md +778 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/why-better.svg +104 -0
  12. package/assets/readme/workflow.svg +83 -0
  13. package/backends/domain_profiles.json +2356 -96
  14. package/backends/engine_registry.yaml +3428 -503
  15. package/backends/query_synonyms_cn.json +35 -0
  16. package/backends/quota_profiles.json +2488 -160
  17. package/backends/source_types_cn.json +109 -0
  18. package/backends/text_lexicon.json +44 -0
  19. package/bin/argo +772 -0
  20. package/bin/argo.js +102 -39
  21. package/config.yaml +5139 -367
  22. package/cordis.patch.yml +21 -0
  23. package/data/countries.json +40 -0
  24. package/data/lang_matrix/lang_capability.json +842 -0
  25. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  26. package/data/regions_cn.json +1 -0
  27. package/docs/ADDING_NEW_ENGINE.md +184 -0
  28. package/docs/ARGO_INTRO.md +124 -0
  29. package/docs/DATA_SOURCE_GAPS.md +81 -0
  30. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  31. package/docs/ENGINE_CATALOG.md +575 -0
  32. package/docs/MCP_SETUP.md +108 -0
  33. package/docs/PLATFORM_PATHS.md +111 -0
  34. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  35. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  36. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  37. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  38. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  39. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  40. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  41. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  42. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  43. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  44. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  45. package/docs/SEARCH_ARCHIVE.md +115 -0
  46. package/docs/assets/architecture.svg +104 -0
  47. package/docs/assets/hero.svg +98 -0
  48. package/docs/engines/bangumi.md +27 -0
  49. package/docs/engines/biorxiv.md +27 -0
  50. package/docs/engines/clawhub.md +27 -0
  51. package/docs/engines/crt_sh.md +27 -0
  52. package/docs/engines/datacite.md +27 -0
  53. package/docs/engines/deps_dev.md +27 -0
  54. package/docs/engines/douban_movie.md +27 -0
  55. package/docs/engines/endoflife.md +27 -0
  56. package/docs/engines/flk_law.md +27 -0
  57. package/docs/engines/fxtwitter.md +27 -0
  58. package/docs/engines/google_news.md +27 -0
  59. package/docs/engines/gov_regulations.md +27 -0
  60. package/docs/engines/guqian.md +27 -0
  61. package/docs/engines/hackernews.md +27 -0
  62. package/docs/engines/iplant.md +27 -0
  63. package/docs/engines/keenable.md +27 -0
  64. package/docs/engines/met_museum.md +27 -0
  65. package/docs/engines/nasa_images.md +27 -0
  66. package/docs/engines/netease_music.md +27 -0
  67. package/docs/engines/npm.md +27 -0
  68. package/docs/engines/nvd.md +27 -0
  69. package/docs/engines/openreview.md +27 -0
  70. package/docs/engines/openstd.md +27 -0
  71. package/docs/engines/parallel_free.md +27 -0
  72. package/docs/engines/people_daily.md +27 -0
  73. package/docs/engines/realtime_index.md +27 -0
  74. package/docs/engines/sec_edgar.md +27 -0
  75. package/docs/engines/seltz.md +27 -0
  76. package/docs/engines/skillsmp.md +27 -0
  77. package/docs/engines/sspai.md +27 -0
  78. package/docs/engines/std_samr.md +27 -0
  79. package/docs/engines/train.md +27 -0
  80. package/docs/engines/un_comtrade.md +27 -0
  81. package/docs/engines/wikisource.md +27 -0
  82. package/docs/engines/zdic.md +27 -0
  83. package/docs/engines/zenodo.md +27 -0
  84. package/docs/engines/zhihu_hot_app.md +27 -0
  85. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  86. package/engines/README.md +10 -0
  87. package/engines/_template_cli.yaml +39 -0
  88. package/engines/_template_http.yaml +37 -0
  89. package/engines/plugins/_template_plugin.py +50 -0
  90. package/engines/specs/_cisa_kev.yaml +54 -0
  91. package/engines/specs/_osv.yaml +63 -0
  92. package/engines/specs/aljazeera.yaml +43 -0
  93. package/engines/specs/antara.yaml +43 -0
  94. package/engines/specs/artic.yaml +20 -0
  95. package/engines/specs/aviation_weather.yaml +39 -0
  96. package/engines/specs/bangumi.yaml +21 -0
  97. package/engines/specs/biorxiv.yaml +24 -0
  98. package/engines/specs/carbon_intensity.yaml +32 -0
  99. package/engines/specs/cfets.yaml +41 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/defillama.yaml +38 -0
  113. package/engines/specs/deps_dev.yaml +21 -0
  114. package/engines/specs/digup.yaml +36 -0
  115. package/engines/specs/doi.yaml +39 -0
  116. package/engines/specs/douban_movie.yaml +22 -0
  117. package/engines/specs/dw_news.yaml +47 -0
  118. package/engines/specs/egov_law.yaml +21 -0
  119. package/engines/specs/elpais.yaml +43 -0
  120. package/engines/specs/endoflife.yaml +21 -0
  121. package/engines/specs/energy_charts.yaml +21 -0
  122. package/engines/specs/factcheck_org.yaml +52 -0
  123. package/engines/specs/faz.yaml +43 -0
  124. package/engines/specs/federal_register.yaml +57 -0
  125. package/engines/specs/firecrawl.yaml +53 -0
  126. package/engines/specs/flk_law.yaml +24 -0
  127. package/engines/specs/folha.yaml +43 -0
  128. package/engines/specs/france24.yaml +44 -0
  129. package/engines/specs/full_fact.yaml +47 -0
  130. package/engines/specs/fxtwitter.yaml +36 -0
  131. package/engines/specs/gbfs_nyc.yaml +32 -0
  132. package/engines/specs/gdacs.yaml +21 -0
  133. package/engines/specs/gdelt.yaml +64 -0
  134. package/engines/specs/gleif.yaml +55 -0
  135. package/engines/specs/google_news.yaml +22 -0
  136. package/engines/specs/gov_regulations.yaml +38 -0
  137. package/engines/specs/guardian_rss.yaml +59 -0
  138. package/engines/specs/hkex.yaml +40 -0
  139. package/engines/specs/hurriyet.yaml +43 -0
  140. package/engines/specs/iplant.yaml +25 -0
  141. package/engines/specs/jikan.yaml +24 -0
  142. package/engines/specs/jolpica.yaml +20 -0
  143. package/engines/specs/k10plus.yaml +20 -0
  144. package/engines/specs/keenable.yaml +44 -0
  145. package/engines/specs/keenable_free.yaml +36 -0
  146. package/engines/specs/lefigaro.yaml +43 -0
  147. package/engines/specs/listenbrainz.yaml +19 -0
  148. package/engines/specs/matichon.yaml +43 -0
  149. package/engines/specs/met_museum.yaml +21 -0
  150. package/engines/specs/nasa_images.yaml +45 -0
  151. package/engines/specs/netease_music.yaml +38 -0
  152. package/engines/specs/nhk.yaml +43 -0
  153. package/engines/specs/nhtsa_vpic.yaml +20 -0
  154. package/engines/specs/noaa_swpc.yaml +20 -0
  155. package/engines/specs/nvd.yaml +38 -0
  156. package/engines/specs/obis.yaml +20 -0
  157. package/engines/specs/opencitations.yaml +52 -0
  158. package/engines/specs/openf1.yaml +21 -0
  159. package/engines/specs/openfoodfacts.yaml +45 -0
  160. package/engines/specs/openligadb.yaml +20 -0
  161. package/engines/specs/openreview.yaml +37 -0
  162. package/engines/specs/openstd.yaml +21 -0
  163. package/engines/specs/people_daily.yaml +22 -0
  164. package/engines/specs/realtime_index.yaml +34 -0
  165. package/engines/specs/ror.yaml +20 -0
  166. package/engines/specs/satnogs.yaml +20 -0
  167. package/engines/specs/sec_edgar.yaml +46 -0
  168. package/engines/specs/shfe.yaml +39 -0
  169. package/engines/specs/skillsmp.yaml +50 -0
  170. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  171. package/engines/specs/soilgrids.yaml +25 -0
  172. package/engines/specs/sspai.yaml +36 -0
  173. package/engines/specs/stackexchange.yaml +46 -0
  174. package/engines/specs/std_samr.yaml +21 -0
  175. package/engines/specs/tass.yaml +43 -0
  176. package/engines/specs/tavily_free.yaml +56 -0
  177. package/engines/specs/tle_mirror.yaml +21 -0
  178. package/engines/specs/train.yaml +37 -0
  179. package/engines/specs/tvmaze.yaml +20 -0
  180. package/engines/specs/twitter_syndication.yaml +31 -0
  181. package/engines/specs/un_comtrade.yaml +23 -0
  182. package/engines/specs/unpaywall.yaml +54 -0
  183. package/engines/specs/vnpress.yaml +43 -0
  184. package/engines/specs/weather.yaml +32 -0
  185. package/engines/specs/who_don.yaml +21 -0
  186. package/engines/specs/who_gho.yaml +20 -0
  187. package/engines/specs/wikimedia_commons.yaml +34 -0
  188. package/engines/specs/wikisource.yaml +22 -0
  189. package/engines/specs/worms.yaml +19 -0
  190. package/engines/specs/x_trending.yaml +34 -0
  191. package/engines/specs/yna.yaml +43 -0
  192. package/engines/specs/ynet.yaml +43 -0
  193. package/engines/specs/yuanbao.yaml +93 -0
  194. package/engines/specs/zbmath.yaml +40 -0
  195. package/engines/specs/zdic.yaml +22 -0
  196. package/engines/specs/zenodo.yaml +27 -0
  197. package/engines/specs/zhihu_hot_app.yaml +35 -0
  198. package/mcp/clients.yaml +106 -0
  199. package/package.json +55 -12
  200. package/packages/dsh-plugin/README.md +152 -0
  201. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  202. package/packages/dsh-plugin/dsh/client.js +1533 -0
  203. package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
  204. package/packages/dsh-plugin/dsh/index.js +527 -0
  205. package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
  206. package/packages/dsh-plugin/dsh/runner.js +666 -0
  207. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  208. package/packages/dsh-plugin/package.json +38 -0
  209. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  210. package/packages/dsh-plugin/test/client.test.js +406 -0
  211. package/packages/dsh-plugin/test/config.test.js +30 -0
  212. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  213. package/packages/dsh-plugin/test/index.test.js +1382 -0
  214. package/references/query-templates.md +96 -0
  215. package/references/query-templates.yaml +118 -0
  216. package/scripts/ab_eval.py +219 -0
  217. package/scripts/ab_eval_p0p1.py +266 -0
  218. package/scripts/adaptive.py +294 -70
  219. package/scripts/answer.py +154 -0
  220. package/scripts/archive_run.py +591 -0
  221. package/scripts/argo_engine_registry.py +172 -0
  222. package/scripts/argo_paths.py +728 -0
  223. package/scripts/article.py +136 -0
  224. package/scripts/article_meta.py +131 -0
  225. package/scripts/arxiv_source.py +201 -0
  226. package/scripts/badcase_ledger.py +252 -0
  227. package/scripts/batch_probe.py +241 -0
  228. package/scripts/benchmark.py +19 -0
  229. package/scripts/bounded_run.py +144 -0
  230. package/scripts/browser_auth.py +323 -0
  231. package/scripts/budget.py +247 -0
  232. package/scripts/budget_cli.py +93 -0
  233. package/scripts/build_release_assets.sh +65 -0
  234. package/scripts/burstiness_detector.py +147 -0
  235. package/scripts/cache.py +1066 -67
  236. package/scripts/cache_guard.py +271 -0
  237. package/scripts/cache_key_vdom.py +56 -0
  238. package/scripts/cache_similarity.py +170 -0
  239. package/scripts/caliber_isolation.py +195 -0
  240. package/scripts/calibrate_lowquality.py +210 -0
  241. package/scripts/candidate_envelope.py +366 -0
  242. package/scripts/chrome_cdp.py +871 -0
  243. package/scripts/ci_drift_gate.py +90 -0
  244. package/scripts/circuit_breaker.py +389 -0
  245. package/scripts/citations.py +334 -0
  246. package/scripts/cite_cli.py +59 -0
  247. package/scripts/cjk_tokens.py +95 -0
  248. package/scripts/clarify.py +47 -24
  249. package/scripts/cli_io.py +240 -0
  250. package/scripts/config.py +683 -45
  251. package/scripts/config_cache.py +402 -0
  252. package/scripts/conn_pool.py +220 -0
  253. package/scripts/content_boundary.py +46 -0
  254. package/scripts/content_security.py +685 -0
  255. package/scripts/content_signals.py +751 -0
  256. package/scripts/crawl.py +74 -35
  257. package/scripts/cross_lingual_trace.py +293 -0
  258. package/scripts/daemon.py +866 -0
  259. package/scripts/digup_bridge.py +214 -0
  260. package/scripts/dispatch_result.py +98 -0
  261. package/scripts/engine_admission.py +366 -0
  262. package/scripts/engine_dispatch.py +982 -0
  263. package/scripts/engine_env.py +539 -0
  264. package/scripts/engine_failure.py +411 -0
  265. package/scripts/engine_families.py +738 -0
  266. package/scripts/engine_policy.py +218 -0
  267. package/scripts/engine_requires.py +190 -0
  268. package/scripts/engine_status.py +322 -0
  269. package/scripts/engine_validate.py +638 -0
  270. package/scripts/engines.py +799 -294
  271. package/scripts/engines_base.py +1521 -0
  272. package/scripts/engines_builders.py +310 -0
  273. package/scripts/engines_builders_batch10.py +204 -0
  274. package/scripts/engines_builders_batch11.py +174 -0
  275. package/scripts/engines_builders_batch12.py +103 -0
  276. package/scripts/engines_builders_batch13.py +340 -0
  277. package/scripts/engines_builders_batch14.py +656 -0
  278. package/scripts/engines_builders_batch15.py +249 -0
  279. package/scripts/engines_builders_batch9.py +1468 -0
  280. package/scripts/engines_builders_cn.py +2283 -0
  281. package/scripts/engines_builders_data.py +2923 -0
  282. package/scripts/engines_builders_data_macro.py +775 -0
  283. package/scripts/engines_builders_feeds.py +255 -0
  284. package/scripts/engines_builders_intl.py +1239 -0
  285. package/scripts/engines_builders_keyless.py +111 -0
  286. package/scripts/engines_builders_search.py +440 -0
  287. package/scripts/engines_builders_tech.py +1359 -0
  288. package/scripts/evidence.py +620 -166
  289. package/scripts/evidence_loop.py +624 -0
  290. package/scripts/evidence_tier.py +298 -0
  291. package/scripts/except_sets.py +25 -0
  292. package/scripts/extract.py +55 -16
  293. package/scripts/fact_align.py +207 -0
  294. package/scripts/fact_crosscheck.py +230 -0
  295. package/scripts/fetch.py +25 -12
  296. package/scripts/fetch_browser.py +87 -0
  297. package/scripts/fetch_jsrun.py +122 -0
  298. package/scripts/fetch_quality.py +225 -0
  299. package/scripts/fetch_render_tinyfish.py +117 -0
  300. package/scripts/fetch_v3.py +1804 -0
  301. package/scripts/focus_extract.py +350 -0
  302. package/scripts/fulltext_store.py +169 -0
  303. package/scripts/fxtwitter_api.py +325 -0
  304. package/scripts/gen_engine_catalog.py +364 -0
  305. package/scripts/gen_native_tools.py +91 -0
  306. package/scripts/gen_routing_snapshot.py +195 -0
  307. package/scripts/health_check.py +140 -402
  308. package/scripts/health_probe.py +23 -8
  309. package/scripts/hot_state.py +134 -0
  310. package/scripts/http_client.py +1053 -0
  311. package/scripts/image_ops.py +430 -0
  312. package/scripts/image_vision/vision_probe.swift +126 -0
  313. package/scripts/install.ps1 +166 -0
  314. package/scripts/install.sh +221 -0
  315. package/scripts/job.py +1186 -0
  316. package/scripts/jump_resolver.py +235 -0
  317. package/scripts/lang_capability.py +150 -0
  318. package/scripts/lang_detect.py +289 -0
  319. package/scripts/lang_matrix_probe.py +329 -0
  320. package/scripts/lang_pref.py +311 -0
  321. package/scripts/link_source.py +340 -0
  322. package/scripts/local_image.py +970 -0
  323. package/scripts/local_seek.py +402 -0
  324. package/scripts/macro_countries.py +97 -0
  325. package/scripts/matrix_search_eval.py +1187 -0
  326. package/scripts/mcp_compact.py +234 -0
  327. package/scripts/mcp_diag.py +8 -3
  328. package/scripts/mcp_handlers.py +492 -0
  329. package/scripts/mcp_handlers_surface.py +157 -0
  330. package/scripts/mcp_launch.sh +124 -0
  331. package/scripts/mcp_local.py +241 -0
  332. package/scripts/mcp_server.py +125 -419
  333. package/scripts/mcp_setup.py +431 -0
  334. package/scripts/mcp_tools.py +331 -0
  335. package/scripts/mcp_transport.py +191 -0
  336. package/scripts/mcp_warm.py +127 -0
  337. package/scripts/min_version_scan.py +335 -0
  338. package/scripts/net_proxy.py +258 -0
  339. package/scripts/network_aware.py +152 -0
  340. package/scripts/output_transparency.py +200 -0
  341. package/scripts/paid_policy.py +98 -0
  342. package/scripts/paper.py +274 -0
  343. package/scripts/pdf_extract.py +518 -0
  344. package/scripts/phrase_filter.py +66 -0
  345. package/scripts/plan.py +422 -0
  346. package/scripts/postprocess_signals.py +150 -0
  347. package/scripts/pre-commit +48 -0
  348. package/scripts/query_enhance.py +157 -0
  349. package/scripts/query_operators.py +196 -0
  350. package/scripts/query_rewriter.py +116 -7
  351. package/scripts/query_signals.py +300 -0
  352. package/scripts/query_templates.py +167 -0
  353. package/scripts/query_understanding.py +411 -0
  354. package/scripts/query_variants.py +298 -0
  355. package/scripts/quota.py +633 -76
  356. package/scripts/rank_signals.py +775 -0
  357. package/scripts/ranking_eval.py +319 -0
  358. package/scripts/readability_extract.py +433 -0
  359. package/scripts/recompute.py +415 -0
  360. package/scripts/recovery.py +616 -0
  361. package/scripts/recovery_l0.py +160 -0
  362. package/scripts/recovery_runner.py +150 -0
  363. package/scripts/redskill/redskill_engine.py +332 -0
  364. package/scripts/regression_p0p1.py +318 -0
  365. package/scripts/replay_eval.py +658 -0
  366. package/scripts/rerank_tfidf.py +225 -0
  367. package/scripts/research.py +607 -403
  368. package/scripts/research_cli.py +361 -0
  369. package/scripts/research_dossier.py +492 -0
  370. package/scripts/research_expand.py +169 -0
  371. package/scripts/research_gates.py +272 -0
  372. package/scripts/research_report.py +209 -0
  373. package/scripts/research_strategy.py +37 -0
  374. package/scripts/research_work_packages.py +195 -0
  375. package/scripts/result_lang.py +311 -0
  376. package/scripts/retry_policy.py +76 -0
  377. package/scripts/robots_guard.py +259 -0
  378. package/scripts/route.py +921 -254
  379. package/scripts/route_bangs.py +40 -0
  380. package/scripts/route_cache.py +243 -0
  381. package/scripts/route_combo.py +917 -0
  382. package/scripts/route_domains.py +341 -0
  383. package/scripts/route_lang.py +605 -0
  384. package/scripts/route_log.py +63 -0
  385. package/scripts/route_policy.py +250 -0
  386. package/scripts/rrf_classic.py +66 -0
  387. package/scripts/screenshot.py +73 -0
  388. package/scripts/search.py +886 -406
  389. package/scripts/search_attribution.py +185 -0
  390. package/scripts/search_benchmark.py +407 -0
  391. package/scripts/search_cli.py +640 -0
  392. package/scripts/search_entry.py +395 -0
  393. package/scripts/search_output.py +690 -0
  394. package/scripts/search_pipeline.py +926 -0
  395. package/scripts/search_rank.py +991 -0
  396. package/scripts/search_types.py +129 -16
  397. package/scripts/seek_locator.py +68 -0
  398. package/scripts/semantic_evidence.py +391 -0
  399. package/scripts/serp_guard.py +336 -0
  400. package/scripts/serp_kill_log.py +74 -0
  401. package/scripts/single_flight.py +120 -0
  402. package/scripts/social_engines/__init__.py +10 -0
  403. package/scripts/social_engines/bilibili_engine.py +131 -0
  404. package/scripts/social_engines/hackernews_engine.py +113 -0
  405. package/scripts/social_engines/reddit_engine.py +172 -0
  406. package/scripts/social_engines/twitter_engine.py +397 -0
  407. package/scripts/social_engines/v2ex_engine.py +101 -0
  408. package/scripts/social_engines/weibo_engine.py +128 -0
  409. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  410. package/scripts/social_engines/zhihu_engine.py +139 -0
  411. package/scripts/social_research.py +201 -0
  412. package/scripts/sogou_weixin_browser.py +318 -0
  413. package/scripts/source_ledger.py +249 -0
  414. package/scripts/stage_timing.py +58 -0
  415. package/scripts/stats_cli.py +107 -0
  416. package/scripts/straggler_policy.py +60 -0
  417. package/scripts/stylometry_detector.py +231 -0
  418. package/scripts/sync_backends.py +353 -0
  419. package/scripts/sync_installs.py +29 -0
  420. package/scripts/template_cli.py +127 -0
  421. package/scripts/text_lexicon.py +72 -0
  422. package/scripts/tfidf_router.py +109 -37
  423. package/scripts/time_utils.py +233 -0
  424. package/scripts/topic_research_profiles.py +643 -0
  425. package/scripts/train.py +737 -0
  426. package/scripts/tweet.py +596 -0
  427. package/scripts/url_canon.py +217 -0
  428. package/scripts/url_optimize.py +56 -0
  429. package/scripts/url_safety.py +236 -0
  430. package/scripts/usage_log.py +116 -0
  431. package/scripts/usage_log_clean.py +113 -0
  432. package/scripts/v2ex_nodes.py +311 -0
  433. package/scripts/variant_recall.py +188 -0
  434. package/scripts/watch.py +205 -0
  435. package/scripts/wechat_article.py +258 -0
  436. package/scripts/wide_research.py +447 -0
  437. package/scripts/wx.py +349 -0
  438. package/scripts/x_article_fetch.py +141 -0
  439. package/scripts/x_search.py +234 -0
  440. package/scripts/x_trending.py +189 -0
  441. package/scripts/yaml_load.py +67 -0
  442. package/scripts/zhihu_quota.py +139 -0
  443. package/sub-skills/ego-search/SKILL.md +253 -0
  444. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  445. package/sub-skills/ego-search/scripts/merge.py +226 -0
  446. package/sub-skills/ego-search/scripts/quality.py +64 -0
  447. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  448. package/sub-skills/ego-search/scripts/safety.py +109 -0
  449. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  450. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  451. package/sub-skills/js-run/SKILL.md +67 -0
  452. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  453. package/sub-skills/js-run/scripts/shim.js +188 -0
  454. package/sub-skills/local-search/SKILL.md +50 -53
  455. package/sub-skills/local-search/bm25.py +186 -0
  456. package/sub-skills/local-search/cli_runner.py +331 -0
  457. package/sub-skills/local-search/config.yaml +152 -55
  458. package/sub-skills/local-search/engine_registry.py +91 -26
  459. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  460. package/sub-skills/local-search/local_search_adapter.py +13 -2
  461. package/sub-skills/local-search/parse_maps.yaml +40 -42
  462. package/sub-skills/local-search/parsers.py +328 -0
  463. package/sub-skills/local-search/search_v3.py +716 -242
  464. package/sub-skills/local-search/smart_router.py +47 -7
  465. package/sub-skills/local-seek/SKILL.md +110 -0
  466. package/sub-skills/local-seek/config/domains.yaml +20 -0
  467. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  468. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  469. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  470. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  471. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  472. package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
@@ -1,25 +1,118 @@
1
1
  #!/usr/bin/env python3
2
- """
3
- search_types.py — 统一类型系统
4
-
5
- 定义 SearchResult 数据类和 normalize_result 统一转换,
6
- 将所有引擎的原始输出统一为结构化格式。
7
-
8
- 字段规范:
9
- - title: 结果标题(必填,截断到 200 字符)
10
- - url: 结果链接(可选)
11
- - snippet: 摘要/片段(截断到 300 字符)
12
- - score: 相关性评分(默认 0.5)
13
- - source: 引擎来源
14
- - metadata: 元数据字典
2
+ """search_types.py — 统一类型系统(轻量记录 + 零依赖 dataclass 替代)。
3
+
4
+ `slot_record` / `slot_field` / `slot_replace` 是为热路径准备的
5
+ `@dataclass` / `field` / `replace` 零依赖替代:`dataclasses` 的导入链是
6
+ `dataclasses → inspect → tokenize → dis`(实测 ~8ms),而 `SearchResult`
7
+ 这类记录在每次搜索都要构造,导入成本全落在固定开销上。
8
+
9
+ 兼容面:挂 `__dataclass_fields__`,标准库的 `dataclasses.replace` /
10
+ `is_dataclass` / `fields` 对实例照常工作(`_field_type` 惰性解析)。
11
+
12
+ 仍不建议新代码接入本模块的 `SearchResult`(引擎输出直接透传 dict,运行时
13
+ 防御在 engines_base/safe_search 里);这里的价值在**替代实现本身**。
15
14
  """
16
15
  from __future__ import annotations
17
16
 
18
- from dataclasses import dataclass, field, asdict
17
+ import sys
18
+ from enum import Enum
19
19
  from typing import Any
20
20
 
21
21
 
22
- @dataclass
22
+ class _FieldMeta:
23
+ """`__dataclass_fields__` 的兼容描述子。
24
+
25
+ `dataclasses.replace` 只读 `.init`;`dataclasses.fields` 会按
26
+ `_field_type is dataclasses._FIELD` 过滤,所以 `_field_type` 做成**惰性
27
+ 属性**:dataclasses 已被 import 时(也就是有人真去调 `fields()` 时)
28
+ 返回真哨兵,没 import 时返回 None。热路径因此仍然零导入——`_FIELD`
29
+ 只在 dataclasses 已经进场后才去取它。
30
+ """
31
+
32
+ __slots__ = ("name", "init")
33
+
34
+ def __init__(self, name: str) -> None:
35
+ self.name = name
36
+ self.init = True
37
+
38
+ @property
39
+ def _field_type(self) -> Any:
40
+ dc = sys.modules.get("dataclasses")
41
+ return getattr(dc, "_FIELD", None) if dc is not None else None
42
+
43
+
44
+ class _SlotField:
45
+ __slots__ = ("factory",)
46
+
47
+ def __init__(self, default_factory: Any) -> None:
48
+ self.factory = default_factory
49
+
50
+
51
+ def slot_field(*, default_factory: Any) -> Any:
52
+ """零依赖 field(default_factory=...) 替代。"""
53
+ return _SlotField(default_factory)
54
+
55
+
56
+ def _class_annotations(cls: Any) -> dict:
57
+ """读类的注解,兼容 PEP 649 惰性注解。
58
+
59
+ `cls.__dict__["__annotations__"]` 在 3.14 的惰性注解下**可能根本不存在**
60
+ (注解由 `__annotate__` 描述子按需计算,不进类字典)——此时取到的是
61
+ `{}`,类会静默变成「零字段」,实例化时报一个与真因毫不相干的
62
+ TypeError。必须走 `getattr` 触发描述子。
63
+ """
64
+ return getattr(cls, "__annotations__", None) or {}
65
+
66
+
67
+ def slot_record(_cls: Any = None, *, frozen: bool = False) -> Any:
68
+ """零依赖 @dataclass 替代(省去 import dataclasses→inspect→tokenize→dis ~8ms)。
69
+
70
+ 挂上 `__dataclass_fields__` 兼容描述子,标准库 `dataclasses.replace` /
71
+ `dataclasses.is_dataclass` / `dataclasses.fields` 对实例照常工作,但热
72
+ 路径自身零导入(`_field_type` 惰性解析,只在 dataclasses 已进场时取哨兵)。
73
+ """
74
+ def _wrap(cls: Any) -> Any:
75
+ ann = list(_class_annotations(cls).keys())
76
+ defaults = {k: cls.__dict__[k] for k in ann if k in cls.__dict__}
77
+
78
+ def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
79
+ if len(args) > len(ann):
80
+ raise TypeError(f"{cls.__name__}.__init__() takes {len(ann)} positional arguments but {len(args)} were given")
81
+ for i, val in enumerate(args):
82
+ object.__setattr__(self, ann[i], val)
83
+ for k in ann[len(args):]:
84
+ if k in kwargs:
85
+ val = kwargs.pop(k)
86
+ elif k in defaults:
87
+ d = defaults[k]
88
+ val = d.factory() if isinstance(d, _SlotField) else d
89
+ else:
90
+ raise TypeError(f"{cls.__name__}.__init__() missing required argument: {k!r}")
91
+ object.__setattr__(self, k, val)
92
+ if kwargs:
93
+ bad = next(iter(kwargs))
94
+ raise TypeError(f"{cls.__name__}.__init__() got an unexpected keyword argument {bad!r}")
95
+
96
+ cls._slot_fields = tuple(ann)
97
+ cls.__dataclass_fields__ = {k: _FieldMeta(k) for k in ann}
98
+ cls.__init__ = __init__
99
+ if frozen:
100
+ def __setattr__(self: Any, name: str, value: Any) -> None:
101
+ raise AttributeError(f"cannot assign to field {name!r}")
102
+ cls.__setattr__ = __setattr__
103
+ return cls
104
+
105
+ return _wrap if _cls is None else _wrap(_cls)
106
+
107
+
108
+ def slot_replace(obj: Any, **changes: Any) -> Any:
109
+ """零依赖 replace(obj, **changes) 替代。"""
110
+ kw = {k: getattr(obj, k) for k in obj._slot_fields}
111
+ kw.update(changes)
112
+ return type(obj)(**kw)
113
+
114
+
115
+ @slot_record
23
116
  class SearchResult:
24
117
  """统一的搜索结果数据类。"""
25
118
  title: str
@@ -27,7 +120,8 @@ class SearchResult:
27
120
  snippet: str = ""
28
121
  score: float = 0.5
29
122
  source: str = ""
30
- metadata: dict = field(default_factory=dict)
123
+ metadata: dict = slot_field(default_factory=dict)
124
+ social_meta: dict | None = None
31
125
 
32
126
  def to_dict(self) -> dict[str, Any]:
33
127
  """转为字典,省略空值/默认值字段。"""
@@ -42,6 +136,8 @@ class SearchResult:
42
136
  d["source"] = self.source
43
137
  if self.metadata:
44
138
  d["metadata"] = self.metadata
139
+ if self.social_meta:
140
+ d["social_meta"] = self.social_meta
45
141
  return d
46
142
 
47
143
 
@@ -69,4 +165,21 @@ def normalize_result(item: dict, engine: str, default_score: float = 0.5) -> Sea
69
165
  score=score,
70
166
  source=item.get("source", engine),
71
167
  metadata=item.get("metadata", {}),
168
+ social_meta=item.get("social_meta"),
72
169
  )
170
+
171
+
172
+ class Stage(str, Enum):
173
+ """进度回调的阶段名(on_progress 契约)。
174
+
175
+ 从 search.py 搬来这里:执行层与加工层(search_pipeline)都要发进度事件,
176
+ 枚举放在任何一侧都会让另一侧反向依赖;search_types 是本仓共享类型的家。
177
+ """
178
+
179
+ START = "start"
180
+ CACHE_HIT = "cache_hit"
181
+ ROUTING = "routing"
182
+ SEARCHING = "searching"
183
+ MERGING = "merging"
184
+ DONE = "done"
185
+ ERROR = "error"
@@ -0,0 +1,68 @@
1
+ #!/usr/bin/env python3
2
+ """seek_locator.py — local-seek 子技能的「安装感知」定位(唯一来源,不写死主机路径)。
3
+
4
+ 本地搜索(argo_local_search / --include-local)需要 local-seek/scripts/seek.py。
5
+ local-seek 随 argo 打包在 sub-skills/ 下(安装感知:寻址锚点是本文件的真实安装位置),
6
+ 也可能作为独立 skill 装在别处。此模块把发现逻辑收紧到一处,供 search.py 与
7
+ mcp_handlers.py 共用,消除两处各自硬编码 __file__ 相对路径与 ~/.agents/skills 等
8
+ 主机路径的重复(Fowler: Duplicated Code 反面却是各自为政、易漂移)。
9
+
10
+ 保持一致 SKILL.md 纪律「禁止在产品代码写死主机 skill 路径」:不再出现
11
+ ~/.agents/skills / ~/.claude/skills 这类字面量,改由环境变量承载自定义/遗留位置。
12
+
13
+ 查找顺序(命中即返回,全部未命中则返回打包位置,由调用方据此 fail 提示):
14
+ 1. ARGO_LOCAL_SEEK_PATH 显式指定 seek.py 的绝对/相对路径(任意自定义位置)
15
+ 2. <ARGO_ROOT>/sub-skills/local-seek/scripts/seek.py
16
+ —— 打包子技能,锚点为本模块的安装位置(symlink 会解析回来源)
17
+ 3. ARGO_LOCAL_SEEK_ROOTS os.pathsep 分隔的候选根目录,逐个检查
18
+ <根>/local-seek/scripts/seek.py(承载遗留/独立装法)
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import os
24
+ from typing import Iterator
25
+
26
+ ARGO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
27
+ _BUNDLED = os.path.join(ARGO_ROOT, "sub-skills", "local-seek", "scripts", "seek.py")
28
+
29
+
30
+ def _candidates() -> Iterator[str]:
31
+ """按优先级产出候选 seek.py 路径(仅路径,未校验存在性)。
32
+
33
+ 优先级:显式文件覆盖 > 显式候选根 > 打包位置(默认)。
34
+ 显式配置(ARGO_LOCAL_SEEK_PATH / ARGO_LOCAL_SEEK_ROOTS)代表用户点名了
35
+ 装在哪,理应先于总是存在的打包子技能;打包位置只作保底。
36
+ """
37
+ override = (os.environ.get("ARGO_LOCAL_SEEK_PATH") or "").strip()
38
+ if override:
39
+ yield os.path.realpath(os.path.expanduser(override))
40
+ roots = (os.environ.get("ARGO_LOCAL_SEEK_ROOTS") or "").strip()
41
+ if roots:
42
+ for root in roots.split(os.pathsep):
43
+ root = root.strip()
44
+ if not root:
45
+ continue
46
+ base = os.path.realpath(os.path.expanduser(root))
47
+ candidate = os.path.join(base, "local-seek", "scripts", "seek.py")
48
+ if os.path.isfile(candidate):
49
+ yield candidate
50
+ yield _BUNDLED
51
+
52
+
53
+ def resolve_seek_py() -> str:
54
+ """返回 local-seek/scripts/seek.py 的绝对路径(命中即返回,未命中返回打包位置)。"""
55
+ for cand in _candidates():
56
+ if cand and os.path.isfile(cand):
57
+ return cand
58
+ return _BUNDLED
59
+
60
+
61
+ def seek_py_exists() -> bool:
62
+ """本地搜索能力是否可用(seek.py 实际存在)。"""
63
+ return os.path.isfile(resolve_seek_py())
64
+
65
+
66
+ if __name__ == "__main__": # pragma: no cover
67
+ import sys
68
+ print(resolve_seek_py())
@@ -0,0 +1,391 @@
1
+ #!/usr/bin/env python3
2
+ """semantic_evidence.py — 可选的语义证据层(classifier.dev,默认关闭)。
3
+
4
+ ## 为什么存在
5
+
6
+ argo 的正文吸收分是**正则密度**(has_numbers / has_definition / …),它测的是
7
+ 「这段文本长得像不像证据块」,不测「它是否支持这条查询的主张」。2026-09-19 的
8
+ 实测(reports/classifier-dev-argo/A、B):密度最高页判 neutral、密度最低页判
9
+ supports 0.99,两者近乎反向;而假前提主张(查询预设「降息」、现实是「加息」)
10
+ 密度分在结构上无法表达,语义判定 5/5 判 contradicts(conf 0.95~1.00)。
11
+
12
+ ## 定位(与热路径解耦)
13
+
14
+ - **只在显式核验链路调用**(`search --verify` / `research --verify`),不进搜索
15
+ 热路径:热命中 78~82ms 且网络为零,一次分类调用 200~680ms,放进热路径是净损失
16
+ - **默认关闭**:config.yaml `semantic_evidence.enabled: false`;显式开启用环境变量
17
+ `ARGO_SEMANTIC_EVIDENCE=1`(env 优先于 config)
18
+ - **fail-open**:超时 / HTTP 错误 / 429 / 解析失败 / 条数不匹配 → 返回 None,
19
+ 调用方行为与未开启时逐位一致(不加字段、不改排序、不改公式)
20
+ - **只加并列字段**:`semantic_support` 挂在结果与 verified 条目上,可信度公式
21
+ (selection 0.40 / absorption 0.35 / freshness 0.15 / original 0.10)不动
22
+
23
+ ## 外部服务约束(2026-09-19 实测,见 reports/classifier-dev-argo/D)
24
+
25
+ - 无密钥、HTTP 通道稳定(1000 条 1.83s、顺序严格保真);CLI 与 MCP 通道当前不可用
26
+ (npm/PyPI 404、MCP tools/call 返回 not_found),因此只走 HTTP
27
+ - 必须带真实 User-Agent(python urllib 默认 UA 被边缘 403)——复用 HttpClient 的
28
+ UA 轮换与 429 退避
29
+ - 标签集必须显式含「none of these」:服务永远返回给定标签之一,不加会把无关文本
30
+ 硬塞进某个标签
31
+ - 同输入置信度有 ±0.08 漂移(标签稳定),所以只做 ≥0.7 的粗门控;实测 conf ≥0.7
32
+ 时严格错误为 0(51 条 entailment 样本)
33
+ - 限额按出口 IP:fast 3000 次/分钟、20000 次/天;空串输入会让整批 400,调用前过滤
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import json
39
+ import logging
40
+ from typing import Any, Optional
41
+
42
+ logger = logging.getLogger("unified_search.semantic_evidence")
43
+
44
+ DEFAULT_ENDPOINT = "https://classifier.dev"
45
+ DEFAULT_TIMEOUT_S = 5.0
46
+ # 未声明 transport 时的默认通道。jev = TypeSafe System One(当前唯一可用);
47
+ # classifier.dev 的匿名路径 2026-10-22 起回 403 proxy_requires_payment。
48
+ DEFAULT_TRANSPORT = "jev"
49
+
50
+ # ── JEV / System One 通道(TypeSafe)──────────────────────────────────────
51
+ # 官方 API:POST https://api.typesafe.ai/v1/systemone,Bearer 鉴权
52
+ # (docs.typesafe.ai/api/system-one)。与 classifier.dev 的 /v1/classify 是
53
+ # **两种形状**:那边一次发 N 条文本 + 一组标签;这边一次发一个 state + 若干
54
+ # 具名问题,每题独立作答(文档原话:questions are evaluated in parallel and
55
+ # in isolation)。所以接过来要做的是形状转换,不是换个 URL。
56
+ DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1/systemone"
57
+ DEFAULT_JEV_MODEL = "jev-latest"
58
+ # key 候选链:argo 推荐名在前;TYPESAFE_API_KEY 是官方 SDK 与 JEV 插件
59
+ # (dsh-verbatim-compaction)用的名字,照文档配的人直接可用。
60
+ JEV_KEY_ENV = ("ARGO_JEV_API_KEY", "TYPESAFE_API_KEY")
61
+ # state 超过这个长度就带长上下文头。实测(2026-10-22 本机):
62
+ # 不带 TypeSafe-Danger-Zone-Long-Context:32K tokens 通过(32350)、
63
+ # 40K 中文字(≈40K tokens)回 `max_tokens_exceeded`(400)
64
+ # 带上:40K 字(40358 tokens)与 80K 字(80358 tokens)均通过
65
+ # 24000 字是保守拐点(正常路径不带头也稳),不是实测上限。
66
+ JEV_LONG_CONTEXT_CHARS = 24000
67
+ # 并发上限。官方 cookbook(classifying_rag_passages)取 4,理由是端点限流;
68
+ # 429/529 要求指数退避,所以不铺大。
69
+ JEV_MAX_WORKERS = 4
70
+
71
+ # conf ≥ 0.7 才给结论:低于此值的判定不可信(B 实测 4 个严格错误全部 ≤0.67)
72
+ SUPPORT_THRESHOLD = 0.7
73
+ # 单条文本上限:核验链路取回的正文 ≤8000 字,再高无收益(A 实测 400→7798 字
74
+ # 置信度反而从 0.73 升到 0.98,说明长文不劣化,但也没必要超发)
75
+ MAX_TEXT_CHARS = 8000
76
+
77
+ SUPPORT_LABELS = (
78
+ "supports the query",
79
+ "contradicts the query",
80
+ "neutral",
81
+ "none of these",
82
+ )
83
+ SOURCE_TYPE_LABELS = (
84
+ "primary data or official announcement",
85
+ "secondary reporting",
86
+ "analysis or opinion",
87
+ "social narrative",
88
+ "none of these",
89
+ )
90
+
91
+
92
+ def _config() -> dict[str, Any]:
93
+ """读 config.yaml 的 semantic_evidence 段(缺失时用内置默认)。"""
94
+ try:
95
+ from config import load_config
96
+ cfg = load_config().get("semantic_evidence")
97
+ return cfg if isinstance(cfg, dict) else {}
98
+ except Exception: # pragma: no cover - 配置不可用按默认关处理
99
+ return {}
100
+
101
+
102
+ def enabled() -> bool:
103
+ """开关判定:环境变量优先,其次 config.yaml;两者都缺省 → 关闭。
104
+
105
+ 用 engine_env.get_env/env_flag 读(全仓唯一的布尔解析),所以
106
+ `ARGO_SEMANTIC_EVIDENCE=off`、写进 ~/.config/argo/env 的写法都生效。
107
+ """
108
+ try:
109
+ from engine_env import env_flag, get_env
110
+ if get_env("ARGO_SEMANTIC_EVIDENCE").strip():
111
+ return env_flag("ARGO_SEMANTIC_EVIDENCE", default=False)
112
+ except Exception: # pragma: no cover - 环境层不可用时不冒险开启
113
+ return False
114
+ return bool(_config().get("enabled", False))
115
+
116
+
117
+ def _endpoint() -> str:
118
+ return str(_config().get("endpoint") or DEFAULT_ENDPOINT).rstrip("/")
119
+
120
+
121
+ def _timeout() -> float:
122
+ try:
123
+ return float(_config().get("timeout_s") or DEFAULT_TIMEOUT_S)
124
+ except (TypeError, ValueError):
125
+ return DEFAULT_TIMEOUT_S
126
+
127
+
128
+ def jev_api_key() -> str:
129
+ """JEV(TypeSafe)key:os.environ 优先,~/.config/argo/env 热读保底。"""
130
+ try:
131
+ from engine_env import get_env
132
+ return (get_env(list(JEV_KEY_ENV)) or "").strip()
133
+ except Exception: # pragma: no cover - 环境层不可用时不冒险
134
+ import os
135
+ for _n in JEV_KEY_ENV:
136
+ _v = (os.environ.get(_n) or "").strip()
137
+ if _v:
138
+ return _v
139
+ return ""
140
+
141
+
142
+ def _transport() -> str:
143
+ """通道选择:`jev`(TypeSafe System One)或 `classify`(classifier.dev)。
144
+
145
+ **纯配置驱动,不做「按键就绪自动选」**。初版按 jev_api_key() 是否就绪自动
146
+ 选,结果是通道取决于本机 ~/.config/argo/env 里有没有那把 key——同一份代码
147
+ 在开发机与 CI 上走不同分支,既有测试在开发机上会静默改道(实测:写进 key
148
+ 后 test_semantic_evidence 的 5 条 HttpClient 桩里 1 条改走 JEV 路径而挂)。
149
+ 机器状态不该改变代码路径。
150
+
151
+ 未声明时的默认是 jev:classifier.dev 的匿名路径 2026-10-22 起回 403
152
+ proxy_requires_payment,JEV 是当前可用的那条。
153
+ """
154
+ declared = str(_config().get("transport") or "").strip().lower()
155
+ return declared if declared in ("jev", "classify") else DEFAULT_TRANSPORT
156
+
157
+
158
+ def _jev_endpoint() -> str:
159
+ """JEV 端点:**独立于 classify 通道的 endpoint 字段**。
160
+
161
+ 不共用是刻意的坑修:config.yaml 里 `endpoint` 长期写的是
162
+ https://classifier.dev,JEV 通道若读同一个字段,就会把 System One 的
163
+ 请求体发到 classifier.dev,拿回「Invalid API key / Create a workspace
164
+ key at /app/keys」这种指向完全错误方向的 401(2026-10-22 实测踩中)。
165
+ 每个通道自带端点,杜绝「换 transport 忘了换 endpoint」。
166
+ """
167
+ return str(_config().get("jev_endpoint") or DEFAULT_JEV_ENDPOINT).rstrip("/")
168
+
169
+
170
+ def _jev_model() -> str:
171
+ return str(_config().get("jev_model") or _config().get("model")
172
+ or DEFAULT_JEV_MODEL)
173
+
174
+
175
+ def _jev_post(path_state: Any, questions: dict[str, Any], *,
176
+ timeout: float) -> Optional[dict[str, Any]]:
177
+ """一次 System One 请求;失败返回 None(fail-open 的全部出口都在这里)。"""
178
+ key = jev_api_key()
179
+ if not key:
180
+ logger.debug("JEV key 未配置(%s),语义层跳过", " / ".join(JEV_KEY_ENV))
181
+ return None
182
+ body = {
183
+ "model": _jev_model(),
184
+ "state": path_state,
185
+ "questions": questions,
186
+ }
187
+ # Content-Type 必须显式给:HttpClient.post 会把 dict 做 json.dumps,
188
+ # 但**从不设 Content-Type**(头只来自浏览器 UA 模板 + extra_headers)。
189
+ # 缺它时 FastAPI 侧把 body 当字符串,回
190
+ # 422 {"loc":["body"],"msg":"Input should be a valid dictionary…"}
191
+ # ——报错指向「请求体不是对象」,而真因是少了一个头(2026-10-22 实测踩中)。
192
+ headers = {"Authorization": f"Bearer {key}",
193
+ "Content-Type": "application/json"}
194
+ # 长上下文要显式带头,否则 state 一过 32K tokens 就 400 max_tokens_exceeded
195
+ if len(json.dumps(path_state, ensure_ascii=False)) > JEV_LONG_CONTEXT_CHARS:
196
+ headers["TypeSafe-Danger-Zone-Long-Context"] = "true"
197
+ try:
198
+ from http_client import HttpClient
199
+ resp = HttpClient(timeout=timeout, max_retries=1, jitter=False).post(
200
+ _jev_endpoint(), body, extra_headers=headers)
201
+ if resp.get("status") != 200:
202
+ logger.debug("JEV 非 200: %s %s", resp.get("status"),
203
+ str(resp.get("text") or resp.get("error") or "")[:200])
204
+ return None
205
+ return json.loads(resp.get("text") or "{}")
206
+ except Exception as e: # fail-open:语义层任何问题都不阻断核验
207
+ logger.debug("JEV 请求失败: %s", type(e).__name__)
208
+ return None
209
+
210
+
211
+ def _jev_choice(text: str, labels: tuple[str, ...] | list[str],
212
+ instructions: str | None, timeout: float
213
+ ) -> Optional[dict[str, Any]]:
214
+ """单条文本 → 一个 Choice 问题。
215
+
216
+ **每条一个请求**,不是「多段一个 state、每段一问」。理由在官方文档里:
217
+ jaggedness 那篇明写「state 里塞与决策无关的细节会降低准确率(context
218
+ rot)」,而判第 i 段时其余各段就是无关细节;cookbook 的
219
+ classifying_rag_passages 也是「One request per passage」。(我们试过 3 段
220
+ 合一,结果全对,但样本太小,不拿它赌准确率。)
221
+ """
222
+ out = _jev_post(text, {
223
+ "cls": {
224
+ "type": "choice",
225
+ "instructions": instructions or "Classify the state into one of the criteria.",
226
+ # criteria 的**值**是描述:jaggedness 说 instructions 与 criteria
227
+ # 不一致时模型会困惑,所以两边用同一句话,不给它两个版本。
228
+ "criteria": {str(lab): str(lab) for lab in labels},
229
+ },
230
+ }, timeout=timeout)
231
+ if not isinstance(out, dict):
232
+ return None
233
+ ans = (out.get("answers") or {}).get("cls")
234
+ if not isinstance(ans, dict) or not ans.get("choice"):
235
+ if isinstance(ans, dict):
236
+ logger.debug("JEV 答案缺 choice: %s", str(ans)[:120])
237
+ return None
238
+ return {
239
+ "label": str(ans.get("choice")),
240
+ "confidence": ans.get("confidence"),
241
+ "scores": ans.get("probabilities"),
242
+ }
243
+
244
+
245
+ def _classify_jev(texts: list[str], labels: tuple[str, ...] | list[str], *,
246
+ instructions: str | None, keep: list[tuple[int, str]],
247
+ timeout: float) -> Optional[list[dict[str, Any]]]:
248
+ """JEV 通道:每条文本一个请求,并发 4。
249
+
250
+ 任一条失败就在该位留 None(与 classifier.dev 通道「整批失败即 None」
251
+ 不同):一次核验有几条正文,坏一条不该把其余判定的价值也丢掉。全部失败
252
+ 时仍返回 None,调用方行为与未开启时一致。
253
+ """
254
+ from concurrent.futures import ThreadPoolExecutor
255
+ texts_kept = [t[:MAX_TEXT_CHARS] for _i, t in keep]
256
+ with ThreadPoolExecutor(max_workers=min(JEV_MAX_WORKERS, len(texts_kept))) as pool:
257
+ results = list(pool.map(
258
+ lambda t: _jev_choice(t, labels, instructions, timeout), texts_kept))
259
+ if not any(r for r in results):
260
+ return None
261
+ out: list[Optional[dict[str, Any]]] = [None] * len(texts)
262
+ for (idx, _t), r in zip(keep, results):
263
+ if isinstance(r, dict):
264
+ out[idx] = r
265
+ return out # type: ignore[return-value]
266
+
267
+
268
+ def classify(texts: list[str], labels: tuple[str, ...] | list[str], *,
269
+ instructions: str | None = None,
270
+ timeout: float | None = None,
271
+ tier: str | None = None) -> Optional[list[dict[str, Any]]]:
272
+ """一次批量分类。任何异常/非 200/条数不匹配 → None。
273
+
274
+ 返回按输入顺序排列的 [{label, confidence, scores}];调用方负责判定阈值。
275
+ 空文本会被过滤后再发(空串会让整批 400),返回列表与输入等长、空缺位为 None。
276
+
277
+ 两个通道的实现分居 _classify_jev(TypeSafe,每条一请求)与下方内联的
278
+ classifier.dev 路径(一次发 N 条);开关判定见 _transport。
279
+ """
280
+ if not texts or len(labels) < 2:
281
+ return None
282
+ keep = [(i, t) for i, t in enumerate(texts) if (t or "").strip()]
283
+ if not keep:
284
+ return None
285
+ to = timeout or _timeout()
286
+ if _transport() == "jev":
287
+ return _classify_jev(texts, labels, instructions=instructions,
288
+ keep=keep, timeout=to)
289
+ body: dict[str, Any] = {
290
+ "labels": [str(x) for x in labels],
291
+ "inputs": [t[:MAX_TEXT_CHARS] for _i, t in keep],
292
+ "tier": tier or str(_config().get("tier") or "fast"),
293
+ }
294
+ if instructions:
295
+ body["instructions"] = instructions
296
+ try:
297
+ from http_client import HttpClient
298
+ # Content-Type 同 JEV 通道:HttpClient.post 不设它,缺了会被服务端
299
+ # 当字符串解析。这条路径此前只在 403(匿名要付费 key)处止步,没走到
300
+ # body 校验,所以同一个坑没暴露——配了有效 workspace key 就会现形。
301
+ resp = HttpClient(timeout=to, max_retries=1, jitter=False).post(
302
+ _endpoint(), body,
303
+ extra_headers={"Content-Type": "application/json"})
304
+ if resp.get("status") != 200:
305
+ logger.debug(f"语义分类非 200: {resp.get('status')} {resp.get('error')}")
306
+ return None
307
+ data = json.loads(resp.get("text") or "{}")
308
+ results = data.get("results")
309
+ if not isinstance(results, list) or len(results) != len(keep):
310
+ logger.debug("语义分类返回条数与输入不匹配,按失败处理")
311
+ return None
312
+ except Exception as e: # fail-open:语义层任何问题都不阻断核验
313
+ logger.debug(f"语义分类失败: {type(e).__name__}")
314
+ return None
315
+ out: list[Optional[dict[str, Any]]] = [None] * len(texts)
316
+ for (idx, _t), r in zip(keep, results):
317
+ if isinstance(r, dict):
318
+ out[idx] = r
319
+ return out # type: ignore[return-value]
320
+
321
+
322
+ def assess_support(query: str, items: list[dict[str, Any]]) -> Optional[dict[str, dict]]:
323
+ """对已抓取的正文批量判定「是否支持查询隐含的主张」。
324
+
325
+ items: [{"url": str, "title": str, "text": str}]
326
+ 返回 {url: {label, confidence, supports, contradicts}};未达阈值时
327
+ supports/contradicts 均为 False(原始 label 与 confidence 仍原样保留,
328
+ 调用方可自行换阈值)。失败返回 None。
329
+ """
330
+ usable = [it for it in items if (it.get("text") or "").strip() and it.get("url")]
331
+ if not usable:
332
+ return None
333
+ texts = [f"{it.get('title') or ''}\n{(it.get('text') or '')[:MAX_TEXT_CHARS]}"
334
+ for it in usable]
335
+ instructions = (
336
+ f"Judge whether each passage supports the claim implied by this query: {query!r}. "
337
+ "Answer 'supports the query' when the passage backs the claim, "
338
+ "'contradicts the query' when it states the opposite or a conflicting figure, "
339
+ "'neutral' when it is on-topic but neither supports nor contradicts, "
340
+ "and 'none of these' when it is unrelated to the query."
341
+ )
342
+ results = classify(texts, SUPPORT_LABELS, instructions=instructions)
343
+ if results is None:
344
+ return None
345
+ out: dict[str, dict] = {}
346
+ for it, r in zip(usable, results):
347
+ if not r:
348
+ continue
349
+ label = str(r.get("label") or "")
350
+ conf = r.get("confidence")
351
+ conf_f = float(conf) if isinstance(conf, (int, float)) else 0.0
352
+ out[str(it["url"])] = {
353
+ "label": label,
354
+ "confidence": conf_f,
355
+ "supports": label == "supports the query" and conf_f >= SUPPORT_THRESHOLD,
356
+ "contradicts": label == "contradicts the query" and conf_f >= SUPPORT_THRESHOLD,
357
+ }
358
+ return out or None
359
+
360
+
361
+ def assess_source_types(items: list[dict[str, Any]]) -> Optional[dict[str, dict]]:
362
+ """来源类型分级(研究取证链路用):一手数据 / 二手报道 / 分析观点 / 社交叙事。
363
+
364
+ items: [{"url": str, "title": str, "text": str}](text 可为摘要)
365
+ """
366
+ usable = [it for it in items if it.get("url") and
367
+ ((it.get("text") or "").strip() or (it.get("title") or "").strip())]
368
+ if not usable:
369
+ return None
370
+ texts = [f"{it.get('title') or ''}\n{(it.get('text') or '')[:MAX_TEXT_CHARS]}"
371
+ for it in usable]
372
+ instructions = (
373
+ "Classify each source by its nature: 'primary data or official announcement' "
374
+ "for original datasets, filings, official releases; 'secondary reporting' for "
375
+ "news coverage of someone else's work; 'analysis or opinion' for commentary; "
376
+ "'social narrative' for forum/social posts."
377
+ )
378
+ results = classify(texts, SOURCE_TYPE_LABELS, instructions=instructions)
379
+ if results is None:
380
+ return None
381
+ out: dict[str, dict] = {}
382
+ for it, r in zip(usable, results):
383
+ if not r:
384
+ continue
385
+ label = str(r.get("label") or "")
386
+ conf = r.get("confidence")
387
+ out[str(it["url"])] = {
388
+ "label": label,
389
+ "confidence": float(conf) if isinstance(conf, (int, float)) else 0.0,
390
+ }
391
+ return out or None