argo-search 2.9.7 → 2.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/README.en.md +73 -150
  2. package/README.es.md +73 -150
  3. package/README.ja.md +79 -156
  4. package/README.ko.md +78 -155
  5. package/README.md +80 -159
  6. package/SKILL.md +3 -3
  7. package/assets/readme/hero.svg +2 -2
  8. package/backends/domain_profiles.json +48 -0
  9. package/backends/engine_registry.yaml +118 -4
  10. package/backends/quota_profiles.json +92 -12
  11. package/backends/source_types_cn.json +27 -1
  12. package/bin/argo +78 -5
  13. package/bin/argo.js +33 -2
  14. package/config.yaml +269 -63
  15. package/docs/ARGO_INTRO.md +5 -3
  16. package/docs/ENGINE_CATALOG.md +50 -39
  17. package/docs/MCP_SETUP.md +3 -1
  18. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  19. package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
  20. package/engines/specs/cfets.yaml +41 -0
  21. package/engines/specs/defillama.yaml +38 -0
  22. package/engines/specs/firecrawl.yaml +1 -1
  23. package/engines/specs/fxtwitter.yaml +5 -0
  24. package/engines/specs/gleif.yaml +55 -0
  25. package/engines/specs/hkex.yaml +40 -0
  26. package/engines/specs/keenable_free.yaml +36 -0
  27. package/engines/specs/shfe.yaml +39 -0
  28. package/engines/specs/stackexchange.yaml +2 -2
  29. package/engines/specs/tavily_free.yaml +56 -0
  30. package/engines/specs/yuanbao.yaml +93 -0
  31. package/mcp/clients.yaml +32 -0
  32. package/package.json +7 -2
  33. package/packages/dsh-plugin/README.md +25 -6
  34. package/packages/dsh-plugin/dsh/client.js +402 -47
  35. package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
  36. package/packages/dsh-plugin/dsh/index.js +135 -1876
  37. package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
  38. package/packages/dsh-plugin/dsh/runner.js +666 -0
  39. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  40. package/packages/dsh-plugin/package.json +3 -3
  41. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  42. package/packages/dsh-plugin/test/client.test.js +177 -2
  43. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  44. package/packages/dsh-plugin/test/index.test.js +44 -0
  45. package/scripts/adaptive.py +126 -43
  46. package/scripts/article_meta.py +131 -0
  47. package/scripts/bench_tail_ab.py +281 -0
  48. package/scripts/bounded_run.py +91 -49
  49. package/scripts/budget.py +247 -0
  50. package/scripts/budget_cli.py +93 -0
  51. package/scripts/build_release_assets.sh +65 -0
  52. package/scripts/cache.py +60 -72
  53. package/scripts/ci_drift_gate.py +90 -0
  54. package/scripts/clock_bridge.py +22 -0
  55. package/scripts/content_boundary.py +46 -0
  56. package/scripts/content_security.py +119 -4
  57. package/scripts/content_signals.py +26 -5
  58. package/scripts/cross_lingual_trace.py +293 -0
  59. package/scripts/dispatch_result.py +105 -0
  60. package/scripts/engine_add.py +602 -0
  61. package/scripts/engine_dispatch.py +58 -86
  62. package/scripts/engine_env.py +5 -0
  63. package/scripts/engine_families.py +6 -0
  64. package/scripts/engine_policy.py +7 -7
  65. package/scripts/engines.py +21 -0
  66. package/scripts/engines_base.py +3 -3
  67. package/scripts/engines_builders.py +12 -0
  68. package/scripts/engines_builders_batch10.py +48 -3
  69. package/scripts/engines_builders_batch14.py +656 -0
  70. package/scripts/engines_builders_batch15.py +249 -0
  71. package/scripts/engines_builders_keyless.py +111 -0
  72. package/scripts/engines_builders_tech.py +0 -1
  73. package/scripts/evidence.py +31 -3
  74. package/scripts/evidence_loop.py +58 -22
  75. package/scripts/fetch_quality.py +13 -0
  76. package/scripts/fetch_v3.py +28 -3
  77. package/scripts/gen_routing_snapshot.py +11 -0
  78. package/scripts/http_client.py +24 -38
  79. package/scripts/install.ps1 +18 -0
  80. package/scripts/install.sh +16 -1
  81. package/scripts/job.py +3 -3
  82. package/scripts/local_seek.py +41 -5
  83. package/scripts/macro_countries.py +15 -0
  84. package/scripts/matrix_search_eval.py +192 -0
  85. package/scripts/mcp_compact.py +234 -0
  86. package/scripts/mcp_handlers.py +60 -642
  87. package/scripts/mcp_launch.sh +10 -8
  88. package/scripts/mcp_local.py +241 -0
  89. package/scripts/mcp_server.py +19 -14
  90. package/scripts/mcp_setup.py +33 -9
  91. package/scripts/mcp_tools.py +1 -0
  92. package/scripts/mcp_transport.py +1 -1
  93. package/scripts/mcp_warm.py +127 -0
  94. package/scripts/net_proxy.py +27 -0
  95. package/scripts/output_transparency.py +200 -0
  96. package/scripts/paid_policy.py +98 -0
  97. package/scripts/quota.py +158 -13
  98. package/scripts/recovery.py +7 -3
  99. package/scripts/recovery_runner.py +150 -0
  100. package/scripts/replay_eval.py +18 -0
  101. package/scripts/research.py +37 -21
  102. package/scripts/research_expand.py +24 -1
  103. package/scripts/retry_policy.py +76 -0
  104. package/scripts/route.py +6 -6
  105. package/scripts/route_cache.py +18 -0
  106. package/scripts/route_combo.py +26 -5
  107. package/scripts/route_domains.py +170 -31
  108. package/scripts/safe_conn.py +52 -0
  109. package/scripts/search.py +15 -27
  110. package/scripts/search_attribution.py +100 -3
  111. package/scripts/search_cli.py +24 -11
  112. package/scripts/search_entry.py +8 -1
  113. package/scripts/search_output.py +99 -15
  114. package/scripts/search_pipeline.py +62 -195
  115. package/scripts/search_rank.py +6 -0
  116. package/scripts/search_types.py +107 -9
  117. package/scripts/semantic_evidence.py +179 -3
  118. package/scripts/serp_guard.py +138 -7
  119. package/scripts/serp_kill_log.py +25 -2
  120. package/scripts/stats_cli.py +13 -1
  121. package/scripts/tail_completion.py +212 -0
  122. package/scripts/url_safety.py +126 -3
  123. package/scripts/usage_log.py +45 -0
  124. package/scripts/wide_research.py +45 -0
  125. package/scripts/zhihu_quota.py +139 -0
  126. package/sub-skills/local-search/bm25.py +27 -3
  127. package/sub-skills/local-search/cli_runner.py +331 -0
  128. package/sub-skills/local-search/parsers.py +328 -0
  129. package/sub-skills/local-search/search_v3.py +31 -674
  130. package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
  131. package/assets/readme/settings-keys.png +0 -0
  132. package/assets/readme/settings-tools.png +0 -0
  133. package/sub-skills/ego-search/references/browser-runtime.md +0 -43
  134. package/sub-skills/ego-search/references/install.md +0 -67
  135. package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
  136. package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
  137. package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
  138. package/sub-skills/js-run/tests/test_jsrun.py +0 -165
  139. package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
  140. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
  141. package/sub-skills/local-seek/references/git-integration.md +0 -54
  142. package/sub-skills/local-seek/references/remote-handoff.md +0 -46
  143. package/sub-skills/local-seek/references/rg-recipes.md +0 -96
  144. package/sub-skills/local-seek/references/strategies.md +0 -52
  145. package/sub-skills/local-seek/references/structural-search.md +0 -59
  146. package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
  147. package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
  148. package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
  149. package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
  150. package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
  151. package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
  152. package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
  153. package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
  154. package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
@@ -8,10 +8,23 @@
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
- from dataclasses import dataclass
12
-
13
11
  from search_rank import filter_results_by_domains
12
+ from search_types import slot_record as dataclass
14
13
  from typing import Any
14
+ from output_transparency import ( # noqa: F401
15
+ _AGGREGATOR_HOSTS,
16
+ _FAILURE_META,
17
+ _FAILURE_NON_ERROR,
18
+ _NON_ERROR_OUTCOME,
19
+ _SEVERITY,
20
+ _UNCERTAINTY_HIGH,
21
+ _UNCERTAINTY_LOW,
22
+ _UNCERTAINTY_MEDIUM,
23
+ _hostname,
24
+ build_failures,
25
+ build_uncertainty,
26
+ decorate_compact_payload,
27
+ )
15
28
 
16
29
 
17
30
  # 本模块的 6 处日志都写在 **except 处理器**里,且分散在 shape_response 的
@@ -72,6 +85,39 @@ _AGENT_RESULT_FIELDS = (
72
85
  )
73
86
 
74
87
 
88
+ def _slim_verify(v: Any) -> dict[str, Any] | None:
89
+ """verify 块的 agent 精简版:保留「核验后改了什么」,丢掉逐条细节。
90
+
91
+ why:`--fields agent` 此前把整个 verify 块剥掉,而这是**唯一**能说明
92
+ 「花几秒核验换来了什么」的字段。保留 revision_summary 与每条 delta,
93
+ 体积约 0.2–0.8 KB;pre/post 绝对分与 title 已在 results[] 可查,不重复。
94
+ """
95
+ if not isinstance(v, dict):
96
+ return None
97
+ out: dict[str, Any] = {}
98
+ for key in ("revision_summary", "pending", "skipped_cached", "semantic"):
99
+ if v.get(key) is not None:
100
+ out[key] = v[key]
101
+ rows = []
102
+ for r in v.get("verified") or []:
103
+ if not isinstance(r, dict):
104
+ continue
105
+ rows.append({k: r[k] for k in
106
+ ("url", "delta", "post_absorption", "fetch_method")
107
+ if r.get(k) is not None})
108
+ if rows:
109
+ out["verified"] = rows
110
+ return out or None
111
+
112
+
113
+ def _slim_reorder(r: Any) -> dict[str, Any] | None:
114
+ """证据回写排序的 agent 精简版:只留结论(是否重排、动了几处)。"""
115
+ if not isinstance(r, dict):
116
+ return None
117
+ out = {k: r[k] for k in ("reordered", "moved") if r.get(k) is not None}
118
+ return out or None
119
+
120
+
75
121
  def _strip_for_agent(payload: dict[str, Any]) -> dict[str, Any]:
76
122
  """--fields agent:输出只留答案内容(P2-2,2026-09-13)。
77
123
 
@@ -93,9 +139,33 @@ def _strip_for_agent(payload: dict[str, Any]) -> dict[str, Any]:
93
139
  # 阶段耗时:默认就带(--no-timing 才没有)。剥掉会让使用者看不到
94
140
  # 「这次慢在哪」,也拿不到自己动手优化所需的依据。
95
141
  "timing",
142
+ # 综合可信度:一个 level 就能决定「直接引用」还是「先核验」。
143
+ # 剥掉它等于让 agent 自己交叉比对来源数/同站/聚合站/引擎失败——
144
+ # 而那正是它最容易漏掉的一步。
145
+ "meta",
146
+ # 结构化失败清单:类别 + 是否可重试 + 建议动作(≤6 条短字段)。
147
+ # 只有 errors 的整行文本时,agent 知道「坏了」但不知道「该重试、
148
+ # 换源还是去配 key」;这两件事的处置完全不同。
149
+ "failures",
150
+ # 信任级别:网页文本一律是不可信外部数据,随答案一起到达,
151
+ # 避免调用方把摘要里的指令当任务执行(输出层声明,非提示词)。
152
+ "content_trust",
153
+ # 核验产出:verify/evidence_reorder 是**花了墙钟换来的**,且是
154
+ # 「该不该引用这批结果」的直接依据。此前被整块剥掉,调用方付了
155
+ # 几秒核验却只看到 verified_count 一个数(2026-10-22 实测)。
156
+ # 值由下方 _slim_verify/_slim_reorder 压缩,不整块照搬。
157
+ "verify", "evidence_reorder",
96
158
  )
97
159
  out: dict[str, Any] = {k: payload[k] for k in keep_top
98
160
  if payload.get(k) is not None}
161
+ for _key, _slim in (("verify", _slim_verify),
162
+ ("evidence_reorder", _slim_reorder)):
163
+ if payload.get(_key) is not None:
164
+ _value = _slim(payload[_key])
165
+ if _value is not None:
166
+ out[_key] = _value
167
+ else:
168
+ out.pop(_key, None)
99
169
  slim_results = []
100
170
  for r in payload.get("results") or []:
101
171
  if not isinstance(r, dict):
@@ -219,7 +289,8 @@ def format_timing(t: dict[str, Any]) -> str:
219
289
  f" 引擎并发 墙钟 {d.get('wall_ms')} ms / 引擎合计 "
220
290
  f"{d.get('engine_sum_ms')} ms → 并发效率 "
221
291
  f"{d.get('parallel_efficiency')}(跑了 {d.get('engines_run')} 个,"
222
- f"早停={d.get('early_stopped')},浪费 {d.get('wasted_ms')} ms)")
292
+ f"早停={d.get('early_stopped')},尾等待 {d.get('tail_wait_ms')} ms,"
293
+ f"弃置 {d.get('abandoned_engine_ms')} ms)")
223
294
  if "overhead_ms" in t:
224
295
  lines.append(
225
296
  f" 固定开销 {t['overhead_ms']} ms"
@@ -331,18 +402,6 @@ def format_text_output(results: dict[str, Any]) -> str:
331
402
  return "\n".join(lines)
332
403
 
333
404
 
334
- # 不算失败的 outcome 状态:这些情况「引擎跑了、没问题」,不该出现在 errors[]
335
- # cancelled = 早停收工时被主动弃置(结果已够,引擎没失败也没超窗)
336
- _NON_ERROR_OUTCOME = frozenset({
337
- "ok", "ok-cached", "partial", "no-results", "no-results-cached",
338
- "cancelled",
339
- # 配额耗尽的跳过是**持续状态**,不是本次查询的失败:标记默认持续到周期
340
- # 边界(一天),若每个查询都把它塞进 errors,调用方会看到一条持续一天的
341
- # 「错误」并误判本次检索降级。状态本身在 --list-engines 可见。
342
- "skipped-quota-exhausted",
343
- })
344
-
345
-
346
405
  def _collect_errors(raw_results: dict[str, list[dict[str, Any]]],
347
406
  engine_outcomes: list[dict[str, Any]] | None = None
348
407
  ) -> list[str]:
@@ -379,6 +438,7 @@ def _collect_errors(raw_results: dict[str, list[dict[str, Any]]],
379
438
  _add(f"{o.get('engine')}: {detail}")
380
439
  return errors
381
440
 
441
+
382
442
  @dataclass(frozen=True)
383
443
  class _ShapeContext:
384
444
  """输出成形的全部选项(请求侧只读 + 一个本地检索钩子)。"""
@@ -603,4 +663,28 @@ def shape_response(ctx: _ShapeContext, result: dict[str, Any]) -> dict[str, Any]
603
663
  # 待核验 0 条」这类自相矛盾的口径。
604
664
  _el["suggested"] = _sug
605
665
  _el["pending_count"] = len(_sug)
666
+ # 综合可信度:读的 results 必须与最终输出同源,所以放在收口(与 count
667
+ # 重算同处)——域过滤与本地命中并入都发生在这之前。只读不写结果,
668
+ # 失败时静默跳过(增强失败不得让一次搜索失败)。
669
+ try:
670
+ result["meta"] = {"uncertainty": build_uncertainty(result)}
671
+ except Exception as e:
672
+ _log(f"可信度评估跳过: {type(e).__name__}")
673
+ # 结构化失败清单:与 meta 同一收口,读的 outcome 与漏斗/可信度同源。
674
+ # 失败时静默跳过(增强失败不得让一次搜索失败)。
675
+ try:
676
+ failures = build_failures(result)
677
+ if failures:
678
+ result["failures"] = failures
679
+ except Exception as e:
680
+ _log(f"失败清单跳过: {type(e).__name__}")
681
+ # 信任级别与边界清洗:所有入口(full/agent/MCP)共用同一个收口,
682
+ # 避免 CLI/agent 档带 content_trust 却未剥离网页自带的 </untrusted-web-content>。
683
+ try:
684
+ from content_boundary import (CONTENT_TRUST_UNTRUSTED,
685
+ sanitize_results_boundary)
686
+ sanitize_results_boundary(result.get("results"))
687
+ result["content_trust"] = CONTENT_TRUST_UNTRUSTED
688
+ except Exception:
689
+ pass
606
690
  return result
@@ -30,7 +30,7 @@ http_client / engines_base / engine_dispatch。
30
30
 
31
31
  from __future__ import annotations
32
32
 
33
- from dataclasses import dataclass, field, replace
33
+ from search_types import slot_record as dataclass, slot_field as field, slot_replace as replace
34
34
  from typing import Any
35
35
 
36
36
  # 模块级曾有一行 `import logging`,而本模块 7 处 `logging.getLogger(...)`
@@ -56,8 +56,9 @@ from variant_recall import augment_with_variants
56
56
  from phrase_filter import apply_phrase_filter, extract_phrases
57
57
  from except_sets import OPT_IMPORT, SHAPE_BENIGN
58
58
  from postprocess_signals import apply_tfidf_polish, detect_number_conflict
59
- from search_attribution import usage_log_attribution
59
+ from search_attribution import usage_log_attribution, is_synthetic_run
60
60
  from search_output import _collect_errors, _slow_query_ttl, build_funnel
61
+ from serp_kill_log import clean_engine_lists
61
62
  from search_rank import (
62
63
  _LANG_SCRIPT,
63
64
  _RERANK_DEGRADED_STATUSES,
@@ -153,7 +154,11 @@ class _SearchRun:
153
154
  engine_latency: dict[str, float] = field(default_factory=dict)
154
155
  wasted_ms: int = 0
155
156
  useful_ms: int = 0
157
+ dispatch_tail: dict[str, int] = field(default_factory=dict)
156
158
  early_stopped: bool = False
159
+ # 早停弃置且仍存活的 (engine, holder, now_fn):finalize 交给
160
+ # tail_completion 后台收割。纯数据字段,不改变任何既有阶段的行为。
161
+ stranded: tuple = ()
157
162
  budget_used_ms: int | None = None
158
163
  budget_total_ms: int | None = None
159
164
  elapsed: int = 0
@@ -236,16 +241,8 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
236
241
  merged = run.merged
237
242
 
238
243
  # 融合输入派生(原先在 execute_search 里,属于加工的第一步)
239
- valid_lists = [
240
- res for res in raw_results.values()
241
- if res and any(isinstance(r, dict) and "error" not in r for r in res)
242
- ]
243
- # 去掉 error-only 列表中的 error 条目
244
- clean_lists = []
245
- for res in valid_lists:
246
- clean = [r for r in res if isinstance(r, dict) and "error" not in r]
247
- if clean:
248
- clean_lists.append(clean)
244
+ # 去 error 与 SERP 守卫同源:缓存命中的结果也要过同一道门(issue #18)。
245
+ clean_lists = clean_engine_lists(raw_results, query)
249
246
 
250
247
  # 查询主语言:噪声门与语言能力加权共用同一个判定(唯一来源,
251
248
  # 避免两处各自算导致行为漂移)。
@@ -387,8 +384,8 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
387
384
  _funnel_deduped = len(merged)
388
385
  if merged:
389
386
  try:
390
- from serp_guard import is_serp_or_jump_url as _is_serp # 轻依赖,锁见 test_hotpath_import_lint.py
391
- merged = [r for r in merged if not _is_serp(r.get("url", ""))]
387
+ from serp_guard import is_unusable_url as _is_unusable # SERP + 内容农场,标定见 test_content_farm_guard.py
388
+ merged = [r for r in merged if not _is_unusable(r.get("url", ""), r.get("title", ""), r.get("snippet", ""))]
392
389
  except ImportError:
393
390
  pass # serp_guard 不可用时跳过(本地五维 rerank 已对 SERP 降权)
394
391
 
@@ -515,176 +512,33 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
515
512
  except OPT_IMPORT + SHAPE_BENIGN: # 守卫不可用不设卡(fail-open)
516
513
  _echo_triggered = False
517
514
  if not merged or _echo_triggered:
518
- # ── 恢复链 L0:early-stop 误判时补跑被跳过的同域引擎 ──
519
- # (同域优先于 L3 换跨域引擎;实现在 recovery_l0)。死线 = 用户
520
- # 声明的整体超时扣掉 dispatch 已耗墙钟——补跑不得把总时长推出
521
- # 用户预算(语义见 recovery_l0 模块 docstring 的死线约束)。
522
- try:
523
- import time as _time
524
- from recovery_l0 import maybe_retry_skipped
525
- try:
526
- _l0_deadline = _time.monotonic() + max(
527
- 0.0, float(req.timeout or 0)
528
- - float(run.budget_used_ms or 0) / 1000.0)
529
- except (TypeError, ValueError):
530
- _l0_deadline = None
531
- _l0_merged, _l0_info = maybe_retry_skipped(
532
- run.early_stopped, engines_combo, raw_results,
533
- available_engines, query, _run_one, _ingest,
534
- max_results, _emit_usage, deadline=_l0_deadline)
535
- except SHAPE_BENIGN:
536
- _l0_merged, _l0_info = [], None
537
- if _l0_merged and _l0_info:
538
- recovery_info = _l0_info
539
- merged = _l0_merged
540
- if not merged or _echo_triggered:
541
- try:
542
- from recovery import run_recovery
543
- tried = list(raw_results.keys()) or list(engines)
544
- fallback_engines = decision.get("engines_fallback") or []
545
- # 域路由零结果:恢复链放行 L3 换引擎。复杂度门此前把简单查询
546
- # 压到 L2——L3 被禁 + 全域零结果 = 域命中查询无解(实测
547
- # macro_data「中国GDP」零结果、恢复链空转)。engines_fallback
548
- # 里是路由的定向保底声明(域未试成员优先),代价可控。
549
- rec_level = _max_rec_level
550
- if _echo_triggered:
551
- # 退化是「引擎选错了」而非「查询写得不好」:L1 放宽 / L2 换词
552
- # 都救不了「上游把整句当单词查」,必须换引擎(L3)。
553
- rec_level = "L3"
554
- elif fallback_engines and decision.get("domain") not in (
555
- None, "", "general", "general_search") \
556
- and (rec_level is None or rec_level < "L3"):
557
- rec_level = "L3"
558
- try:
559
- enabled_set = set(available_engines())
560
- except Exception:
561
- enabled_set = None
562
-
563
- def _recovery_executor(rq: str, rengines: list[str]) -> list[dict[str, Any]]:
564
- """恢复执行器:对冲竞速跑候选引擎,取首个非空。跳过缓存避免污染。
565
-
566
- 并行形态(2026-09-29):原为严格串行——域查询零结果时 recovery
567
- 段占 process_ms 的 79–86%(实测中位 906ms;「黄金价格」2019ms、
568
- 「比特币行情」1989ms),是体感「有时特别慢」的直接原因。改为
569
- 「起步间隔 = grace 的有界并发」(bounded_run.run_bounded 的
570
- stagger 语义,与 dispatch 串行路径同一套对冲哲学,见
571
- engine_dispatch._run_engines_bounded):首引擎先跑,grace 内
572
- 交付仍只付一次调用,超时才补发下一匹;任一引擎交付非空即收工。
573
- grace 复用 hooks.primary_grace_s(标定依据见 search._PRIMARY_GRACE_S),
574
- 不新增开关面。总等待上限 = grace×(n-1) + timeout + 1s ≤ 串行版
575
- n×timeout,最坏情况也不慢于改造前。消融:把 primary_grace 调到
576
- 大于引擎超时上限即完全退回严格串行(与 ARGO_SERIAL_STAGGER_S
577
- 同思路)。daemon 线程不 join,卡住的 HTTP 读拖不垮止损。
578
- """
579
- def _fire(eng: str) -> list[dict[str, Any]]:
580
- _dk = {k: v for k in ("domain", "sub_domain") if (v := getattr(req, "engine_" + k, None))}
581
- # 记下真正发出的调用:恢复段走的不是 dispatch,若不单独记账,
582
- # 漏斗会算出「routed 2 → called 2 → returned 5」这种自相矛盾的
583
- # 账(实测「python 怎么读csv」),而这正是最需要被看见的路径。
584
- _recovery_engines.add(eng)
585
- try:
586
- # 恢复路径同样携带时间窗,避免恢复时丢弃用户约束;
587
- # --no-cache 也要透传:本函数的契约是「跳过缓存避免污染」,
588
- # 不透传会让恢复段既读到用户明确拒绝的旧缓存,又把自己的
589
- # 临时查询写进缓存(dispatch 那条路径是透传的,两处不一致)。
590
- res = engine_search(rq, eng, n=max_results,
591
- timeout=timeout, depth=depth, mode=mode,
592
- since=since_iso, until=until_iso,
593
- skip_cache=skip_cache, **_dk)
594
- except Exception as e:
595
- import sys
596
- sys.stderr.write(
597
- f"[recovery] 恢复链异常({type(e).__name__}): {e}\n")
598
- res = []
599
- goods = [r for r in (res or [])
600
- if isinstance(r, dict) and "error" not in r]
601
- # 恢复段同样进配额账:它真的打了引擎(engine_search),
602
- # 而 quota_batch.add 的全仓唯一常规入口在 dispatch 的
603
- # _run_one 里——恢复链不走那条路,不在这里补一行,
604
- # 这批消耗就落在本地账本之外(见下方 flush 处注释)。
605
- # add 是 list.append(GIL 下原子),对冲线程并发调用安全。
606
- quota_batch.add(eng, bool(goods))
607
- if goods:
608
- for r in goods:
609
- r.setdefault("_engine", eng)
610
- r.setdefault("_recovered", True)
611
- # 回声不算「恢复成功」(判据与理由见
612
- # postprocess_signals.drop_echo_items):不拦则 L1 放宽
613
- # 查询后收工,L3 换引擎永不执行。
614
- try:
615
- from postprocess_signals import drop_echo_items as _drop_echo
616
- goods = _drop_echo(goods, rq)
617
- except OPT_IMPORT + SHAPE_BENIGN:
618
- pass
619
- return goods
620
-
621
- if len(rengines) <= 1:
622
- return _fire(rengines[0]) if rengines else []
623
- grace = max(0.0, float(hooks.primary_grace_s or 0.0))
624
- wait_s = grace * (len(rengines) - 1) \
625
- + max(1.0, float(timeout)) + 1.0
626
-
627
- def _enough(finished: list[tuple[str, Any]]) -> bool:
628
- # 任一引擎交付非空即收工:不再起排队中的引擎(省调用),
629
- # 已在跑的归 unfinished(daemon,结果丢弃但配额已如实记)。
630
- return any(isinstance(v, list) and v for _i, v in finished)
631
-
632
- finished, _unfinished = run_bounded(
633
- rengines, _fire, wait_s, max_workers=3,
634
- enough=_enough, stagger_s=grace)
635
- # finished 按完成先后排列:首个非空即「最快非空」。与串行版
636
- # 「首个非空」的差别只在来源引擎——恢复候选都过 run_recovery
637
- # 的 query-signal 门,质量由它把关。
638
- for _item, value in finished:
639
- if isinstance(value, list) and value:
640
- return value
641
- return []
642
-
643
- rec_results, rec_result = run_recovery(
644
- query, tried, _recovery_executor,
645
- engines_fallback=fallback_engines, enabled=enabled_set, mode=mode,
646
- max_level=rec_level)
647
- recovery_info = rec_result.to_dict()
648
- # P2-6:恢复使用日志——query 截断脱敏,只记概览不记明细
649
- if _emit_usage is not None:
650
- try:
651
- _emit_usage("recovery", {
652
- "query": (query[:60] if query else query),
653
- "triggered": recovery_info.get("triggered"),
654
- "recovered": recovery_info.get("recovered"),
655
- "level_used": recovery_info.get("level_used"),
656
- "strategy_used": recovery_info.get("strategy_used"),
657
- "steps_tried": len(recovery_info.get("steps_tried") or []),
658
- "final_query": (recovery_info.get("final_query") or "")[:60],
659
- "note": recovery_info.get("note", ""),
660
- })
661
- except SHAPE_BENIGN: # 使用日志记录失败不影响恢复结果本身
662
- pass
663
- if rec_results:
664
- # 恢复结果复检壳口径:主链的壳过滤跑在恢复链之前,恢复产出
665
- # 是唯一可能把壳带回最终输出的入口(唯一出口在
666
- # serp_guard.drop_shell_urls,L0/L3/充分性三处共用)。
667
- try:
668
- from recovery_l0 import finalize_l3_results
669
- merged = finalize_l3_results(rec_results, max_results)
670
- except ImportError:
671
- merged = deduplicate_by_url(rec_results)[:max_results]
672
- # 恢复引擎按引擎分组记回 raw_results:engines_used 此前不含
673
- # 救援引擎(provenance 断链,实测恢复成功后 engines_used 仍
674
- # 只列原 combo),自适应学习也看不到恢复成功信号。
675
- _rec_by_eng: dict[str, list] = {}
676
- for r in merged:
677
- eng = r.get("_engine") or r.get("source") or ""
678
- if eng:
679
- r.setdefault("consensus_engines", [eng])
680
- _rec_by_eng.setdefault(eng, []).append(r)
681
- for _eng, _lst in _rec_by_eng.items():
682
- raw_results.setdefault(_eng, _lst)
683
- except ImportError:
684
- pass # recovery 模块不可用
685
- except Exception as e:
686
- _log(
687
- f"错误恢复跳过: {type(e).__name__}")
515
+ from recovery_runner import run_pipeline_recovery
516
+ _dk = {k: v for k in ("domain", "sub_domain") if (v := getattr(req, "engine_" + k, None))}
517
+ merged, _rec_info = run_pipeline_recovery(
518
+ query=query,
519
+ merged=merged,
520
+ raw_results=raw_results,
521
+ decision=decision,
522
+ engines=engines,
523
+ available_engines_fn=available_engines,
524
+ max_results=max_results,
525
+ max_rec_level=_max_rec_level,
526
+ echo_triggered=_echo_triggered,
527
+ primary_grace_s=float(hooks.primary_grace_s or 0.0),
528
+ timeout=timeout,
529
+ depth=depth,
530
+ mode=mode,
531
+ since_iso=since_iso,
532
+ until_iso=until_iso,
533
+ skip_cache=skip_cache,
534
+ quota_batch=quota_batch,
535
+ engine_search_fn=engine_search,
536
+ emit_usage_fn=_emit_usage,
537
+ recovery_engines_set=_recovery_engines,
538
+ engine_domain_extra=_dk,
539
+ )
540
+ if _rec_info:
541
+ recovery_info = _rec_info
688
542
 
689
543
  # 配额记账落盘:整批一次写入文件(同一次搜索的 N 个引擎合并为一次写)。
690
544
  # 位置是契约:必须在**最后一个 quota add 点**之后。D6 补搜走后,
@@ -886,6 +740,7 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
886
740
  funnel = run.funnel
887
741
  result_payload = run.result_payload
888
742
  wasted_ms, useful_ms = run.wasted_ms, run.useful_ms
743
+ dispatch_tail = run.dispatch_tail
889
744
  early_stopped = run.early_stopped
890
745
  budget_used_ms, budget_total_ms = run.budget_used_ms, run.budget_total_ms
891
746
  elapsed = run.elapsed
@@ -920,7 +775,7 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
920
775
  # 自适应学习(单事务批量写入 + cache_guard 统一状态态过滤)
921
776
  try:
922
777
  import contextlib
923
- from adaptive import get_learner
778
+ from adaptive import get_learner, scenario_of
924
779
  learner = get_learner()
925
780
  _outcome_of = {o.get("engine"): o
926
781
  for o in engine_outcomes if isinstance(o, dict)}
@@ -945,12 +800,27 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
945
800
  quality = None
946
801
  learner.record(eng, success=success, latency_ms=latency,
947
802
  cost=0.0 if cost >= 0.85 else 0.001, empty=empty,
948
- quality=quality)
803
+ quality=quality, **scenario_of(decision))
949
804
  except ImportError:
950
805
  pass
951
806
  except Exception as e:
952
807
  _log(f"自适应学习记录跳过: {type(e).__name__}")
953
808
 
809
+ # 早停后台补全(2026-10-25):弃置引擎的交付不再丢弃,宽限内完成的
810
+ # 记学习器真实成绩 + 合并写 L2 缓存(ARGO_TAIL_COMPLETE_S 控制,0=关)。
811
+ # 放在缓存写与学习记账之后:补全读的是刚写的主缓存条目;主响应已定,
812
+ # 这里的一切都是锦上添花,异常 fail-open。
813
+ if run.stranded:
814
+ try:
815
+ from tail_completion import hand_off
816
+ hand_off(run.stranded, query=query,
817
+ cache_engine_key=cache_engine_key,
818
+ max_results=max_results, domain=domain,
819
+ mode=mode, depth=depth, decision=decision,
820
+ skip_cache=skip_cache, cache_instance=cache)
821
+ except Exception as e:
822
+ _log(f"后台补全移交跳过: {type(e).__name__}")
823
+
954
824
  # 语言偏好:记录本轮查询语 + 输出观测快照(默认中英 + 系统 + 习惯)
955
825
  lang_pref_info: dict[str, Any] | None = None
956
826
  try:
@@ -979,15 +849,15 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
979
849
  # 字符(与 recovery 流同一脱敏纪律);记录失败静默,绝不拖累主路径。
980
850
  # 取自 req 而非局部别名:_emit_usage 是 postprocess 的局部变量,
981
851
  # 本函数(finalize)不在其作用域内,直接引用会 NameError。
982
- if req.emit_usage_log is not None:
852
+ if req.emit_usage_log is not None and not is_synthetic_run(raw_results):
983
853
  try:
984
- # 引擎级耗时归因(2026-10-15 起):写在 search_attribution,本处只接线。
854
+ # 归因走 search_attribution;合成引擎(基准/混沌)整条不入日志。
985
855
  req.emit_usage_log("query", {
986
856
  "query": (query[:60] if query else query),
987
857
  "count": len(merged),
988
858
  "elapsed_ms": round(elapsed),
989
859
  "engines_used": list(raw_results.keys())[:8],
990
- **usage_log_attribution(engine_latency),
860
+ **usage_log_attribution(engine_latency, engine_outcomes=engine_outcomes),
991
861
  "errors": len(_collect_errors(raw_results, engine_outcomes)),
992
862
  "recovered": bool(recovery_info.get("recovered")) if recovery_info else False,
993
863
  "early_stopped": bool(early_stopped),
@@ -1054,19 +924,16 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
1054
924
  out["timing"] = timing.summary()
1055
925
  out["timing"]["elapsed_ms"] = elapsed
1056
926
  out["timing"]["dispatch"] = {
1057
- # 用 dispatch 自己那口单调钟量出来的墙钟(budget_used_ms),不用外层
1058
- # 这笔 time.time() 差值:useful/wasted 都是单调钟算的,混用两种钟会
1059
- # 对不上账。注意 2026-10-10 记账改版起 useful + wasted 与 wall 没有
1060
- # 恒等式关系,别再按恒等式读仪表(口径真源:engine_dispatch 记账注释)。
927
+ # 墙钟/尾等待/弃置的口径真源见 dispatch_result.py;useful + wasted 与
928
+ # wall 没有恒等式(2026-10-10 起),不要再按恒等式读仪表。
1061
929
  "wall_ms": budget_used_ms,
1062
930
  "engines_run": len(engine_latency),
1063
931
  "engine_sum_ms": eng_sum,
1064
932
  "parallel_efficiency": (round(eng_sum / budget_used_ms, 2)
1065
933
  if budget_used_ms else None),
1066
- # useful = 最后一个贡献结果引擎的完成时刻;wasted = 被编排主动弃置
1067
- # 的引擎已花墙钟之和。与 wall 无恒等式(真源:engine_dispatch 记账注释)。
1068
934
  "useful_ms": useful_ms,
1069
935
  "wasted_ms": wasted_ms,
936
+ **dispatch_tail,
1070
937
  "early_stopped": early_stopped,
1071
938
  }
1072
939
  if budget_total_ms is not None:
@@ -96,6 +96,12 @@ _ENGINE_FUSION_WEIGHTS: dict[str, float] = {
96
96
  "stackoverflow": 1.1, "imdb": 1.2, "thesportsdb": 1.2, "itunes": 1.2,
97
97
  "finviz": 1.2, "sina_quote": 1.2, "tencent_quote": 1.2, "eastmoney": 1.2,
98
98
  "fred": 1.3, "worldbank": 1.3, "nbs_stats": 1.3, "eurostat": 1.3, "fx_rate": 1.2, "frankfurter": 1.2,
99
+ # 2026-10-25 补表:学术聚合与官方一手数据此前表外=1.0 基线。实证缺口:
100
+ # 学术域查询常由 core 独家交付(当日实测组合里 arxiv/local_pubmed 零结果、
101
+ # 只 core 有货)却拿基线权重;官方 open data / 法律 / 标准系同理。
102
+ "core": 1.3, "openreview": 1.3, "zenodo": 1.2, "datacite": 1.2,
103
+ "sec_edgar": 1.3, "federal_register": 1.3, "eu_opendata": 1.3,
104
+ "egov_law": 1.3, "openstd": 1.3, "who_gho": 1.3, "usgs": 1.3, "nasa_cmr": 1.3,
99
105
  # 通用引擎(基线)
100
106
  "duckduckgo": 1.0, "local_bing": 1.0, "local_duckduckgo": 1.0,
101
107
  "local_google": 1.0, "anysearch": 1.05, "byted": 1.1, "bocha": 1.0,
@@ -1,20 +1,118 @@
1
1
  #!/usr/bin/env python3
2
- """search_types.py — 统一类型系统(已废弃,勿接入)
2
+ """search_types.py — 统一类型系统(轻量记录 + 零依赖 dataclass 替代)。
3
3
 
4
- 早期设计声明的统一 schema 层,从未被任何模块 import(grep 全仓库无引用):
5
- - 引擎输出直接透传 dict,运行时防御在 engines_base/safe_search 与
6
- search.py 的 execute_search 里完成
7
- - 保留本文件仅避免旧版外部脚本 import 断裂;新代码禁止使用
4
+ `slot_record` / `slot_field` / `slot_replace` 是为热路径准备的
5
+ `@dataclass` / `field` / `replace` 零依赖替代:`dataclasses` 的导入链是
6
+ `dataclasses → inspect → tokenize → dis`(实测 ~8ms),而 `SearchResult`
7
+ 这类记录在每次搜索都要构造,导入成本全落在固定开销上。
8
+
9
+ 兼容面:挂 `__dataclass_fields__`,标准库的 `dataclasses.replace` /
10
+ `is_dataclass` / `fields` 对实例照常工作(`_field_type` 惰性解析)。
11
+
12
+ 仍不建议新代码接入本模块的 `SearchResult`(引擎输出直接透传 dict,运行时
13
+ 防御在 engines_base/safe_search 里);这里的价值在**替代实现本身**。
8
14
  """
9
15
  from __future__ import annotations
10
16
 
17
+ import sys
11
18
  from enum import Enum
12
-
13
- from dataclasses import dataclass, field, asdict
14
19
  from typing import Any
15
20
 
16
21
 
17
- @dataclass
22
+ class _FieldMeta:
23
+ """`__dataclass_fields__` 的兼容描述子。
24
+
25
+ `dataclasses.replace` 只读 `.init`;`dataclasses.fields` 会按
26
+ `_field_type is dataclasses._FIELD` 过滤,所以 `_field_type` 做成**惰性
27
+ 属性**:dataclasses 已被 import 时(也就是有人真去调 `fields()` 时)
28
+ 返回真哨兵,没 import 时返回 None。热路径因此仍然零导入——`_FIELD`
29
+ 只在 dataclasses 已经进场后才去取它。
30
+ """
31
+
32
+ __slots__ = ("name", "init")
33
+
34
+ def __init__(self, name: str) -> None:
35
+ self.name = name
36
+ self.init = True
37
+
38
+ @property
39
+ def _field_type(self) -> Any:
40
+ dc = sys.modules.get("dataclasses")
41
+ return getattr(dc, "_FIELD", None) if dc is not None else None
42
+
43
+
44
+ class _SlotField:
45
+ __slots__ = ("factory",)
46
+
47
+ def __init__(self, default_factory: Any) -> None:
48
+ self.factory = default_factory
49
+
50
+
51
+ def slot_field(*, default_factory: Any) -> Any:
52
+ """零依赖 field(default_factory=...) 替代。"""
53
+ return _SlotField(default_factory)
54
+
55
+
56
+ def _class_annotations(cls: Any) -> dict:
57
+ """读类的注解,兼容 PEP 649 惰性注解。
58
+
59
+ `cls.__dict__["__annotations__"]` 在 3.14 的惰性注解下**可能根本不存在**
60
+ (注解由 `__annotate__` 描述子按需计算,不进类字典)——此时取到的是
61
+ `{}`,类会静默变成「零字段」,实例化时报一个与真因毫不相干的
62
+ TypeError。必须走 `getattr` 触发描述子。
63
+ """
64
+ return getattr(cls, "__annotations__", None) or {}
65
+
66
+
67
+ def slot_record(_cls: Any = None, *, frozen: bool = False) -> Any:
68
+ """零依赖 @dataclass 替代(省去 import dataclasses→inspect→tokenize→dis ~8ms)。
69
+
70
+ 挂上 `__dataclass_fields__` 兼容描述子,标准库 `dataclasses.replace` /
71
+ `dataclasses.is_dataclass` / `dataclasses.fields` 对实例照常工作,但热
72
+ 路径自身零导入(`_field_type` 惰性解析,只在 dataclasses 已进场时取哨兵)。
73
+ """
74
+ def _wrap(cls: Any) -> Any:
75
+ ann = list(_class_annotations(cls).keys())
76
+ defaults = {k: cls.__dict__[k] for k in ann if k in cls.__dict__}
77
+
78
+ def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
79
+ if len(args) > len(ann):
80
+ raise TypeError(f"{cls.__name__}.__init__() takes {len(ann)} positional arguments but {len(args)} were given")
81
+ for i, val in enumerate(args):
82
+ object.__setattr__(self, ann[i], val)
83
+ for k in ann[len(args):]:
84
+ if k in kwargs:
85
+ val = kwargs.pop(k)
86
+ elif k in defaults:
87
+ d = defaults[k]
88
+ val = d.factory() if isinstance(d, _SlotField) else d
89
+ else:
90
+ raise TypeError(f"{cls.__name__}.__init__() missing required argument: {k!r}")
91
+ object.__setattr__(self, k, val)
92
+ if kwargs:
93
+ bad = next(iter(kwargs))
94
+ raise TypeError(f"{cls.__name__}.__init__() got an unexpected keyword argument {bad!r}")
95
+
96
+ cls._slot_fields = tuple(ann)
97
+ cls.__dataclass_fields__ = {k: _FieldMeta(k) for k in ann}
98
+ cls.__init__ = __init__
99
+ if frozen:
100
+ def __setattr__(self: Any, name: str, value: Any) -> None:
101
+ raise AttributeError(f"cannot assign to field {name!r}")
102
+ cls.__setattr__ = __setattr__
103
+ return cls
104
+
105
+ return _wrap if _cls is None else _wrap(_cls)
106
+
107
+
108
+ def slot_replace(obj: Any, **changes: Any) -> Any:
109
+ """零依赖 replace(obj, **changes) 替代。"""
110
+ kw = {k: getattr(obj, k) for k in obj._slot_fields}
111
+ kw.update(changes)
112
+ return type(obj)(**kw)
113
+
114
+
115
+ @slot_record
18
116
  class SearchResult:
19
117
  """统一的搜索结果数据类。"""
20
118
  title: str
@@ -22,7 +120,7 @@ class SearchResult:
22
120
  snippet: str = ""
23
121
  score: float = 0.5
24
122
  source: str = ""
25
- metadata: dict = field(default_factory=dict)
123
+ metadata: dict = slot_field(default_factory=dict)
26
124
  social_meta: dict | None = None
27
125
 
28
126
  def to_dict(self) -> dict[str, Any]: