argo-search 2.9.8 → 2.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.en.md +30 -20
  2. package/README.es.md +30 -20
  3. package/README.ja.md +30 -20
  4. package/README.ko.md +30 -20
  5. package/README.md +32 -22
  6. package/SKILL.md +1 -1
  7. package/bin/argo +4 -0
  8. package/bin/argo.js +1 -1
  9. package/config.yaml +52 -5
  10. package/docs/ARGO_INTRO.md +5 -3
  11. package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
  12. package/package.json +3 -2
  13. package/packages/dsh-plugin/README.md +3 -2
  14. package/packages/dsh-plugin/dsh/client.js +98 -32
  15. package/packages/dsh-plugin/dsh/diagnostics.js +43 -0
  16. package/packages/dsh-plugin/dsh/index.js +23 -1
  17. package/packages/dsh-plugin/package.json +2 -2
  18. package/packages/dsh-plugin/test/client.test.js +2 -1
  19. package/scripts/adaptive.py +102 -42
  20. package/scripts/bench_tail_ab.py +281 -0
  21. package/scripts/bounded_run.py +91 -49
  22. package/scripts/clock_bridge.py +22 -0
  23. package/scripts/content_security.py +119 -4
  24. package/scripts/dispatch_result.py +9 -2
  25. package/scripts/engine_add.py +602 -0
  26. package/scripts/engine_dispatch.py +9 -0
  27. package/scripts/evidence_loop.py +4 -0
  28. package/scripts/fetch_quality.py +13 -0
  29. package/scripts/fetch_v3.py +3 -3
  30. package/scripts/http_client.py +24 -38
  31. package/scripts/image_vision/vision_probe +0 -0
  32. package/scripts/local_engines.py +321 -0
  33. package/scripts/matrix_search_eval.py +111 -1
  34. package/scripts/mcp_transport.py +1 -1
  35. package/scripts/net_proxy.py +27 -0
  36. package/scripts/recovery.py +7 -3
  37. package/scripts/route_combo.py +19 -5
  38. package/scripts/route_domains.py +138 -12
  39. package/scripts/safe_conn.py +52 -0
  40. package/scripts/search_entry.py +1 -0
  41. package/scripts/search_pipeline.py +20 -2
  42. package/scripts/search_rank.py +6 -0
  43. package/scripts/serp_guard.py +23 -0
  44. package/scripts/tail_completion.py +212 -0
  45. package/scripts/url_safety.py +126 -3
  46. package/scripts/usage_log.py +45 -0
@@ -47,8 +47,10 @@ _STOPWORDS_EN = {"the", "a", "an", "of", "for", "to", "please", "how", "what"}
47
47
 
48
48
  # 常见「过长尾修饰」:地点/时间/程度等,去掉后主体更聚焦
49
49
  _TAIL_MODIFIERS = re.compile(
50
- r"(最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
51
- r"知乎|贴吧|论坛|大全|汇总|排行榜|一览)")
50
+ r"(?:\b\d{4}年?(?:度)?|\b\d{1,2}月|"
51
+ r"最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
52
+ r"知乎|贴吧|论坛|大全|汇总|排行榜|一览|数据|报告|统计|分析|趋势)"
53
+ )
52
54
 
53
55
  _QUOTE_CHARS = re.compile(r'["\'“”‘’「」『』]')
54
56
  _TOKEN_RE = re.compile(r"[\u4e00-\u9fff]|[a-zA-Z0-9]+")
@@ -119,9 +121,11 @@ def _load_synonyms() -> dict[str, Any]:
119
121
  # ── L1 放宽 ────────────────────────────────────────────────────────────────────
120
122
 
121
123
  def relax_query(query: str) -> str:
122
- """L1:去引号 / 去停用词 / 去过长尾修饰。"""
124
+ """L1:去引号 / 去停用词 / 去过长尾修饰与孤立年份时间词。"""
123
125
  q = _QUOTE_CHARS.sub(" ", query)
124
126
  q = _TAIL_MODIFIERS.sub(" ", q)
127
+ # 清理孤立残留的年月单字
128
+ q = re.sub(r"(?<=\s)[年月](?=\s|$)|^([年月]\s+)", " ", q)
125
129
  # 去停用词(按 token)
126
130
  tokens = re.findall(r"[\u4e00-\u9fffA-Za-z0-9]+", q)
127
131
  kept = []
@@ -502,6 +502,20 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
502
502
  # 学习器在此处首次解析(惰性,见模块顶部 _get_learner):走到这里说明
503
503
  # 真的要装配 combo,这笔钱躲不掉;不进本函数的路径(缓存命中等)不付。
504
504
  _adaptive_learner = _get_learner()
505
+
506
+ # 场景分桶取分(2026-10-24):优先用 (engine, domain, lang) 的桶分,
507
+ # 桶无样本时 adaptive.get_score 自动回退全局分。判据与旧口径一致,
508
+ # 只是多了一个维度——「引擎在 A 域强、B 域弱」现在可被路由区分。
509
+ _dom_name = str(domain.get("name") or "")
510
+ _q_lang = str((features or {}).get("primary_lang") or "")
511
+
512
+ def _score_of(e: str) -> float:
513
+ try:
514
+ return _adaptive_learner.get_score(e, _dom_name, _q_lang)
515
+ except TypeError:
516
+ # 旧版 adaptive(无 domain/lang 形参)回退全局分
517
+ return _adaptive_learner.get_score(e)
518
+
505
519
  combo = domain.get("engines_combo", [])
506
520
  primary = domain.get("primary", "anysearch")
507
521
  fallback = domain.get("fallback")
@@ -581,7 +595,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
581
595
  # 成员与 fallback,自动进入恢复链 L3 兜底。
582
596
  if primary and (
583
597
  _adaptive_learner is None
584
- or _adaptive_learner.get_score(primary) >= _VERTICAL_FLOOR):
598
+ or _score_of(primary) >= _VERTICAL_FLOOR):
585
599
  protect.add(primary)
586
600
  # 仅 modal_card 整 combo 免 cost 裁剪(结构化路径不可被 anysearch 顶替)
587
601
  if domain_name == "modal_card":
@@ -608,14 +622,14 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
608
622
  # 自动回归 combo,且它仍是域 fallback(恢复链 L3 候选)。
609
623
  if domain_name in _VERTICAL_PROTECT:
610
624
  protect = {e for e in original
611
- if _adaptive_learner.get_score(e) >= _VERTICAL_FLOOR
625
+ if _score_of(e) >= _VERTICAL_FLOOR
612
626
  } | protect
613
627
  filtered = [
614
628
  e for e in filtered
615
629
  if e in protect
616
630
  or (e == primary
617
- and _adaptive_learner.get_score(e) >= _VERTICAL_FLOOR)
618
- or _adaptive_learner.get_score(e) >= 0.3
631
+ and _score_of(e) >= _VERTICAL_FLOOR)
632
+ or _score_of(e) >= 0.3
619
633
  ]
620
634
  if not filtered:
621
635
  filtered = original
@@ -662,7 +676,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
662
676
  seen_fam.add(f)
663
677
  members = [x for x in rest if _fam(x) == f]
664
678
  if len(members) > 1:
665
- scored = [(x, _adaptive_learner.get_score(x)) for x in members]
679
+ scored = [(x, _score_of(x)) for x in members]
666
680
  top_score = max(s for _, s in scored)
667
681
  laggards = [x for x, s in scored if top_score - s >= 0.15]
668
682
  if laggards and len(laggards) < len(scored):
@@ -70,10 +70,103 @@ _ZH_LANG_GATED_DOMAINS = frozenset({
70
70
  })
71
71
 
72
72
 
73
- # ── 域预编译(内容指纹缓存) ──────────────────────────────────────────────────
73
+ # ── 域预编译(内容指纹缓存与锚点快筛) ──────────────────────────────────────────
74
74
 
75
75
  _compiled_domains: list[dict[str, Any]] | None = None
76
76
  _compiled_domains_id: tuple | None = None # (name, patterns/exclude_if 内容指纹)
77
+ _compiled_domains_ref: int | None = None
78
+ _anchor_matcher: re.Pattern | None = None
79
+ _anchor_to_domains: dict[str, list[int]] = {}
80
+ _unconstrained_domains: set[int] = set()
81
+
82
+
83
+ def _extract_anchors(src: Any) -> set[str] | None:
84
+ """从正则源码中提取一组必须包含其一的字面锚点词集合(全小写)。
85
+
86
+ 若无法提取安全锚点(如含有纯通配、字符集类 []、通用转义字符类),返回 None。
87
+ """
88
+ if not isinstance(src, str) or not src:
89
+ return None
90
+ s = re.sub(r"^\(\?[a-zA-Z]+\)", "", src)
91
+ s = re.sub(r"\\b", "", s)
92
+ if s.startswith("(") and s.endswith(")"):
93
+ s = s[1:-1]
94
+ parts: list[str] = []
95
+ depth = 0
96
+ cur: list[str] = []
97
+ for ch in s:
98
+ if ch in ("(", "["):
99
+ depth += 1
100
+ elif ch in (")", "]"):
101
+ depth -= 1
102
+ if ch == "|" and depth == 0:
103
+ parts.append("".join(cur))
104
+ cur = []
105
+ else:
106
+ cur.append(ch)
107
+ if cur:
108
+ parts.append("".join(cur))
109
+
110
+ anchors: set[str] = set()
111
+ for part in parts:
112
+ # 正则字符类与元字符无法保证字面子串存在
113
+ if re.search(r"\[.*?\]", part) or re.search(r"\\[wWdDsS]", part):
114
+ return None
115
+ runs = re.findall(r"[\u4e00-\u9fff]{1,}|[a-zA-Z0-9_\-\.]{2,}", part)
116
+ if not runs:
117
+ return None
118
+ for r in runs:
119
+ anchors.add(r.lower())
120
+ return anchors if anchors else None
121
+
122
+
123
+ def _init_anchor_index(compiled: list[dict[str, Any]]) -> None:
124
+ """初始化全局锚点自动机索引(方案一:Aho-Corasick 前置快速分桶)。"""
125
+ global _anchor_matcher, _anchor_to_domains, _unconstrained_domains
126
+ _anchor_to_domains = {}
127
+ _unconstrained_domains = set()
128
+ all_anchors: set[str] = set()
129
+
130
+ for idx, domain in enumerate(compiled):
131
+ entries = _pattern_entries(domain)
132
+ if not entries:
133
+ _unconstrained_domains.add(idx)
134
+ continue
135
+ dom_anchors: set[str] = set()
136
+ unconstrained = False
137
+ for p in entries:
138
+ match_src, _ = _split_pattern_entry(p)
139
+ anc = _extract_anchors(match_src)
140
+ if anc is None:
141
+ unconstrained = True
142
+ break
143
+ dom_anchors.update(anc)
144
+ if unconstrained or not dom_anchors:
145
+ _unconstrained_domains.add(idx)
146
+ else:
147
+ for term in dom_anchors:
148
+ _anchor_to_domains.setdefault(term, []).append(idx)
149
+ all_anchors.add(term)
150
+
151
+ if all_anchors:
152
+ sorted_terms = sorted(all_anchors, key=len, reverse=True)
153
+ escaped = [re.escape(t) for t in sorted_terms]
154
+ _anchor_matcher = re.compile("|".join(escaped), re.IGNORECASE)
155
+ else:
156
+ _anchor_matcher = None
157
+
158
+
159
+ def _get_candidate_domain_indices(query: str) -> set[int] | None:
160
+ """通过锚点词快速扫描返回候选域索引集合;无索引时返回 None。"""
161
+ if _anchor_matcher is None:
162
+ return None
163
+ candidates = set(_unconstrained_domains)
164
+ for m in _anchor_matcher.finditer(query):
165
+ term = m.group(0).lower()
166
+ doms = _anchor_to_domains.get(term)
167
+ if doms:
168
+ candidates.update(doms)
169
+ return candidates
77
170
 
78
171
 
79
172
  def _ensure_domain_compiled(domain: dict[str, Any]) -> list[tuple[re.Pattern, re.Pattern | None]]:
@@ -134,18 +227,30 @@ def _split_pattern_entry(entry: Any) -> tuple[Any, Any]:
134
227
  return entry, None
135
228
 
136
229
 
137
- def _domain_fires(domain: dict[str, Any], query: str) -> bool:
138
- """一个域是否成立:任一触发词命中,且它自己的否决词没命中。"""
230
+ def _domain_fires_with_span(domain: dict[str, Any], query: str) -> tuple[bool, int, int]:
231
+ """一个域是否成立,并返回命中的 (是否成立, match_span_start, match_span_len)。"""
139
232
  for match_re, unless_re in _ensure_domain_compiled(domain):
140
233
  if unless_re is not None and unless_re.search(query):
141
234
  continue
142
- if match_re.search(query):
143
- return True
144
- return False
235
+ m = match_re.search(query)
236
+ if m is not None:
237
+ start, end = m.span()
238
+ return True, start, end - start
239
+ return False, 0, 0
240
+
241
+
242
+ def _domain_fires(domain: dict[str, Any], query: str) -> bool:
243
+ """一个域是否成立:任一触发词命中,且它自己的否决词没命中。"""
244
+ fires, _, _ = _domain_fires_with_span(domain, query)
245
+ return fires
145
246
 
146
247
 
147
248
  def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]:
148
- global _compiled_domains, _compiled_domains_id
249
+ global _compiled_domains, _compiled_domains_id, _compiled_domains_ref
250
+ # 快速指针比对:当传入同一 domains 引用且已编译时 O(1) 返回,免除 97 域的 SHA1 开销
251
+ dom_ref = id(domains)
252
+ if _compiled_domains is not None and _compiled_domains_ref == dom_ref:
253
+ return _compiled_domains
149
254
  # 用内容指纹替代 id(domains),避免 GC 后 id 复用导致缓存失效/错误。
150
255
  # 指纹含规则本体(只记长度会漏「改正则不改条数」的编辑),且必须覆盖
151
256
  # exclude_if / intent_required——它们和 patterns 一样会改变命中结果。
@@ -169,6 +274,8 @@ def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]
169
274
  for idx, domain in enumerate(domains)
170
275
  ]
171
276
  _compiled_domains_id = dom_fp
277
+ _compiled_domains_ref = dom_ref
278
+ _init_anchor_index(_compiled_domains)
172
279
  return _compiled_domains
173
280
 
174
281
 
@@ -311,20 +418,39 @@ def match_domains(query: str, domains: list[dict[str, Any]] | None = None,
311
418
  domains = get_domains()
312
419
  compiled = _get_compiled_domains(domains)
313
420
  hits: list[dict[str, Any]] = []
421
+ secondary_hits: list[tuple[dict[str, Any], float]] = []
314
422
  catch_all: dict[str, Any] | None = None
315
423
  non_zh = bool(primary_lang and primary_lang not in ("zh", "mixed", "other"))
316
- for domain in compiled:
424
+ candidates = _get_candidate_domain_indices(query)
425
+ q_len = max(len(query), 1)
426
+
427
+ for idx, domain in enumerate(compiled):
317
428
  if not domain.get("patterns", []):
318
429
  catch_all = domain
319
430
  continue
320
431
  if non_zh and domain.get("name") in _ZH_LANG_GATED_DOMAINS:
321
432
  continue
322
- if _domain_fires(domain, query):
323
- hits.append(domain)
324
- if len(hits) >= max_n:
325
- break
433
+ if candidates is not None and idx not in candidates:
434
+ continue
435
+ fires, span_start, span_len = _domain_fires_with_span(domain, query)
436
+ if fires:
437
+ if not hits:
438
+ hits.append(domain)
439
+ else:
440
+ pos_ratio = span_start / q_len
441
+ # 方案二:跨度高(特异性强)优先提拔;同跨度下严格保持前序特化域顺位稳定
442
+ score = (span_len * 2.0) - (idx * 0.1) + (pos_ratio * 0.01)
443
+ secondary_hits.append((domain, score))
444
+ if len(hits) + len(secondary_hits) >= max_n:
445
+ break
446
+
447
+ if secondary_hits:
448
+ secondary_hits.sort(key=lambda x: x[1], reverse=True)
449
+ hits.extend(h[0] for h in secondary_hits)
450
+
326
451
  hits = _social_domain_first(hits)
327
452
  hits = _intent_gate(hits, query)
453
+ hits = hits[:max_n]
328
454
  return hits if hits else ([catch_all] if catch_all else [])
329
455
 
330
456
 
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env python3
2
+ """safe_conn.py — 解析即锁定的 HTTP/HTTPS 连接类(DNS rebinding / TOCTOU 防护)
3
+
4
+ 独立成模块的理由:``http.client`` 会连带拉起 ``ssl``(实测 ≈14 ms)。
5
+ ``url_safety`` 被 ``robots_guard`` 等在**模块级**导入(见 robots_guard 的
6
+ 「轻导入」注释),若把 ``http.client`` 放在 ``url_safety`` 里,
7
+ ``test_robots_guard.test_module_import_stays_light`` 会红。连接类只在真正
8
+ 建连时经 ``net_proxy.open_connection`` 导入,不进轻导入路径。
9
+
10
+ 原理:``check_url()`` 在请求前解析一次 DNS,``http.client`` 在 ``connect()``
11
+ 时再解析一次——两次之间域名可改指内网(DNS rebinding)。本类在 ``connect()``
12
+ 里解析一次、过滤私网、把结果钉给本次 TCP 连接,窗口关闭;SNI/Host 仍用原域名。
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import http.client
18
+
19
+ from url_safety import pin_host
20
+
21
+
22
+ class SafeHTTPConnection(http.client.HTTPConnection):
23
+ """解析即锁定的 HTTP 连接:TCP 只连校验过的公网 IP,Host 头仍用原域名。"""
24
+
25
+ def connect(self):
26
+ pinned = pin_host(self.host)
27
+ if pinned is None:
28
+ return super().connect()
29
+ host = self.host
30
+ self.host = pinned
31
+ try:
32
+ super().connect()
33
+ finally:
34
+ self.host = host
35
+
36
+
37
+ class SafeHTTPSConnection(http.client.HTTPSConnection):
38
+ """解析即锁定的 HTTPS 连接:TCP 钉公网 IP,SNI/证书校验仍用原域名。"""
39
+
40
+ def connect(self):
41
+ pinned = pin_host(self.host)
42
+ if pinned is None:
43
+ return super().connect()
44
+ host = self.host
45
+ self.host = pinned
46
+ try:
47
+ http.client.HTTPConnection.connect(self) # 仅 TCP
48
+ finally:
49
+ self.host = host
50
+ server_hostname = self._tunnel_host or host
51
+ self.sock = self._context.wrap_socket(
52
+ self.sock, server_hostname=server_hostname)
@@ -392,4 +392,5 @@ def dispatch(req: _SearchRequest, run: _SearchRun, hooks: _SearchHooks) -> _Sear
392
392
  quota_batch=quota_batch,
393
393
  run_one=_run_one,
394
394
  ingest=_ingest,
395
+ stranded=_dispatch.stranded,
395
396
  )
@@ -156,6 +156,9 @@ class _SearchRun:
156
156
  useful_ms: int = 0
157
157
  dispatch_tail: dict[str, int] = field(default_factory=dict)
158
158
  early_stopped: bool = False
159
+ # 早停弃置且仍存活的 (engine, holder, now_fn):finalize 交给
160
+ # tail_completion 后台收割。纯数据字段,不改变任何既有阶段的行为。
161
+ stranded: tuple = ()
159
162
  budget_used_ms: int | None = None
160
163
  budget_total_ms: int | None = None
161
164
  elapsed: int = 0
@@ -772,7 +775,7 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
772
775
  # 自适应学习(单事务批量写入 + cache_guard 统一状态态过滤)
773
776
  try:
774
777
  import contextlib
775
- from adaptive import get_learner
778
+ from adaptive import get_learner, scenario_of
776
779
  learner = get_learner()
777
780
  _outcome_of = {o.get("engine"): o
778
781
  for o in engine_outcomes if isinstance(o, dict)}
@@ -797,12 +800,27 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
797
800
  quality = None
798
801
  learner.record(eng, success=success, latency_ms=latency,
799
802
  cost=0.0 if cost >= 0.85 else 0.001, empty=empty,
800
- quality=quality)
803
+ quality=quality, **scenario_of(decision))
801
804
  except ImportError:
802
805
  pass
803
806
  except Exception as e:
804
807
  _log(f"自适应学习记录跳过: {type(e).__name__}")
805
808
 
809
+ # 早停后台补全(2026-10-25):弃置引擎的交付不再丢弃,宽限内完成的
810
+ # 记学习器真实成绩 + 合并写 L2 缓存(ARGO_TAIL_COMPLETE_S 控制,0=关)。
811
+ # 放在缓存写与学习记账之后:补全读的是刚写的主缓存条目;主响应已定,
812
+ # 这里的一切都是锦上添花,异常 fail-open。
813
+ if run.stranded:
814
+ try:
815
+ from tail_completion import hand_off
816
+ hand_off(run.stranded, query=query,
817
+ cache_engine_key=cache_engine_key,
818
+ max_results=max_results, domain=domain,
819
+ mode=mode, depth=depth, decision=decision,
820
+ skip_cache=skip_cache, cache_instance=cache)
821
+ except Exception as e:
822
+ _log(f"后台补全移交跳过: {type(e).__name__}")
823
+
806
824
  # 语言偏好:记录本轮查询语 + 输出观测快照(默认中英 + 系统 + 习惯)
807
825
  lang_pref_info: dict[str, Any] | None = None
808
826
  try:
@@ -96,6 +96,12 @@ _ENGINE_FUSION_WEIGHTS: dict[str, float] = {
96
96
  "stackoverflow": 1.1, "imdb": 1.2, "thesportsdb": 1.2, "itunes": 1.2,
97
97
  "finviz": 1.2, "sina_quote": 1.2, "tencent_quote": 1.2, "eastmoney": 1.2,
98
98
  "fred": 1.3, "worldbank": 1.3, "nbs_stats": 1.3, "eurostat": 1.3, "fx_rate": 1.2, "frankfurter": 1.2,
99
+ # 2026-10-25 补表:学术聚合与官方一手数据此前表外=1.0 基线。实证缺口:
100
+ # 学术域查询常由 core 独家交付(当日实测组合里 arxiv/local_pubmed 零结果、
101
+ # 只 core 有货)却拿基线权重;官方 open data / 法律 / 标准系同理。
102
+ "core": 1.3, "openreview": 1.3, "zenodo": 1.2, "datacite": 1.2,
103
+ "sec_edgar": 1.3, "federal_register": 1.3, "eu_opendata": 1.3,
104
+ "egov_law": 1.3, "openstd": 1.3, "who_gho": 1.3, "usgs": 1.3, "nasa_cmr": 1.3,
99
105
  # 通用引擎(基线)
100
106
  "duckduckgo": 1.0, "local_bing": 1.0, "local_duckduckgo": 1.0,
101
107
  "local_google": 1.0, "anysearch": 1.05, "byted": 1.1, "bocha": 1.0,
@@ -334,3 +334,26 @@ def filter_content_farms(results: list) -> list:
334
334
  kept.append(r)
335
335
  return kept
336
336
 
337
+
338
+ # ── 自适应放宽建议(2026-10-23 方案四)───────────────────────────────────
339
+
340
+ _RELAX_TAIL_RE = re.compile(
341
+ r"(?:\b\d{4}年?(?:度)?|\b\d{1,2}月|"
342
+ r"最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
343
+ r"知乎|贴吧|论坛|大全|汇总|排行榜|一览|数据|报告|统计|分析|趋势)"
344
+ )
345
+
346
+
347
+ def suggest_relaxed_query(query: str) -> str:
348
+ """针对 SERP 守卫击杀场景的自适应放宽查询(剥离过度约束的修饰与年份词)。"""
349
+ if not query or not isinstance(query, str):
350
+ return ""
351
+ q = _RELAX_TAIL_RE.sub(" ", query)
352
+ q = re.sub(r"(?<=\s)[年月](?=\s|$)|^([年月]\s+)", " ", q)
353
+ tokens = re.findall(r"[\u4e00-\u9fffA-Za-z0-9]+", q)
354
+ stopwords = {"的", "了", "呢", "吧", "请问", "怎么", "如何", "一下", "关于", "有关", "求", "谁知道",
355
+ "the", "a", "an", "of", "for", "to", "how", "what"}
356
+ kept = [t for t in tokens if t.lower() not in stopwords]
357
+ relaxed = " ".join(kept).strip()
358
+ return relaxed or re.sub(r"\s+", " ", q).strip() or query
359
+
@@ -0,0 +1,212 @@
1
+ """tail_completion.py — 早停后台补全:慢源交付从「丢弃」变「补齐」。
2
+
3
+ ## 为什么(2026-10-25)
4
+
5
+ 七天窗口 1185 条真实查询里 74% 发生 early-stop。被放弃的交付不是低质货:
6
+ 离线回放(bench_tail_ab --quality)显示被放弃交付 81% 来自 anysearch,相关度
7
+ 代理比全库均线高 0.125;五条对照查询里四条「唯一有结果的就是慢源」(证据与
8
+ 口径见 docs/QUALITY_OF_SLOW_SOURCES.md)。早停省的是**用户等待**,不该连
9
+ **交付**一起丢——那是在拿准确性换速度。缩尾宽已被实证否决,本模块是拍板的
10
+ 正确方向:慢源不拖住快源(早交付),但慢源的交付进缓存(后台补全)。
11
+
12
+ ## 做什么
13
+
14
+ engine_dispatch 在早停弃置引擎时把 (engine, holder, now_fn) 交给本模块——
15
+ 主响应与既有记账逐位不变(cancelled outcome / abandoned_ms 照记)。本模块
16
+ 起一个 daemon 收割线程,在主流程返回后继续等被弃置的引擎,宽限内完成的:
17
+
18
+ 1. 学习器记真实成绩(success / latency / quality / 场景分桶标签)——此前
19
+ cancelled 被 cache_guard.should_record_adaptive 跳过,慢源的真实表现
20
+ 从没进过学习器,场景分桶(engine×domain×lang)也少一笔真数据;
21
+ 2. 把交付合并写回 L2 combo 缓存(URL 去重、新条目追加、缺省 score 用词面
22
+ 相关度)——同查询第二次免费拿到「快源+慢源」的全量覆盖。
23
+
24
+ timeout 弃置不收:那是引擎确实超窗(甚至卡死),补全等于鼓励慢源;
25
+ cancelled 是「编排收工决定,引擎没失败」——两者语义不同(见
26
+ engine_dispatch._settle_pending 的注释)。
27
+
28
+ ## 进程语义
29
+
30
+ 收割线程是 daemon;CLI 进程退出前由 atexit 宽限等待(hand_off 首次调用时
31
+ 注册,无 stranded 的进程零成本、不注册)。MCP/daemon 常驻进程不退出,
32
+ 自然收割全部。宽限从**入队时刻**起算整批(job 级 deadline),不做逐引擎
33
+ 累加——实测被放弃交付从早停时刻起的剩余等待 p50 2306ms / p90 3883ms。
34
+
35
+ ## 开关
36
+
37
+ ARGO_TAIL_COMPLETE_S:宽限秒数(默认 3.0;0=禁用整个机制,行为回到
38
+ 纯丢弃)。改动记得同步 references/usage.md 的开关总表(usage_doc_gates)。
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import atexit
44
+ import os
45
+ import threading
46
+
47
+ _LOCK = threading.Lock()
48
+ _JOBS: list[dict] = []
49
+ _WORKER: threading.Thread | None = None
50
+ _ATEXIT_REGISTERED = False
51
+ _STATS = {"salvaged": 0, "expired": 0, "recorded": 0}
52
+ _POLL_INTERVAL_S = 0.05
53
+
54
+
55
+ def grace_seconds() -> float:
56
+ """收割宽限(ARGO_TAIL_COMPLETE_S,默认 3.0;<=0 禁用)。"""
57
+ try:
58
+ return float(os.environ.get("ARGO_TAIL_COMPLETE_S", "3.0") or 3.0)
59
+ except ValueError:
60
+ return 3.0
61
+
62
+
63
+ def hand_off(stranded, *, query, cache_engine_key, max_results,
64
+ domain, mode, depth, decision, skip_cache=False,
65
+ cache_instance=None) -> None:
66
+ """把早停弃置的引擎交给后台收割。
67
+
68
+ ``stranded`` 是 engine_dispatch 产出的 ``[(engine, holder, now_fn), ...]``:
69
+ holder["r"] 出现即完成(与 _ingest_holder 同判据),now_fn 用于算真实
70
+ latency(dispatch 可能跑在单调钟上,收割线程不能用墙钟去减)。
71
+ ``cache_instance`` 是本次请求的缓存实例(req.cache):补全读写的必须是
72
+ 同一实例(L1 内存态 + L2 落盘一致),模块级没有全局缓存可拿。
73
+ 其余参数与 finalize 的 cache.set / learner.record 同源同口径。
74
+ 任何异常 fail-open:补全失败只丢「锦上添花」,不影响主响应。
75
+ """
76
+ grace = grace_seconds()
77
+ if grace <= 0 or not stranded:
78
+ return
79
+ global _ATEXIT_REGISTERED
80
+ with _LOCK:
81
+ if not _ATEXIT_REGISTERED:
82
+ atexit.register(wait_for_completion)
83
+ _ATEXIT_REGISTERED = True
84
+ _JOBS.append({
85
+ "stranded": list(stranded),
86
+ "deadline": _now_wall() + grace,
87
+ "query": query, "cache_engine_key": cache_engine_key,
88
+ "max_results": max_results, "domain": domain,
89
+ "mode": mode, "depth": depth, "decision": decision or {},
90
+ "skip_cache": bool(skip_cache) or cache_instance is None,
91
+ "cache": cache_instance,
92
+ })
93
+ _start_worker_locked()
94
+
95
+
96
+ def wait_for_completion(timeout_s: float | None = None) -> None:
97
+ """等收割线程收尾(atexit 出口 / 测试用)。宽限耗尽即返回,不无限等。"""
98
+ t = _WORKER
99
+ if t is not None and t.is_alive():
100
+ budget = timeout_s if timeout_s is not None else grace_seconds() + 0.5
101
+ t.join(timeout=budget)
102
+
103
+
104
+ def stats() -> dict:
105
+ """收割计数(可观测口,供诊断;不改任何行为)。"""
106
+ with _LOCK:
107
+ return dict(_STATS)
108
+
109
+
110
+ def _now_wall() -> float:
111
+ import time
112
+ return time.time()
113
+
114
+
115
+ def _start_worker_locked() -> None:
116
+ global _WORKER
117
+ if _WORKER is not None and _WORKER.is_alive():
118
+ return
119
+ _WORKER = threading.Thread(target=_worker, daemon=True,
120
+ name="argo-tail-completion")
121
+ _WORKER.start()
122
+
123
+
124
+ def _worker() -> None:
125
+ """串行收割队列:宽限内完成的记学习器+合并缓存,超宽限标 expired。"""
126
+ import time
127
+ while True:
128
+ with _LOCK:
129
+ if not _JOBS:
130
+ return
131
+ job = _JOBS.pop(0)
132
+ for eng, holder, now_fn in job["stranded"]:
133
+ remaining = job["deadline"] - _now_wall()
134
+ while holder.get("r") is None and remaining > 0:
135
+ time.sleep(min(_POLL_INTERVAL_S, remaining))
136
+ remaining = job["deadline"] - _now_wall()
137
+ res = (holder.get("r") or (None,))[1] if holder.get("r") else None
138
+ if res is None:
139
+ with _LOCK:
140
+ _STATS["expired"] += 1
141
+ continue
142
+ goods = [r for r in (res or [])
143
+ if isinstance(r, dict) and "error" not in r]
144
+ with _LOCK:
145
+ _STATS["salvaged"] += 1
146
+ try:
147
+ _record_adaptive(eng, goods, holder, now_fn, job)
148
+ except Exception:
149
+ pass
150
+ if not job["skip_cache"]:
151
+ try:
152
+ _merge_into_cache(eng, goods, job)
153
+ except Exception:
154
+ pass
155
+
156
+
157
+ def _record_adaptive(eng, goods, holder, now_fn, job) -> None:
158
+ """学习器记真实成绩(口径与 search_pipeline.finalize 对齐)。"""
159
+ from adaptive import get_learner, scenario_of
160
+ from query_signals import result_relevance
161
+ success = bool(goods)
162
+ try:
163
+ lat_ms = float((now_fn() - holder.get("t0", now_fn())) * 1000.0)
164
+ except Exception:
165
+ lat_ms = 0.0
166
+ quality = None
167
+ if success:
168
+ try:
169
+ quality = result_relevance(job["query"], goods)
170
+ except Exception:
171
+ quality = None
172
+ with _LOCK:
173
+ _STATS["recorded"] += 1
174
+ get_learner().record(eng, success=success, latency_ms=max(0.0, lat_ms),
175
+ cost=0.001, empty=not success, quality=quality,
176
+ **scenario_of(job["decision"]))
177
+
178
+
179
+ def _merge_into_cache(eng, goods, job) -> None:
180
+ """交付合并写回 L2 combo 缓存:读现条目 → URL 去重追加 → 同 key 覆盖。
181
+
182
+ 只在已有缓存条目时合并(本机制补的是「快源已写的那份」);主缓存被守卫
183
+ 拒写时这里不硬造。score 缺省用词面相关度,保证追加条目有可排序的分数。
184
+ """
185
+ from query_signals import result_relevance
186
+ cache = job["cache"]
187
+ cur = cache.get(job["query"], job["cache_engine_key"],
188
+ job["max_results"], domain=job["domain"],
189
+ mode=job["mode"], depth=job["depth"])
190
+ if not cur or not isinstance(cur.get("results"), list) or not goods:
191
+ return
192
+ existing = {r.get("url") for r in cur["results"]
193
+ if isinstance(r, dict)}
194
+ new: list[dict] = []
195
+ for r in goods:
196
+ if not isinstance(r, dict) or r.get("url") in existing:
197
+ continue
198
+ item = dict(r)
199
+ if "score" not in item:
200
+ try:
201
+ item["score"] = round(float(result_relevance(job["query"], [r])), 4)
202
+ except Exception:
203
+ item["score"] = 0.3
204
+ new.append(item)
205
+ if not new:
206
+ return
207
+ payload = dict(cur)
208
+ payload["results"] = list(cur["results"]) + new
209
+ payload["engines_used"] = sorted({*(cur.get("engines_used") or []), eng})
210
+ cache.set(job["query"], job["cache_engine_key"], job["max_results"],
211
+ payload, domain=job["domain"], mode=job["mode"],
212
+ depth=job["depth"])