argo-search 2.9.8 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +30 -20
- package/README.es.md +30 -20
- package/README.ja.md +30 -20
- package/README.ko.md +30 -20
- package/README.md +32 -22
- package/SKILL.md +1 -1
- package/bin/argo +4 -0
- package/bin/argo.js +1 -1
- package/config.yaml +52 -5
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/package.json +3 -2
- package/packages/dsh-plugin/README.md +3 -2
- package/packages/dsh-plugin/dsh/client.js +98 -32
- package/packages/dsh-plugin/dsh/diagnostics.js +43 -0
- package/packages/dsh-plugin/dsh/index.js +23 -1
- package/packages/dsh-plugin/package.json +2 -2
- package/packages/dsh-plugin/test/client.test.js +2 -1
- package/scripts/adaptive.py +102 -42
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_security.py +119 -4
- package/scripts/dispatch_result.py +9 -2
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +9 -0
- package/scripts/evidence_loop.py +4 -0
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +3 -3
- package/scripts/http_client.py +24 -38
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/matrix_search_eval.py +111 -1
- package/scripts/mcp_transport.py +1 -1
- package/scripts/net_proxy.py +27 -0
- package/scripts/recovery.py +7 -3
- package/scripts/route_combo.py +19 -5
- package/scripts/route_domains.py +138 -12
- package/scripts/safe_conn.py +52 -0
- package/scripts/search_entry.py +1 -0
- package/scripts/search_pipeline.py +20 -2
- package/scripts/search_rank.py +6 -0
- package/scripts/serp_guard.py +23 -0
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
package/scripts/recovery.py
CHANGED
|
@@ -47,8 +47,10 @@ _STOPWORDS_EN = {"the", "a", "an", "of", "for", "to", "please", "how", "what"}
|
|
|
47
47
|
|
|
48
48
|
# 常见「过长尾修饰」:地点/时间/程度等,去掉后主体更聚焦
|
|
49
49
|
_TAIL_MODIFIERS = re.compile(
|
|
50
|
-
r"(
|
|
51
|
-
r"
|
|
50
|
+
r"(?:\b\d{4}年?(?:度)?|\b\d{1,2}月|"
|
|
51
|
+
r"最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
|
|
52
|
+
r"知乎|贴吧|论坛|大全|汇总|排行榜|一览|数据|报告|统计|分析|趋势)"
|
|
53
|
+
)
|
|
52
54
|
|
|
53
55
|
_QUOTE_CHARS = re.compile(r'["\'“”‘’「」『』]')
|
|
54
56
|
_TOKEN_RE = re.compile(r"[\u4e00-\u9fff]|[a-zA-Z0-9]+")
|
|
@@ -119,9 +121,11 @@ def _load_synonyms() -> dict[str, Any]:
|
|
|
119
121
|
# ── L1 放宽 ────────────────────────────────────────────────────────────────────
|
|
120
122
|
|
|
121
123
|
def relax_query(query: str) -> str:
|
|
122
|
-
"""L1:去引号 / 去停用词 /
|
|
124
|
+
"""L1:去引号 / 去停用词 / 去过长尾修饰与孤立年份时间词。"""
|
|
123
125
|
q = _QUOTE_CHARS.sub(" ", query)
|
|
124
126
|
q = _TAIL_MODIFIERS.sub(" ", q)
|
|
127
|
+
# 清理孤立残留的年月单字
|
|
128
|
+
q = re.sub(r"(?<=\s)[年月](?=\s|$)|^([年月]\s+)", " ", q)
|
|
125
129
|
# 去停用词(按 token)
|
|
126
130
|
tokens = re.findall(r"[\u4e00-\u9fffA-Za-z0-9]+", q)
|
|
127
131
|
kept = []
|
package/scripts/route_combo.py
CHANGED
|
@@ -502,6 +502,20 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
502
502
|
# 学习器在此处首次解析(惰性,见模块顶部 _get_learner):走到这里说明
|
|
503
503
|
# 真的要装配 combo,这笔钱躲不掉;不进本函数的路径(缓存命中等)不付。
|
|
504
504
|
_adaptive_learner = _get_learner()
|
|
505
|
+
|
|
506
|
+
# 场景分桶取分(2026-10-24):优先用 (engine, domain, lang) 的桶分,
|
|
507
|
+
# 桶无样本时 adaptive.get_score 自动回退全局分。判据与旧口径一致,
|
|
508
|
+
# 只是多了一个维度——「引擎在 A 域强、B 域弱」现在可被路由区分。
|
|
509
|
+
_dom_name = str(domain.get("name") or "")
|
|
510
|
+
_q_lang = str((features or {}).get("primary_lang") or "")
|
|
511
|
+
|
|
512
|
+
def _score_of(e: str) -> float:
|
|
513
|
+
try:
|
|
514
|
+
return _adaptive_learner.get_score(e, _dom_name, _q_lang)
|
|
515
|
+
except TypeError:
|
|
516
|
+
# 旧版 adaptive(无 domain/lang 形参)回退全局分
|
|
517
|
+
return _adaptive_learner.get_score(e)
|
|
518
|
+
|
|
505
519
|
combo = domain.get("engines_combo", [])
|
|
506
520
|
primary = domain.get("primary", "anysearch")
|
|
507
521
|
fallback = domain.get("fallback")
|
|
@@ -581,7 +595,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
581
595
|
# 成员与 fallback,自动进入恢复链 L3 兜底。
|
|
582
596
|
if primary and (
|
|
583
597
|
_adaptive_learner is None
|
|
584
|
-
or
|
|
598
|
+
or _score_of(primary) >= _VERTICAL_FLOOR):
|
|
585
599
|
protect.add(primary)
|
|
586
600
|
# 仅 modal_card 整 combo 免 cost 裁剪(结构化路径不可被 anysearch 顶替)
|
|
587
601
|
if domain_name == "modal_card":
|
|
@@ -608,14 +622,14 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
608
622
|
# 自动回归 combo,且它仍是域 fallback(恢复链 L3 候选)。
|
|
609
623
|
if domain_name in _VERTICAL_PROTECT:
|
|
610
624
|
protect = {e for e in original
|
|
611
|
-
if
|
|
625
|
+
if _score_of(e) >= _VERTICAL_FLOOR
|
|
612
626
|
} | protect
|
|
613
627
|
filtered = [
|
|
614
628
|
e for e in filtered
|
|
615
629
|
if e in protect
|
|
616
630
|
or (e == primary
|
|
617
|
-
and
|
|
618
|
-
or
|
|
631
|
+
and _score_of(e) >= _VERTICAL_FLOOR)
|
|
632
|
+
or _score_of(e) >= 0.3
|
|
619
633
|
]
|
|
620
634
|
if not filtered:
|
|
621
635
|
filtered = original
|
|
@@ -662,7 +676,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
662
676
|
seen_fam.add(f)
|
|
663
677
|
members = [x for x in rest if _fam(x) == f]
|
|
664
678
|
if len(members) > 1:
|
|
665
|
-
scored = [(x,
|
|
679
|
+
scored = [(x, _score_of(x)) for x in members]
|
|
666
680
|
top_score = max(s for _, s in scored)
|
|
667
681
|
laggards = [x for x, s in scored if top_score - s >= 0.15]
|
|
668
682
|
if laggards and len(laggards) < len(scored):
|
package/scripts/route_domains.py
CHANGED
|
@@ -70,10 +70,103 @@ _ZH_LANG_GATED_DOMAINS = frozenset({
|
|
|
70
70
|
})
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
# ──
|
|
73
|
+
# ── 域预编译(内容指纹缓存与锚点快筛) ──────────────────────────────────────────
|
|
74
74
|
|
|
75
75
|
_compiled_domains: list[dict[str, Any]] | None = None
|
|
76
76
|
_compiled_domains_id: tuple | None = None # (name, patterns/exclude_if 内容指纹)
|
|
77
|
+
_compiled_domains_ref: int | None = None
|
|
78
|
+
_anchor_matcher: re.Pattern | None = None
|
|
79
|
+
_anchor_to_domains: dict[str, list[int]] = {}
|
|
80
|
+
_unconstrained_domains: set[int] = set()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _extract_anchors(src: Any) -> set[str] | None:
|
|
84
|
+
"""从正则源码中提取一组必须包含其一的字面锚点词集合(全小写)。
|
|
85
|
+
|
|
86
|
+
若无法提取安全锚点(如含有纯通配、字符集类 []、通用转义字符类),返回 None。
|
|
87
|
+
"""
|
|
88
|
+
if not isinstance(src, str) or not src:
|
|
89
|
+
return None
|
|
90
|
+
s = re.sub(r"^\(\?[a-zA-Z]+\)", "", src)
|
|
91
|
+
s = re.sub(r"\\b", "", s)
|
|
92
|
+
if s.startswith("(") and s.endswith(")"):
|
|
93
|
+
s = s[1:-1]
|
|
94
|
+
parts: list[str] = []
|
|
95
|
+
depth = 0
|
|
96
|
+
cur: list[str] = []
|
|
97
|
+
for ch in s:
|
|
98
|
+
if ch in ("(", "["):
|
|
99
|
+
depth += 1
|
|
100
|
+
elif ch in (")", "]"):
|
|
101
|
+
depth -= 1
|
|
102
|
+
if ch == "|" and depth == 0:
|
|
103
|
+
parts.append("".join(cur))
|
|
104
|
+
cur = []
|
|
105
|
+
else:
|
|
106
|
+
cur.append(ch)
|
|
107
|
+
if cur:
|
|
108
|
+
parts.append("".join(cur))
|
|
109
|
+
|
|
110
|
+
anchors: set[str] = set()
|
|
111
|
+
for part in parts:
|
|
112
|
+
# 正则字符类与元字符无法保证字面子串存在
|
|
113
|
+
if re.search(r"\[.*?\]", part) or re.search(r"\\[wWdDsS]", part):
|
|
114
|
+
return None
|
|
115
|
+
runs = re.findall(r"[\u4e00-\u9fff]{1,}|[a-zA-Z0-9_\-\.]{2,}", part)
|
|
116
|
+
if not runs:
|
|
117
|
+
return None
|
|
118
|
+
for r in runs:
|
|
119
|
+
anchors.add(r.lower())
|
|
120
|
+
return anchors if anchors else None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _init_anchor_index(compiled: list[dict[str, Any]]) -> None:
|
|
124
|
+
"""初始化全局锚点自动机索引(方案一:Aho-Corasick 前置快速分桶)。"""
|
|
125
|
+
global _anchor_matcher, _anchor_to_domains, _unconstrained_domains
|
|
126
|
+
_anchor_to_domains = {}
|
|
127
|
+
_unconstrained_domains = set()
|
|
128
|
+
all_anchors: set[str] = set()
|
|
129
|
+
|
|
130
|
+
for idx, domain in enumerate(compiled):
|
|
131
|
+
entries = _pattern_entries(domain)
|
|
132
|
+
if not entries:
|
|
133
|
+
_unconstrained_domains.add(idx)
|
|
134
|
+
continue
|
|
135
|
+
dom_anchors: set[str] = set()
|
|
136
|
+
unconstrained = False
|
|
137
|
+
for p in entries:
|
|
138
|
+
match_src, _ = _split_pattern_entry(p)
|
|
139
|
+
anc = _extract_anchors(match_src)
|
|
140
|
+
if anc is None:
|
|
141
|
+
unconstrained = True
|
|
142
|
+
break
|
|
143
|
+
dom_anchors.update(anc)
|
|
144
|
+
if unconstrained or not dom_anchors:
|
|
145
|
+
_unconstrained_domains.add(idx)
|
|
146
|
+
else:
|
|
147
|
+
for term in dom_anchors:
|
|
148
|
+
_anchor_to_domains.setdefault(term, []).append(idx)
|
|
149
|
+
all_anchors.add(term)
|
|
150
|
+
|
|
151
|
+
if all_anchors:
|
|
152
|
+
sorted_terms = sorted(all_anchors, key=len, reverse=True)
|
|
153
|
+
escaped = [re.escape(t) for t in sorted_terms]
|
|
154
|
+
_anchor_matcher = re.compile("|".join(escaped), re.IGNORECASE)
|
|
155
|
+
else:
|
|
156
|
+
_anchor_matcher = None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _get_candidate_domain_indices(query: str) -> set[int] | None:
|
|
160
|
+
"""通过锚点词快速扫描返回候选域索引集合;无索引时返回 None。"""
|
|
161
|
+
if _anchor_matcher is None:
|
|
162
|
+
return None
|
|
163
|
+
candidates = set(_unconstrained_domains)
|
|
164
|
+
for m in _anchor_matcher.finditer(query):
|
|
165
|
+
term = m.group(0).lower()
|
|
166
|
+
doms = _anchor_to_domains.get(term)
|
|
167
|
+
if doms:
|
|
168
|
+
candidates.update(doms)
|
|
169
|
+
return candidates
|
|
77
170
|
|
|
78
171
|
|
|
79
172
|
def _ensure_domain_compiled(domain: dict[str, Any]) -> list[tuple[re.Pattern, re.Pattern | None]]:
|
|
@@ -134,18 +227,30 @@ def _split_pattern_entry(entry: Any) -> tuple[Any, Any]:
|
|
|
134
227
|
return entry, None
|
|
135
228
|
|
|
136
229
|
|
|
137
|
-
def
|
|
138
|
-
"""
|
|
230
|
+
def _domain_fires_with_span(domain: dict[str, Any], query: str) -> tuple[bool, int, int]:
|
|
231
|
+
"""一个域是否成立,并返回命中的 (是否成立, match_span_start, match_span_len)。"""
|
|
139
232
|
for match_re, unless_re in _ensure_domain_compiled(domain):
|
|
140
233
|
if unless_re is not None and unless_re.search(query):
|
|
141
234
|
continue
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
235
|
+
m = match_re.search(query)
|
|
236
|
+
if m is not None:
|
|
237
|
+
start, end = m.span()
|
|
238
|
+
return True, start, end - start
|
|
239
|
+
return False, 0, 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _domain_fires(domain: dict[str, Any], query: str) -> bool:
|
|
243
|
+
"""一个域是否成立:任一触发词命中,且它自己的否决词没命中。"""
|
|
244
|
+
fires, _, _ = _domain_fires_with_span(domain, query)
|
|
245
|
+
return fires
|
|
145
246
|
|
|
146
247
|
|
|
147
248
|
def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
148
|
-
global _compiled_domains, _compiled_domains_id
|
|
249
|
+
global _compiled_domains, _compiled_domains_id, _compiled_domains_ref
|
|
250
|
+
# 快速指针比对:当传入同一 domains 引用且已编译时 O(1) 返回,免除 97 域的 SHA1 开销
|
|
251
|
+
dom_ref = id(domains)
|
|
252
|
+
if _compiled_domains is not None and _compiled_domains_ref == dom_ref:
|
|
253
|
+
return _compiled_domains
|
|
149
254
|
# 用内容指纹替代 id(domains),避免 GC 后 id 复用导致缓存失效/错误。
|
|
150
255
|
# 指纹含规则本体(只记长度会漏「改正则不改条数」的编辑),且必须覆盖
|
|
151
256
|
# exclude_if / intent_required——它们和 patterns 一样会改变命中结果。
|
|
@@ -169,6 +274,8 @@ def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]
|
|
|
169
274
|
for idx, domain in enumerate(domains)
|
|
170
275
|
]
|
|
171
276
|
_compiled_domains_id = dom_fp
|
|
277
|
+
_compiled_domains_ref = dom_ref
|
|
278
|
+
_init_anchor_index(_compiled_domains)
|
|
172
279
|
return _compiled_domains
|
|
173
280
|
|
|
174
281
|
|
|
@@ -311,20 +418,39 @@ def match_domains(query: str, domains: list[dict[str, Any]] | None = None,
|
|
|
311
418
|
domains = get_domains()
|
|
312
419
|
compiled = _get_compiled_domains(domains)
|
|
313
420
|
hits: list[dict[str, Any]] = []
|
|
421
|
+
secondary_hits: list[tuple[dict[str, Any], float]] = []
|
|
314
422
|
catch_all: dict[str, Any] | None = None
|
|
315
423
|
non_zh = bool(primary_lang and primary_lang not in ("zh", "mixed", "other"))
|
|
316
|
-
|
|
424
|
+
candidates = _get_candidate_domain_indices(query)
|
|
425
|
+
q_len = max(len(query), 1)
|
|
426
|
+
|
|
427
|
+
for idx, domain in enumerate(compiled):
|
|
317
428
|
if not domain.get("patterns", []):
|
|
318
429
|
catch_all = domain
|
|
319
430
|
continue
|
|
320
431
|
if non_zh and domain.get("name") in _ZH_LANG_GATED_DOMAINS:
|
|
321
432
|
continue
|
|
322
|
-
if
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
433
|
+
if candidates is not None and idx not in candidates:
|
|
434
|
+
continue
|
|
435
|
+
fires, span_start, span_len = _domain_fires_with_span(domain, query)
|
|
436
|
+
if fires:
|
|
437
|
+
if not hits:
|
|
438
|
+
hits.append(domain)
|
|
439
|
+
else:
|
|
440
|
+
pos_ratio = span_start / q_len
|
|
441
|
+
# 方案二:跨度高(特异性强)优先提拔;同跨度下严格保持前序特化域顺位稳定
|
|
442
|
+
score = (span_len * 2.0) - (idx * 0.1) + (pos_ratio * 0.01)
|
|
443
|
+
secondary_hits.append((domain, score))
|
|
444
|
+
if len(hits) + len(secondary_hits) >= max_n:
|
|
445
|
+
break
|
|
446
|
+
|
|
447
|
+
if secondary_hits:
|
|
448
|
+
secondary_hits.sort(key=lambda x: x[1], reverse=True)
|
|
449
|
+
hits.extend(h[0] for h in secondary_hits)
|
|
450
|
+
|
|
326
451
|
hits = _social_domain_first(hits)
|
|
327
452
|
hits = _intent_gate(hits, query)
|
|
453
|
+
hits = hits[:max_n]
|
|
328
454
|
return hits if hits else ([catch_all] if catch_all else [])
|
|
329
455
|
|
|
330
456
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""safe_conn.py — 解析即锁定的 HTTP/HTTPS 连接类(DNS rebinding / TOCTOU 防护)
|
|
3
|
+
|
|
4
|
+
独立成模块的理由:``http.client`` 会连带拉起 ``ssl``(实测 ≈14 ms)。
|
|
5
|
+
``url_safety`` 被 ``robots_guard`` 等在**模块级**导入(见 robots_guard 的
|
|
6
|
+
「轻导入」注释),若把 ``http.client`` 放在 ``url_safety`` 里,
|
|
7
|
+
``test_robots_guard.test_module_import_stays_light`` 会红。连接类只在真正
|
|
8
|
+
建连时经 ``net_proxy.open_connection`` 导入,不进轻导入路径。
|
|
9
|
+
|
|
10
|
+
原理:``check_url()`` 在请求前解析一次 DNS,``http.client`` 在 ``connect()``
|
|
11
|
+
时再解析一次——两次之间域名可改指内网(DNS rebinding)。本类在 ``connect()``
|
|
12
|
+
里解析一次、过滤私网、把结果钉给本次 TCP 连接,窗口关闭;SNI/Host 仍用原域名。
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import http.client
|
|
18
|
+
|
|
19
|
+
from url_safety import pin_host
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SafeHTTPConnection(http.client.HTTPConnection):
|
|
23
|
+
"""解析即锁定的 HTTP 连接:TCP 只连校验过的公网 IP,Host 头仍用原域名。"""
|
|
24
|
+
|
|
25
|
+
def connect(self):
|
|
26
|
+
pinned = pin_host(self.host)
|
|
27
|
+
if pinned is None:
|
|
28
|
+
return super().connect()
|
|
29
|
+
host = self.host
|
|
30
|
+
self.host = pinned
|
|
31
|
+
try:
|
|
32
|
+
super().connect()
|
|
33
|
+
finally:
|
|
34
|
+
self.host = host
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SafeHTTPSConnection(http.client.HTTPSConnection):
|
|
38
|
+
"""解析即锁定的 HTTPS 连接:TCP 钉公网 IP,SNI/证书校验仍用原域名。"""
|
|
39
|
+
|
|
40
|
+
def connect(self):
|
|
41
|
+
pinned = pin_host(self.host)
|
|
42
|
+
if pinned is None:
|
|
43
|
+
return super().connect()
|
|
44
|
+
host = self.host
|
|
45
|
+
self.host = pinned
|
|
46
|
+
try:
|
|
47
|
+
http.client.HTTPConnection.connect(self) # 仅 TCP
|
|
48
|
+
finally:
|
|
49
|
+
self.host = host
|
|
50
|
+
server_hostname = self._tunnel_host or host
|
|
51
|
+
self.sock = self._context.wrap_socket(
|
|
52
|
+
self.sock, server_hostname=server_hostname)
|
package/scripts/search_entry.py
CHANGED
|
@@ -156,6 +156,9 @@ class _SearchRun:
|
|
|
156
156
|
useful_ms: int = 0
|
|
157
157
|
dispatch_tail: dict[str, int] = field(default_factory=dict)
|
|
158
158
|
early_stopped: bool = False
|
|
159
|
+
# 早停弃置且仍存活的 (engine, holder, now_fn):finalize 交给
|
|
160
|
+
# tail_completion 后台收割。纯数据字段,不改变任何既有阶段的行为。
|
|
161
|
+
stranded: tuple = ()
|
|
159
162
|
budget_used_ms: int | None = None
|
|
160
163
|
budget_total_ms: int | None = None
|
|
161
164
|
elapsed: int = 0
|
|
@@ -772,7 +775,7 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
|
|
|
772
775
|
# 自适应学习(单事务批量写入 + cache_guard 统一状态态过滤)
|
|
773
776
|
try:
|
|
774
777
|
import contextlib
|
|
775
|
-
from adaptive import get_learner
|
|
778
|
+
from adaptive import get_learner, scenario_of
|
|
776
779
|
learner = get_learner()
|
|
777
780
|
_outcome_of = {o.get("engine"): o
|
|
778
781
|
for o in engine_outcomes if isinstance(o, dict)}
|
|
@@ -797,12 +800,27 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
|
|
|
797
800
|
quality = None
|
|
798
801
|
learner.record(eng, success=success, latency_ms=latency,
|
|
799
802
|
cost=0.0 if cost >= 0.85 else 0.001, empty=empty,
|
|
800
|
-
quality=quality)
|
|
803
|
+
quality=quality, **scenario_of(decision))
|
|
801
804
|
except ImportError:
|
|
802
805
|
pass
|
|
803
806
|
except Exception as e:
|
|
804
807
|
_log(f"自适应学习记录跳过: {type(e).__name__}")
|
|
805
808
|
|
|
809
|
+
# 早停后台补全(2026-10-25):弃置引擎的交付不再丢弃,宽限内完成的
|
|
810
|
+
# 记学习器真实成绩 + 合并写 L2 缓存(ARGO_TAIL_COMPLETE_S 控制,0=关)。
|
|
811
|
+
# 放在缓存写与学习记账之后:补全读的是刚写的主缓存条目;主响应已定,
|
|
812
|
+
# 这里的一切都是锦上添花,异常 fail-open。
|
|
813
|
+
if run.stranded:
|
|
814
|
+
try:
|
|
815
|
+
from tail_completion import hand_off
|
|
816
|
+
hand_off(run.stranded, query=query,
|
|
817
|
+
cache_engine_key=cache_engine_key,
|
|
818
|
+
max_results=max_results, domain=domain,
|
|
819
|
+
mode=mode, depth=depth, decision=decision,
|
|
820
|
+
skip_cache=skip_cache, cache_instance=cache)
|
|
821
|
+
except Exception as e:
|
|
822
|
+
_log(f"后台补全移交跳过: {type(e).__name__}")
|
|
823
|
+
|
|
806
824
|
# 语言偏好:记录本轮查询语 + 输出观测快照(默认中英 + 系统 + 习惯)
|
|
807
825
|
lang_pref_info: dict[str, Any] | None = None
|
|
808
826
|
try:
|
package/scripts/search_rank.py
CHANGED
|
@@ -96,6 +96,12 @@ _ENGINE_FUSION_WEIGHTS: dict[str, float] = {
|
|
|
96
96
|
"stackoverflow": 1.1, "imdb": 1.2, "thesportsdb": 1.2, "itunes": 1.2,
|
|
97
97
|
"finviz": 1.2, "sina_quote": 1.2, "tencent_quote": 1.2, "eastmoney": 1.2,
|
|
98
98
|
"fred": 1.3, "worldbank": 1.3, "nbs_stats": 1.3, "eurostat": 1.3, "fx_rate": 1.2, "frankfurter": 1.2,
|
|
99
|
+
# 2026-10-25 补表:学术聚合与官方一手数据此前表外=1.0 基线。实证缺口:
|
|
100
|
+
# 学术域查询常由 core 独家交付(当日实测组合里 arxiv/local_pubmed 零结果、
|
|
101
|
+
# 只 core 有货)却拿基线权重;官方 open data / 法律 / 标准系同理。
|
|
102
|
+
"core": 1.3, "openreview": 1.3, "zenodo": 1.2, "datacite": 1.2,
|
|
103
|
+
"sec_edgar": 1.3, "federal_register": 1.3, "eu_opendata": 1.3,
|
|
104
|
+
"egov_law": 1.3, "openstd": 1.3, "who_gho": 1.3, "usgs": 1.3, "nasa_cmr": 1.3,
|
|
99
105
|
# 通用引擎(基线)
|
|
100
106
|
"duckduckgo": 1.0, "local_bing": 1.0, "local_duckduckgo": 1.0,
|
|
101
107
|
"local_google": 1.0, "anysearch": 1.05, "byted": 1.1, "bocha": 1.0,
|
package/scripts/serp_guard.py
CHANGED
|
@@ -334,3 +334,26 @@ def filter_content_farms(results: list) -> list:
|
|
|
334
334
|
kept.append(r)
|
|
335
335
|
return kept
|
|
336
336
|
|
|
337
|
+
|
|
338
|
+
# ── 自适应放宽建议(2026-10-23 方案四)───────────────────────────────────
|
|
339
|
+
|
|
340
|
+
_RELAX_TAIL_RE = re.compile(
|
|
341
|
+
r"(?:\b\d{4}年?(?:度)?|\b\d{1,2}月|"
|
|
342
|
+
r"最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
|
|
343
|
+
r"知乎|贴吧|论坛|大全|汇总|排行榜|一览|数据|报告|统计|分析|趋势)"
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def suggest_relaxed_query(query: str) -> str:
|
|
348
|
+
"""针对 SERP 守卫击杀场景的自适应放宽查询(剥离过度约束的修饰与年份词)。"""
|
|
349
|
+
if not query or not isinstance(query, str):
|
|
350
|
+
return ""
|
|
351
|
+
q = _RELAX_TAIL_RE.sub(" ", query)
|
|
352
|
+
q = re.sub(r"(?<=\s)[年月](?=\s|$)|^([年月]\s+)", " ", q)
|
|
353
|
+
tokens = re.findall(r"[\u4e00-\u9fffA-Za-z0-9]+", q)
|
|
354
|
+
stopwords = {"的", "了", "呢", "吧", "请问", "怎么", "如何", "一下", "关于", "有关", "求", "谁知道",
|
|
355
|
+
"the", "a", "an", "of", "for", "to", "how", "what"}
|
|
356
|
+
kept = [t for t in tokens if t.lower() not in stopwords]
|
|
357
|
+
relaxed = " ".join(kept).strip()
|
|
358
|
+
return relaxed or re.sub(r"\s+", " ", q).strip() or query
|
|
359
|
+
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""tail_completion.py — 早停后台补全:慢源交付从「丢弃」变「补齐」。
|
|
2
|
+
|
|
3
|
+
## 为什么(2026-10-25)
|
|
4
|
+
|
|
5
|
+
七天窗口 1185 条真实查询里 74% 发生 early-stop。被放弃的交付不是低质货:
|
|
6
|
+
离线回放(bench_tail_ab --quality)显示被放弃交付 81% 来自 anysearch,相关度
|
|
7
|
+
代理比全库均线高 0.125;五条对照查询里四条「唯一有结果的就是慢源」(证据与
|
|
8
|
+
口径见 docs/QUALITY_OF_SLOW_SOURCES.md)。早停省的是**用户等待**,不该连
|
|
9
|
+
**交付**一起丢——那是在拿准确性换速度。缩尾宽已被实证否决,本模块是拍板的
|
|
10
|
+
正确方向:慢源不拖住快源(早交付),但慢源的交付进缓存(后台补全)。
|
|
11
|
+
|
|
12
|
+
## 做什么
|
|
13
|
+
|
|
14
|
+
engine_dispatch 在早停弃置引擎时把 (engine, holder, now_fn) 交给本模块——
|
|
15
|
+
主响应与既有记账逐位不变(cancelled outcome / abandoned_ms 照记)。本模块
|
|
16
|
+
起一个 daemon 收割线程,在主流程返回后继续等被弃置的引擎,宽限内完成的:
|
|
17
|
+
|
|
18
|
+
1. 学习器记真实成绩(success / latency / quality / 场景分桶标签)——此前
|
|
19
|
+
cancelled 被 cache_guard.should_record_adaptive 跳过,慢源的真实表现
|
|
20
|
+
从没进过学习器,场景分桶(engine×domain×lang)也少一笔真数据;
|
|
21
|
+
2. 把交付合并写回 L2 combo 缓存(URL 去重、新条目追加、缺省 score 用词面
|
|
22
|
+
相关度)——同查询第二次免费拿到「快源+慢源」的全量覆盖。
|
|
23
|
+
|
|
24
|
+
timeout 弃置不收:那是引擎确实超窗(甚至卡死),补全等于鼓励慢源;
|
|
25
|
+
cancelled 是「编排收工决定,引擎没失败」——两者语义不同(见
|
|
26
|
+
engine_dispatch._settle_pending 的注释)。
|
|
27
|
+
|
|
28
|
+
## 进程语义
|
|
29
|
+
|
|
30
|
+
收割线程是 daemon;CLI 进程退出前由 atexit 宽限等待(hand_off 首次调用时
|
|
31
|
+
注册,无 stranded 的进程零成本、不注册)。MCP/daemon 常驻进程不退出,
|
|
32
|
+
自然收割全部。宽限从**入队时刻**起算整批(job 级 deadline),不做逐引擎
|
|
33
|
+
累加——实测被放弃交付从早停时刻起的剩余等待 p50 2306ms / p90 3883ms。
|
|
34
|
+
|
|
35
|
+
## 开关
|
|
36
|
+
|
|
37
|
+
ARGO_TAIL_COMPLETE_S:宽限秒数(默认 3.0;0=禁用整个机制,行为回到
|
|
38
|
+
纯丢弃)。改动记得同步 references/usage.md 的开关总表(usage_doc_gates)。
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import atexit
|
|
44
|
+
import os
|
|
45
|
+
import threading
|
|
46
|
+
|
|
47
|
+
_LOCK = threading.Lock()
|
|
48
|
+
_JOBS: list[dict] = []
|
|
49
|
+
_WORKER: threading.Thread | None = None
|
|
50
|
+
_ATEXIT_REGISTERED = False
|
|
51
|
+
_STATS = {"salvaged": 0, "expired": 0, "recorded": 0}
|
|
52
|
+
_POLL_INTERVAL_S = 0.05
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def grace_seconds() -> float:
|
|
56
|
+
"""收割宽限(ARGO_TAIL_COMPLETE_S,默认 3.0;<=0 禁用)。"""
|
|
57
|
+
try:
|
|
58
|
+
return float(os.environ.get("ARGO_TAIL_COMPLETE_S", "3.0") or 3.0)
|
|
59
|
+
except ValueError:
|
|
60
|
+
return 3.0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def hand_off(stranded, *, query, cache_engine_key, max_results,
|
|
64
|
+
domain, mode, depth, decision, skip_cache=False,
|
|
65
|
+
cache_instance=None) -> None:
|
|
66
|
+
"""把早停弃置的引擎交给后台收割。
|
|
67
|
+
|
|
68
|
+
``stranded`` 是 engine_dispatch 产出的 ``[(engine, holder, now_fn), ...]``:
|
|
69
|
+
holder["r"] 出现即完成(与 _ingest_holder 同判据),now_fn 用于算真实
|
|
70
|
+
latency(dispatch 可能跑在单调钟上,收割线程不能用墙钟去减)。
|
|
71
|
+
``cache_instance`` 是本次请求的缓存实例(req.cache):补全读写的必须是
|
|
72
|
+
同一实例(L1 内存态 + L2 落盘一致),模块级没有全局缓存可拿。
|
|
73
|
+
其余参数与 finalize 的 cache.set / learner.record 同源同口径。
|
|
74
|
+
任何异常 fail-open:补全失败只丢「锦上添花」,不影响主响应。
|
|
75
|
+
"""
|
|
76
|
+
grace = grace_seconds()
|
|
77
|
+
if grace <= 0 or not stranded:
|
|
78
|
+
return
|
|
79
|
+
global _ATEXIT_REGISTERED
|
|
80
|
+
with _LOCK:
|
|
81
|
+
if not _ATEXIT_REGISTERED:
|
|
82
|
+
atexit.register(wait_for_completion)
|
|
83
|
+
_ATEXIT_REGISTERED = True
|
|
84
|
+
_JOBS.append({
|
|
85
|
+
"stranded": list(stranded),
|
|
86
|
+
"deadline": _now_wall() + grace,
|
|
87
|
+
"query": query, "cache_engine_key": cache_engine_key,
|
|
88
|
+
"max_results": max_results, "domain": domain,
|
|
89
|
+
"mode": mode, "depth": depth, "decision": decision or {},
|
|
90
|
+
"skip_cache": bool(skip_cache) or cache_instance is None,
|
|
91
|
+
"cache": cache_instance,
|
|
92
|
+
})
|
|
93
|
+
_start_worker_locked()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def wait_for_completion(timeout_s: float | None = None) -> None:
|
|
97
|
+
"""等收割线程收尾(atexit 出口 / 测试用)。宽限耗尽即返回,不无限等。"""
|
|
98
|
+
t = _WORKER
|
|
99
|
+
if t is not None and t.is_alive():
|
|
100
|
+
budget = timeout_s if timeout_s is not None else grace_seconds() + 0.5
|
|
101
|
+
t.join(timeout=budget)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def stats() -> dict:
|
|
105
|
+
"""收割计数(可观测口,供诊断;不改任何行为)。"""
|
|
106
|
+
with _LOCK:
|
|
107
|
+
return dict(_STATS)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _now_wall() -> float:
|
|
111
|
+
import time
|
|
112
|
+
return time.time()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _start_worker_locked() -> None:
|
|
116
|
+
global _WORKER
|
|
117
|
+
if _WORKER is not None and _WORKER.is_alive():
|
|
118
|
+
return
|
|
119
|
+
_WORKER = threading.Thread(target=_worker, daemon=True,
|
|
120
|
+
name="argo-tail-completion")
|
|
121
|
+
_WORKER.start()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _worker() -> None:
|
|
125
|
+
"""串行收割队列:宽限内完成的记学习器+合并缓存,超宽限标 expired。"""
|
|
126
|
+
import time
|
|
127
|
+
while True:
|
|
128
|
+
with _LOCK:
|
|
129
|
+
if not _JOBS:
|
|
130
|
+
return
|
|
131
|
+
job = _JOBS.pop(0)
|
|
132
|
+
for eng, holder, now_fn in job["stranded"]:
|
|
133
|
+
remaining = job["deadline"] - _now_wall()
|
|
134
|
+
while holder.get("r") is None and remaining > 0:
|
|
135
|
+
time.sleep(min(_POLL_INTERVAL_S, remaining))
|
|
136
|
+
remaining = job["deadline"] - _now_wall()
|
|
137
|
+
res = (holder.get("r") or (None,))[1] if holder.get("r") else None
|
|
138
|
+
if res is None:
|
|
139
|
+
with _LOCK:
|
|
140
|
+
_STATS["expired"] += 1
|
|
141
|
+
continue
|
|
142
|
+
goods = [r for r in (res or [])
|
|
143
|
+
if isinstance(r, dict) and "error" not in r]
|
|
144
|
+
with _LOCK:
|
|
145
|
+
_STATS["salvaged"] += 1
|
|
146
|
+
try:
|
|
147
|
+
_record_adaptive(eng, goods, holder, now_fn, job)
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
if not job["skip_cache"]:
|
|
151
|
+
try:
|
|
152
|
+
_merge_into_cache(eng, goods, job)
|
|
153
|
+
except Exception:
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _record_adaptive(eng, goods, holder, now_fn, job) -> None:
|
|
158
|
+
"""学习器记真实成绩(口径与 search_pipeline.finalize 对齐)。"""
|
|
159
|
+
from adaptive import get_learner, scenario_of
|
|
160
|
+
from query_signals import result_relevance
|
|
161
|
+
success = bool(goods)
|
|
162
|
+
try:
|
|
163
|
+
lat_ms = float((now_fn() - holder.get("t0", now_fn())) * 1000.0)
|
|
164
|
+
except Exception:
|
|
165
|
+
lat_ms = 0.0
|
|
166
|
+
quality = None
|
|
167
|
+
if success:
|
|
168
|
+
try:
|
|
169
|
+
quality = result_relevance(job["query"], goods)
|
|
170
|
+
except Exception:
|
|
171
|
+
quality = None
|
|
172
|
+
with _LOCK:
|
|
173
|
+
_STATS["recorded"] += 1
|
|
174
|
+
get_learner().record(eng, success=success, latency_ms=max(0.0, lat_ms),
|
|
175
|
+
cost=0.001, empty=not success, quality=quality,
|
|
176
|
+
**scenario_of(job["decision"]))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _merge_into_cache(eng, goods, job) -> None:
|
|
180
|
+
"""交付合并写回 L2 combo 缓存:读现条目 → URL 去重追加 → 同 key 覆盖。
|
|
181
|
+
|
|
182
|
+
只在已有缓存条目时合并(本机制补的是「快源已写的那份」);主缓存被守卫
|
|
183
|
+
拒写时这里不硬造。score 缺省用词面相关度,保证追加条目有可排序的分数。
|
|
184
|
+
"""
|
|
185
|
+
from query_signals import result_relevance
|
|
186
|
+
cache = job["cache"]
|
|
187
|
+
cur = cache.get(job["query"], job["cache_engine_key"],
|
|
188
|
+
job["max_results"], domain=job["domain"],
|
|
189
|
+
mode=job["mode"], depth=job["depth"])
|
|
190
|
+
if not cur or not isinstance(cur.get("results"), list) or not goods:
|
|
191
|
+
return
|
|
192
|
+
existing = {r.get("url") for r in cur["results"]
|
|
193
|
+
if isinstance(r, dict)}
|
|
194
|
+
new: list[dict] = []
|
|
195
|
+
for r in goods:
|
|
196
|
+
if not isinstance(r, dict) or r.get("url") in existing:
|
|
197
|
+
continue
|
|
198
|
+
item = dict(r)
|
|
199
|
+
if "score" not in item:
|
|
200
|
+
try:
|
|
201
|
+
item["score"] = round(float(result_relevance(job["query"], [r])), 4)
|
|
202
|
+
except Exception:
|
|
203
|
+
item["score"] = 0.3
|
|
204
|
+
new.append(item)
|
|
205
|
+
if not new:
|
|
206
|
+
return
|
|
207
|
+
payload = dict(cur)
|
|
208
|
+
payload["results"] = list(cur["results"]) + new
|
|
209
|
+
payload["engines_used"] = sorted({*(cur.get("engines_used") or []), eng})
|
|
210
|
+
cache.set(job["query"], job["cache_engine_key"], job["max_results"],
|
|
211
|
+
payload, domain=job["domain"], mode=job["mode"],
|
|
212
|
+
depth=job["depth"])
|