argo-search 2.9.7 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +73 -150
- package/README.es.md +73 -150
- package/README.ja.md +79 -156
- package/README.ko.md +78 -155
- package/README.md +80 -159
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +78 -5
- package/bin/argo.js +33 -2
- package/config.yaml +269 -63
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +7 -2
- package/packages/dsh-plugin/README.md +25 -6
- package/packages/dsh-plugin/dsh/client.js +402 -47
- package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
- package/packages/dsh-plugin/dsh/index.js +135 -1876
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +3 -3
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +177 -2
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +126 -43
- package/scripts/article_meta.py +131 -0
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +119 -4
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +105 -0
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +58 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +58 -22
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +28 -3
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/http_client.py +24 -38
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +192 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/net_proxy.py +27 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery.py +7 -3
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +26 -5
- package/scripts/route_domains.py +170 -31
- package/scripts/safe_conn.py +52 -0
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +8 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +62 -195
- package/scripts/search_rank.py +6 -0
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +138 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
package/scripts/route_domains.py
CHANGED
|
@@ -70,10 +70,123 @@ _ZH_LANG_GATED_DOMAINS = frozenset({
|
|
|
70
70
|
})
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
# ──
|
|
73
|
+
# ── 域预编译(内容指纹缓存与锚点快筛) ──────────────────────────────────────────
|
|
74
74
|
|
|
75
75
|
_compiled_domains: list[dict[str, Any]] | None = None
|
|
76
76
|
_compiled_domains_id: tuple | None = None # (name, patterns/exclude_if 内容指纹)
|
|
77
|
+
_compiled_domains_ref: int | None = None
|
|
78
|
+
_anchor_matcher: re.Pattern | None = None
|
|
79
|
+
_anchor_to_domains: dict[str, list[int]] = {}
|
|
80
|
+
_unconstrained_domains: set[int] = set()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _extract_anchors(src: Any) -> set[str] | None:
|
|
84
|
+
"""从正则源码中提取一组必须包含其一的字面锚点词集合(全小写)。
|
|
85
|
+
|
|
86
|
+
若无法提取安全锚点(如含有纯通配、字符集类 []、通用转义字符类),返回 None。
|
|
87
|
+
"""
|
|
88
|
+
if not isinstance(src, str) or not src:
|
|
89
|
+
return None
|
|
90
|
+
s = re.sub(r"^\(\?[a-zA-Z]+\)", "", src)
|
|
91
|
+
s = re.sub(r"\\b", "", s)
|
|
92
|
+
if s.startswith("(") and s.endswith(")"):
|
|
93
|
+
s = s[1:-1]
|
|
94
|
+
parts: list[str] = []
|
|
95
|
+
depth = 0
|
|
96
|
+
cur: list[str] = []
|
|
97
|
+
for ch in s:
|
|
98
|
+
if ch in ("(", "["):
|
|
99
|
+
depth += 1
|
|
100
|
+
elif ch in (")", "]"):
|
|
101
|
+
depth -= 1
|
|
102
|
+
if ch == "|" and depth == 0:
|
|
103
|
+
parts.append("".join(cur))
|
|
104
|
+
cur = []
|
|
105
|
+
else:
|
|
106
|
+
cur.append(ch)
|
|
107
|
+
if cur:
|
|
108
|
+
parts.append("".join(cur))
|
|
109
|
+
|
|
110
|
+
anchors: set[str] = set()
|
|
111
|
+
for part in parts:
|
|
112
|
+
# 正则字符类与元字符无法保证字面子串存在
|
|
113
|
+
if re.search(r"\[.*?\]", part) or re.search(r"\\[wWdDsS]", part):
|
|
114
|
+
return None
|
|
115
|
+
runs = re.findall(r"[\u4e00-\u9fff]{1,}|[a-zA-Z0-9_\-\.]{2,}", part)
|
|
116
|
+
if not runs:
|
|
117
|
+
return None
|
|
118
|
+
for r in runs:
|
|
119
|
+
anchors.add(r.lower())
|
|
120
|
+
return anchors if anchors else None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _init_anchor_index(compiled: list[dict[str, Any]]) -> None:
|
|
124
|
+
"""初始化全局锚点自动机索引(方案一:Aho-Corasick 前置快速分桶)。"""
|
|
125
|
+
global _anchor_matcher, _anchor_to_domains, _unconstrained_domains
|
|
126
|
+
_anchor_to_domains = {}
|
|
127
|
+
_unconstrained_domains = set()
|
|
128
|
+
all_anchors: set[str] = set()
|
|
129
|
+
|
|
130
|
+
for idx, domain in enumerate(compiled):
|
|
131
|
+
entries = _pattern_entries(domain)
|
|
132
|
+
if not entries:
|
|
133
|
+
_unconstrained_domains.add(idx)
|
|
134
|
+
continue
|
|
135
|
+
dom_anchors: set[str] = set()
|
|
136
|
+
unconstrained = False
|
|
137
|
+
for p in entries:
|
|
138
|
+
match_src, _ = _split_pattern_entry(p)
|
|
139
|
+
anc = _extract_anchors(match_src)
|
|
140
|
+
if anc is None:
|
|
141
|
+
unconstrained = True
|
|
142
|
+
break
|
|
143
|
+
dom_anchors.update(anc)
|
|
144
|
+
if unconstrained or not dom_anchors:
|
|
145
|
+
_unconstrained_domains.add(idx)
|
|
146
|
+
else:
|
|
147
|
+
for term in dom_anchors:
|
|
148
|
+
_anchor_to_domains.setdefault(term, []).append(idx)
|
|
149
|
+
all_anchors.add(term)
|
|
150
|
+
|
|
151
|
+
if all_anchors:
|
|
152
|
+
sorted_terms = sorted(all_anchors, key=len, reverse=True)
|
|
153
|
+
escaped = [re.escape(t) for t in sorted_terms]
|
|
154
|
+
_anchor_matcher = re.compile("|".join(escaped), re.IGNORECASE)
|
|
155
|
+
else:
|
|
156
|
+
_anchor_matcher = None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _get_candidate_domain_indices(query: str) -> set[int] | None:
|
|
160
|
+
"""通过锚点词快速扫描返回候选域索引集合;无索引时返回 None。"""
|
|
161
|
+
if _anchor_matcher is None:
|
|
162
|
+
return None
|
|
163
|
+
candidates = set(_unconstrained_domains)
|
|
164
|
+
for m in _anchor_matcher.finditer(query):
|
|
165
|
+
term = m.group(0).lower()
|
|
166
|
+
doms = _anchor_to_domains.get(term)
|
|
167
|
+
if doms:
|
|
168
|
+
candidates.update(doms)
|
|
169
|
+
return candidates
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _ensure_domain_compiled(domain: dict[str, Any]) -> list[tuple[re.Pattern, re.Pattern | None]]:
|
|
173
|
+
"""按需编译单个域的触发词与意图正则(扫到哪个域才编译哪个域)。"""
|
|
174
|
+
rules = domain.get("_rules")
|
|
175
|
+
if rules is not None:
|
|
176
|
+
return rules
|
|
177
|
+
rules = []
|
|
178
|
+
for p in _pattern_entries(domain):
|
|
179
|
+
match_src, unless_src = _split_pattern_entry(p)
|
|
180
|
+
match_re = _try_compile(match_src)
|
|
181
|
+
if match_re is None:
|
|
182
|
+
continue # 触发词写坏 → 丢这一条(与旧行为一致)
|
|
183
|
+
rules.append((match_re, _try_compile(unless_src)))
|
|
184
|
+
intent_src = domain.get("intent_required")
|
|
185
|
+
domain["_intent"] = (
|
|
186
|
+
_try_compile(intent_src) if isinstance(intent_src, str) and intent_src else None
|
|
187
|
+
)
|
|
188
|
+
domain["_rules"] = rules
|
|
189
|
+
return rules
|
|
77
190
|
|
|
78
191
|
|
|
79
192
|
def _compile_domain_patterns(domains: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
@@ -85,22 +198,9 @@ def _compile_domain_patterns(domains: list[dict[str, Any]]) -> list[dict[str, An
|
|
|
85
198
|
"""
|
|
86
199
|
compiled = []
|
|
87
200
|
for idx, domain in enumerate(domains):
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
match_re = _try_compile(match_src)
|
|
92
|
-
if match_re is None:
|
|
93
|
-
continue # 触发词写坏 → 丢这一条(与旧行为一致)
|
|
94
|
-
rules.append((match_re, _try_compile(unless_src)))
|
|
95
|
-
intent_re = None
|
|
96
|
-
intent_src = domain.get("intent_required")
|
|
97
|
-
if isinstance(intent_src, str) and intent_src:
|
|
98
|
-
try:
|
|
99
|
-
intent_re = re.compile(intent_src)
|
|
100
|
-
except re.error:
|
|
101
|
-
intent_re = None
|
|
102
|
-
compiled.append({**domain, "_idx": idx, "_rules": rules,
|
|
103
|
-
"_intent": intent_re})
|
|
201
|
+
entry = {**domain, "_idx": idx, "_rules": None, "_intent": None}
|
|
202
|
+
_ensure_domain_compiled(entry)
|
|
203
|
+
compiled.append(entry)
|
|
104
204
|
return compiled
|
|
105
205
|
|
|
106
206
|
|
|
@@ -127,18 +227,30 @@ def _split_pattern_entry(entry: Any) -> tuple[Any, Any]:
|
|
|
127
227
|
return entry, None
|
|
128
228
|
|
|
129
229
|
|
|
130
|
-
def
|
|
131
|
-
"""
|
|
132
|
-
for match_re, unless_re in domain
|
|
230
|
+
def _domain_fires_with_span(domain: dict[str, Any], query: str) -> tuple[bool, int, int]:
|
|
231
|
+
"""一个域是否成立,并返回命中的 (是否成立, match_span_start, match_span_len)。"""
|
|
232
|
+
for match_re, unless_re in _ensure_domain_compiled(domain):
|
|
133
233
|
if unless_re is not None and unless_re.search(query):
|
|
134
234
|
continue
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
235
|
+
m = match_re.search(query)
|
|
236
|
+
if m is not None:
|
|
237
|
+
start, end = m.span()
|
|
238
|
+
return True, start, end - start
|
|
239
|
+
return False, 0, 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _domain_fires(domain: dict[str, Any], query: str) -> bool:
|
|
243
|
+
"""一个域是否成立:任一触发词命中,且它自己的否决词没命中。"""
|
|
244
|
+
fires, _, _ = _domain_fires_with_span(domain, query)
|
|
245
|
+
return fires
|
|
138
246
|
|
|
139
247
|
|
|
140
248
|
def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
141
|
-
global _compiled_domains, _compiled_domains_id
|
|
249
|
+
global _compiled_domains, _compiled_domains_id, _compiled_domains_ref
|
|
250
|
+
# 快速指针比对:当传入同一 domains 引用且已编译时 O(1) 返回,免除 97 域的 SHA1 开销
|
|
251
|
+
dom_ref = id(domains)
|
|
252
|
+
if _compiled_domains is not None and _compiled_domains_ref == dom_ref:
|
|
253
|
+
return _compiled_domains
|
|
142
254
|
# 用内容指纹替代 id(domains),避免 GC 后 id 复用导致缓存失效/错误。
|
|
143
255
|
# 指纹含规则本体(只记长度会漏「改正则不改条数」的编辑),且必须覆盖
|
|
144
256
|
# exclude_if / intent_required——它们和 patterns 一样会改变命中结果。
|
|
@@ -155,8 +267,15 @@ def _get_compiled_domains(domains: list[dict[str, Any]]) -> list[dict[str, Any]]
|
|
|
155
267
|
)
|
|
156
268
|
if _compiled_domains is not None and _compiled_domains_id == dom_fp:
|
|
157
269
|
return _compiled_domains
|
|
158
|
-
|
|
270
|
+
# 槽位懒初始化:实际正则在 _domain_fires 扫到该域时才按需编译,
|
|
271
|
+
# 冷启动路由省去未扫到域(及被语言门跳过的域)的正则编译税。
|
|
272
|
+
_compiled_domains = [
|
|
273
|
+
{**domain, "_idx": idx, "_rules": None, "_intent": None}
|
|
274
|
+
for idx, domain in enumerate(domains)
|
|
275
|
+
]
|
|
159
276
|
_compiled_domains_id = dom_fp
|
|
277
|
+
_compiled_domains_ref = dom_ref
|
|
278
|
+
_init_anchor_index(_compiled_domains)
|
|
160
279
|
return _compiled_domains
|
|
161
280
|
|
|
162
281
|
|
|
@@ -168,7 +287,8 @@ def warm_compiled_domains() -> None:
|
|
|
168
287
|
懒编译与全量回退都会自愈,预热是纯优化。
|
|
169
288
|
"""
|
|
170
289
|
try:
|
|
171
|
-
_get_compiled_domains(get_domains())
|
|
290
|
+
for d in _get_compiled_domains(get_domains()):
|
|
291
|
+
_ensure_domain_compiled(d)
|
|
172
292
|
except Exception:
|
|
173
293
|
pass
|
|
174
294
|
|
|
@@ -298,20 +418,39 @@ def match_domains(query: str, domains: list[dict[str, Any]] | None = None,
|
|
|
298
418
|
domains = get_domains()
|
|
299
419
|
compiled = _get_compiled_domains(domains)
|
|
300
420
|
hits: list[dict[str, Any]] = []
|
|
421
|
+
secondary_hits: list[tuple[dict[str, Any], float]] = []
|
|
301
422
|
catch_all: dict[str, Any] | None = None
|
|
302
423
|
non_zh = bool(primary_lang and primary_lang not in ("zh", "mixed", "other"))
|
|
303
|
-
|
|
424
|
+
candidates = _get_candidate_domain_indices(query)
|
|
425
|
+
q_len = max(len(query), 1)
|
|
426
|
+
|
|
427
|
+
for idx, domain in enumerate(compiled):
|
|
304
428
|
if not domain.get("patterns", []):
|
|
305
429
|
catch_all = domain
|
|
306
430
|
continue
|
|
307
431
|
if non_zh and domain.get("name") in _ZH_LANG_GATED_DOMAINS:
|
|
308
432
|
continue
|
|
309
|
-
if
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
433
|
+
if candidates is not None and idx not in candidates:
|
|
434
|
+
continue
|
|
435
|
+
fires, span_start, span_len = _domain_fires_with_span(domain, query)
|
|
436
|
+
if fires:
|
|
437
|
+
if not hits:
|
|
438
|
+
hits.append(domain)
|
|
439
|
+
else:
|
|
440
|
+
pos_ratio = span_start / q_len
|
|
441
|
+
# 方案二:跨度高(特异性强)优先提拔;同跨度下严格保持前序特化域顺位稳定
|
|
442
|
+
score = (span_len * 2.0) - (idx * 0.1) + (pos_ratio * 0.01)
|
|
443
|
+
secondary_hits.append((domain, score))
|
|
444
|
+
if len(hits) + len(secondary_hits) >= max_n:
|
|
445
|
+
break
|
|
446
|
+
|
|
447
|
+
if secondary_hits:
|
|
448
|
+
secondary_hits.sort(key=lambda x: x[1], reverse=True)
|
|
449
|
+
hits.extend(h[0] for h in secondary_hits)
|
|
450
|
+
|
|
313
451
|
hits = _social_domain_first(hits)
|
|
314
452
|
hits = _intent_gate(hits, query)
|
|
453
|
+
hits = hits[:max_n]
|
|
315
454
|
return hits if hits else ([catch_all] if catch_all else [])
|
|
316
455
|
|
|
317
456
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""safe_conn.py — 解析即锁定的 HTTP/HTTPS 连接类(DNS rebinding / TOCTOU 防护)
|
|
3
|
+
|
|
4
|
+
独立成模块的理由:``http.client`` 会连带拉起 ``ssl``(实测 ≈14 ms)。
|
|
5
|
+
``url_safety`` 被 ``robots_guard`` 等在**模块级**导入(见 robots_guard 的
|
|
6
|
+
「轻导入」注释),若把 ``http.client`` 放在 ``url_safety`` 里,
|
|
7
|
+
``test_robots_guard.test_module_import_stays_light`` 会红。连接类只在真正
|
|
8
|
+
建连时经 ``net_proxy.open_connection`` 导入,不进轻导入路径。
|
|
9
|
+
|
|
10
|
+
原理:``check_url()`` 在请求前解析一次 DNS,``http.client`` 在 ``connect()``
|
|
11
|
+
时再解析一次——两次之间域名可改指内网(DNS rebinding)。本类在 ``connect()``
|
|
12
|
+
里解析一次、过滤私网、把结果钉给本次 TCP 连接,窗口关闭;SNI/Host 仍用原域名。
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import http.client
|
|
18
|
+
|
|
19
|
+
from url_safety import pin_host
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SafeHTTPConnection(http.client.HTTPConnection):
|
|
23
|
+
"""解析即锁定的 HTTP 连接:TCP 只连校验过的公网 IP,Host 头仍用原域名。"""
|
|
24
|
+
|
|
25
|
+
def connect(self):
|
|
26
|
+
pinned = pin_host(self.host)
|
|
27
|
+
if pinned is None:
|
|
28
|
+
return super().connect()
|
|
29
|
+
host = self.host
|
|
30
|
+
self.host = pinned
|
|
31
|
+
try:
|
|
32
|
+
super().connect()
|
|
33
|
+
finally:
|
|
34
|
+
self.host = host
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SafeHTTPSConnection(http.client.HTTPSConnection):
|
|
38
|
+
"""解析即锁定的 HTTPS 连接:TCP 钉公网 IP,SNI/证书校验仍用原域名。"""
|
|
39
|
+
|
|
40
|
+
def connect(self):
|
|
41
|
+
pinned = pin_host(self.host)
|
|
42
|
+
if pinned is None:
|
|
43
|
+
return super().connect()
|
|
44
|
+
host = self.host
|
|
45
|
+
self.host = pinned
|
|
46
|
+
try:
|
|
47
|
+
http.client.HTTPConnection.connect(self) # 仅 TCP
|
|
48
|
+
finally:
|
|
49
|
+
self.host = host
|
|
50
|
+
server_hostname = self._tunnel_host or host
|
|
51
|
+
self.sock = self._context.wrap_socket(
|
|
52
|
+
self.sock, server_hostname=server_hostname)
|
package/scripts/search.py
CHANGED
|
@@ -214,7 +214,6 @@ def _missing_env_for(eng: str) -> list[str]:
|
|
|
214
214
|
# 学习跳过判定。
|
|
215
215
|
|
|
216
216
|
from engine_dispatch import ( # noqa: E402
|
|
217
|
-
_QUOTA_ERROR_KEYWORDS,
|
|
218
217
|
classify_engine_outcome as _classify_engine_outcome,
|
|
219
218
|
run_dispatch,
|
|
220
219
|
)
|
|
@@ -547,18 +546,7 @@ def execute_search(query: str, decision: dict[str, Any], max_results: int,
|
|
|
547
546
|
|
|
548
547
|
# ── 统一入口 ──────────────────────────────────────────────────────────────────
|
|
549
548
|
|
|
550
|
-
|
|
551
|
-
"""include-local 三态解析:True/False 显式,None=自动(fast/budget 开)。
|
|
552
|
-
|
|
553
|
-
本地文件命中此前默认全关,模型与用户在正常使用中永远看不到本机内容
|
|
554
|
-
(2026-09-29 审查:「本地内容搜索被忽略」)。自动开的安全垫已齐备:
|
|
555
|
-
宽根守卫(home/根目录不扫)+ 3s 子进程超时 + L1 并行提交(不占墙钟,
|
|
556
|
-
常规项目目录 rg 实测 33ms)。auto/deep 保持显式开启——深度模式由
|
|
557
|
-
调用方自主决定证据面。
|
|
558
|
-
"""
|
|
559
|
-
if include_local is None:
|
|
560
|
-
return mode in ("fast", "budget")
|
|
561
|
-
return bool(include_local)
|
|
549
|
+
from local_seek import resolve_include_local as _resolve_include_local
|
|
562
550
|
|
|
563
551
|
|
|
564
552
|
def super_search(query: str, engine: str = "auto", n: int = 5, explain: bool = False,
|
|
@@ -785,13 +773,13 @@ def super_search(query: str, engine: str = "auto", n: int = 5, explain: bool = F
|
|
|
785
773
|
file=sys.stderr)
|
|
786
774
|
if search_query != original_query:
|
|
787
775
|
print(f"[改写] {original_query} → {search_query}", file=sys.stderr)
|
|
788
|
-
# L1
|
|
789
|
-
|
|
790
|
-
include_local = _resolve_include_local(include_local, mode)
|
|
776
|
+
# L1 并行化:本地搜索与主搜索并行提交(仅显式开启时在宽根下提示)。
|
|
777
|
+
_local_seek_explicit = include_local is not None
|
|
778
|
+
include_local = _resolve_include_local(include_local, mode, query)
|
|
791
779
|
_local_seek_future = None
|
|
792
780
|
if include_local:
|
|
793
781
|
_local_seek_future = _get_local_seek_executor().submit(
|
|
794
|
-
_run_local_seek, query, n)
|
|
782
|
+
_run_local_seek, query, n, warn=_local_seek_explicit)
|
|
795
783
|
|
|
796
784
|
result = execute_search(
|
|
797
785
|
query=search_query, decision=decision, max_results=n,
|
|
@@ -863,15 +851,15 @@ def super_search(query: str, engine: str = "auto", n: int = 5, explain: bool = F
|
|
|
863
851
|
"url-seed: seed URL was not fetched; results are related discovery only"
|
|
864
852
|
)
|
|
865
853
|
# 恢复链两种结局分开说(2026-10-10 方案甲):「恢复成功换了引擎」与
|
|
866
|
-
#
|
|
867
|
-
#
|
|
854
|
+
# 「触发了但无可用策略」对消费者的含义完全不同。失败那一半再细分:
|
|
855
|
+
# 源被跳过/鉴权失败时不要写成「查询没有召回」(2026-10-22 修正),
|
|
856
|
+
# 具体判据与措辞在 search_attribution.zero_result_recovery_note。
|
|
868
857
|
_rec_info = result.get("recovery") or {}
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
"the query had no recall — try rephrasing")
|
|
858
|
+
from search_attribution import zero_result_recovery_note
|
|
859
|
+
_rec_note = zero_result_recovery_note(
|
|
860
|
+
_rec_info, result.get("engine_outcomes"))
|
|
861
|
+
if _rec_note is not None:
|
|
862
|
+
extra_lim.append(_rec_note)
|
|
875
863
|
elif result.get("recovery"):
|
|
876
864
|
extra_lim.append("recovery used; engine fallback may differ from primary route")
|
|
877
865
|
# 执行归因:把「为什么只有这么几条 / 为什么只跑了一个源」写进随答案一起
|
|
@@ -942,8 +930,8 @@ def super_search(query: str, engine: str = "auto", n: int = 5, explain: bool = F
|
|
|
942
930
|
from serp_kill_log import drain_serp_guard_kills
|
|
943
931
|
for _kill in drain_serp_guard_kills():
|
|
944
932
|
result.setdefault("errors", []).append(
|
|
945
|
-
f"serp_guard: {_kill['engine']}
|
|
946
|
-
f"
|
|
933
|
+
f"serp_guard: {_kill['engine']} 头部结果均未覆盖 20% 查询词,"
|
|
934
|
+
f"整页按答非所问丢弃({_kill['results']} 条)——"
|
|
947
935
|
"如需复核可设 ARGO_SERP_GUARD=0 重跑")
|
|
948
936
|
except ImportError:
|
|
949
937
|
pass
|
|
@@ -44,15 +44,112 @@ def top_engine_latency(engine_latency: dict[str, Any] | None,
|
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def usage_log_attribution(engine_latency: dict[str, Any] | None,
|
|
47
|
-
top_n: int = DEFAULT_TOP_N
|
|
48
|
-
|
|
47
|
+
top_n: int = DEFAULT_TOP_N,
|
|
48
|
+
engine_outcomes: Iterable[dict[str, Any]] | None = None
|
|
49
|
+
) -> dict[str, Any]:
|
|
50
|
+
"""usage_log query 流的归因字段(慢源 Top-N + 失败类别)。
|
|
49
51
|
|
|
50
52
|
返回 dict 以便调用方 **展开进日志记录**,未来新增归因维度(如
|
|
51
53
|
「等待预算被谁耗掉」「缓存未命中的原因分布」)时只扩这里,
|
|
52
54
|
主管线那一行 `**usage_log_attribution(...)` 不动——归因是会长大的,
|
|
53
55
|
写入点不该跟着长(主管线有模块大小门禁,见模块 docstring)。
|
|
56
|
+
|
|
57
|
+
失败类别只记「哪类」不记条数:errors 已经记了条数,两者互补。
|
|
58
|
+
"""
|
|
59
|
+
out: dict[str, Any] = {
|
|
60
|
+
"engine_latency_top": top_engine_latency(engine_latency, top_n),
|
|
61
|
+
}
|
|
62
|
+
kinds = failure_kinds(engine_outcomes)
|
|
63
|
+
if kinds:
|
|
64
|
+
out["failure_kinds"] = kinds
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# 合成引擎名(只存在于基准/混沌测试的 fake combo)。它们出现在 engines_used
|
|
69
|
+
# 里就说明这一跑不是真实检索;写进生产 usage_log 会污染慢源排行与总量统计
|
|
70
|
+
# (2026-10-21 实测 "hedge bench" 32 条,dead_primary/fast_backup 进了慢源榜)。
|
|
71
|
+
_SYNTHETIC_ENGINES: frozenset[str] = frozenset({
|
|
72
|
+
"dead_primary", "dead_backup", "fast_backup", "slow_backup",
|
|
73
|
+
"slow_primary", "quick_primary", "fast_primary",
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def is_synthetic_run(engines: Iterable[str] | None) -> bool:
|
|
78
|
+
"""这组引擎是否来自基准/测试的合成组合(有一个即判合成)。"""
|
|
79
|
+
return any(str(e) in _SYNTHETIC_ENGINES for e in (engines or []))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# 「源被跳过/没资格调用」的状态:空结果时要把它们与「查询没有召回」分开。
|
|
83
|
+
# 只收持续性状态(配额/缺 env/鉴权),不含 timeout/error——后者是本次
|
|
84
|
+
# 真的失败了,原表述已够用。
|
|
85
|
+
SKIPPED_SOURCE_STATUSES: frozenset[str] = frozenset({
|
|
86
|
+
"skipped-quota-exhausted", "skipped-missing-env",
|
|
87
|
+
"quota-exhausted", "auth-failed",
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def skipped_sources(engine_outcomes: Iterable[dict[str, Any]] | None,
|
|
92
|
+
limit: int = 4) -> list[str]:
|
|
93
|
+
"""空结果时被跳过/鉴权失败的引擎名(去重、保序、限量)。"""
|
|
94
|
+
out: list[str] = []
|
|
95
|
+
seen: set[str] = set()
|
|
96
|
+
for o in (engine_outcomes or []):
|
|
97
|
+
if not isinstance(o, dict):
|
|
98
|
+
continue
|
|
99
|
+
if str(o.get("status") or "") not in SKIPPED_SOURCE_STATUSES:
|
|
100
|
+
continue
|
|
101
|
+
name = str(o.get("engine") or "?")
|
|
102
|
+
if name not in seen:
|
|
103
|
+
seen.add(name)
|
|
104
|
+
out.append(name)
|
|
105
|
+
if len(out) >= max(0, int(limit)):
|
|
106
|
+
break
|
|
107
|
+
return out
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def zero_result_recovery_note(recovery: Any,
|
|
111
|
+
engine_outcomes: Iterable[dict[str, Any]] | None
|
|
112
|
+
) -> str | None:
|
|
113
|
+
"""恢复链触发但没救回来时的局限声明;不需要声明时返回 None。
|
|
114
|
+
|
|
115
|
+
「没有召回」与「源被跳过/鉴权失败」处置完全不同:前者改写查询,
|
|
116
|
+
后者看 failures 或补凭证。此前一律写成 no recall,实测 bocha 配额
|
|
117
|
+
耗尽(403 not enough money)被报成「查询无召回」,调用方白跑一轮。
|
|
54
118
|
"""
|
|
55
|
-
|
|
119
|
+
if not isinstance(recovery, dict) or not recovery.get("triggered") \
|
|
120
|
+
or recovery.get("recovered"):
|
|
121
|
+
return None
|
|
122
|
+
note = recovery.get("note") or "no note"
|
|
123
|
+
skipped = skipped_sources(engine_outcomes)
|
|
124
|
+
if skipped:
|
|
125
|
+
return (
|
|
126
|
+
"recovery triggered but no usable strategy ("
|
|
127
|
+
f"{note}); empty results came from skipped/failed sources "
|
|
128
|
+
f"({', '.join(skipped)}) — see failures; this is not necessarily "
|
|
129
|
+
"a no-recall query"
|
|
130
|
+
)
|
|
131
|
+
return (
|
|
132
|
+
"recovery triggered but no usable strategy ("
|
|
133
|
+
f"{note}); empty results likely mean the query had no recall — "
|
|
134
|
+
"try rephrasing"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def failure_kinds(engine_outcomes: Iterable[dict[str, Any]] | None,
|
|
139
|
+
limit: int = 6) -> list[str]:
|
|
140
|
+
"""本次调用的失败类别(去重、稳定排序),写进 usage_log 供长期归因。
|
|
141
|
+
|
|
142
|
+
errors 只记「几条」,回答不了「哪类失败在反复出现」;类别来自
|
|
143
|
+
output_transparency.build_failures(与 agent 档 failures 同一口径),
|
|
144
|
+
不在日志侧另写一份分类。
|
|
145
|
+
"""
|
|
146
|
+
try:
|
|
147
|
+
from output_transparency import build_failures
|
|
148
|
+
except ImportError: # pragma: no cover - 单文件分发时的降级
|
|
149
|
+
return []
|
|
150
|
+
rows = build_failures({"engine_outcomes": list(engine_outcomes or [])},
|
|
151
|
+
limit=limit)
|
|
152
|
+
return sorted({str(r.get("category") or "unknown") for r in rows})
|
|
56
153
|
|
|
57
154
|
|
|
58
155
|
def aggregate_slow_engines(rows: Iterable[dict[str, Any]],
|
package/scripts/search_cli.py
CHANGED
|
@@ -328,6 +328,27 @@ def _format_engine_summary(summary: dict[str, Any]) -> str:
|
|
|
328
328
|
return "\n".join(lines)
|
|
329
329
|
|
|
330
330
|
|
|
331
|
+
def _merge_verify_timing(timing: dict[str, Any], verify_ms: float) -> None:
|
|
332
|
+
"""把 verify 的墙钟并回 timing 的三个总数,保持互相自洽。
|
|
333
|
+
|
|
334
|
+
why:verify 发生在主搜索快照之后。此前只并回 stages_ms/process_ms,
|
|
335
|
+
elapsed_ms 仍是主搜索的数(实测 1173 vs 10967),调用方最先读的总账
|
|
336
|
+
比它的分项之和还小 8 倍。三个数必须一起走,否则「这次为什么慢」会被
|
|
337
|
+
带偏到主搜索上。
|
|
338
|
+
"""
|
|
339
|
+
total = float(timing.get("stages_ms") or 0) + verify_ms
|
|
340
|
+
timing.setdefault("stages", []).append({
|
|
341
|
+
"stage": "verify",
|
|
342
|
+
"ms": round(verify_ms, 1),
|
|
343
|
+
"pct": round(verify_ms * 100.0 / total, 1) if total > 0 else 0.0,
|
|
344
|
+
})
|
|
345
|
+
timing["stages_ms"] = round(total, 1)
|
|
346
|
+
timing["process_ms"] = round(
|
|
347
|
+
float(timing.get("process_ms") or 0) + verify_ms, 1)
|
|
348
|
+
timing["elapsed_ms"] = round(
|
|
349
|
+
max(float(timing.get("elapsed_ms") or 0), total), 1)
|
|
350
|
+
|
|
351
|
+
|
|
331
352
|
def main():
|
|
332
353
|
# 入口第一件事:钉住 stdout/stderr 编码(实现唯一,见 cli_io)。
|
|
333
354
|
# 放在这里而不是 import 期:本模块被 search.py 与 bin/argo 两条路调用,
|
|
@@ -579,19 +600,11 @@ def main():
|
|
|
579
600
|
# (process_ms 定格处)之后,净增 280–1160ms 完全不进任何账——实测
|
|
580
601
|
# wall 776–1746ms vs process_ms 494–701ms,「这次为什么慢」归因指错
|
|
581
602
|
# 方向。快照后移做不到(verify 是装配后的可选后处理),补一个 stage
|
|
582
|
-
#
|
|
603
|
+
# 并把耗时并回三个总数(stages_ms / process_ms / elapsed_ms),
|
|
604
|
+
# 三者保持自洽(2026-10-22 补 elapsed_ms)。
|
|
583
605
|
if _timing is not None and "timing" in results and args.verify:
|
|
584
606
|
_verify_ms = (time.perf_counter() - _t_verify0) * 1000.0
|
|
585
|
-
|
|
586
|
-
_total = float(_t.get("stages_ms") or 0) + _verify_ms
|
|
587
|
-
_t.setdefault("stages", []).append({
|
|
588
|
-
"stage": "verify",
|
|
589
|
-
"ms": round(_verify_ms, 1),
|
|
590
|
-
"pct": round(_verify_ms * 100.0 / _total, 1) if _total > 0 else 0.0,
|
|
591
|
-
})
|
|
592
|
-
_t["stages_ms"] = round(_total, 1)
|
|
593
|
-
_t["process_ms"] = round(
|
|
594
|
-
float(_t.get("process_ms") or 0) + _verify_ms, 1)
|
|
607
|
+
_merge_verify_timing(results["timing"], _verify_ms)
|
|
595
608
|
|
|
596
609
|
if args.json_output:
|
|
597
610
|
public = {k: v for k, v in results.items() if not k.startswith("_")}
|
package/scripts/search_entry.py
CHANGED
|
@@ -28,7 +28,7 @@ engine_dispatch。
|
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
30
|
import time
|
|
31
|
-
from
|
|
31
|
+
from search_types import slot_record as dataclass, slot_field as field, slot_replace as replace
|
|
32
32
|
from typing import Any, Callable, Optional
|
|
33
33
|
|
|
34
34
|
from engine_dispatch import run_dispatch
|
|
@@ -361,6 +361,11 @@ def dispatch(req: _SearchRequest, run: _SearchRun, hooks: _SearchHooks) -> _Sear
|
|
|
361
361
|
engine_latency = _dispatch.engine_latency
|
|
362
362
|
wasted_ms = _dispatch.wasted_ms
|
|
363
363
|
useful_ms = _dispatch.useful_ms
|
|
364
|
+
dispatch_tail = {
|
|
365
|
+
"first_sufficient_ms": _dispatch.first_sufficient_ms,
|
|
366
|
+
"tail_wait_ms": _dispatch.tail_wait_ms,
|
|
367
|
+
"abandoned_engine_ms": _dispatch.abandoned_engine_ms,
|
|
368
|
+
}
|
|
364
369
|
early_stopped = _dispatch.early_stopped
|
|
365
370
|
budget_used_ms = _dispatch.budget_used_ms
|
|
366
371
|
budget_total_ms = _dispatch.budget_total_ms
|
|
@@ -378,6 +383,7 @@ def dispatch(req: _SearchRequest, run: _SearchRun, hooks: _SearchHooks) -> _Sear
|
|
|
378
383
|
engine_latency=engine_latency,
|
|
379
384
|
wasted_ms=wasted_ms,
|
|
380
385
|
useful_ms=useful_ms,
|
|
386
|
+
dispatch_tail=dispatch_tail,
|
|
381
387
|
early_stopped=early_stopped,
|
|
382
388
|
budget_used_ms=budget_used_ms,
|
|
383
389
|
budget_total_ms=budget_total_ms,
|
|
@@ -386,4 +392,5 @@ def dispatch(req: _SearchRequest, run: _SearchRun, hooks: _SearchHooks) -> _Sear
|
|
|
386
392
|
quota_batch=quota_batch,
|
|
387
393
|
run_one=_run_one,
|
|
388
394
|
ingest=_ingest,
|
|
395
|
+
stranded=_dispatch.stranded,
|
|
389
396
|
)
|