argo-search 2.9.7 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +73 -150
- package/README.es.md +73 -150
- package/README.ja.md +79 -156
- package/README.ko.md +78 -155
- package/README.md +80 -159
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +78 -5
- package/bin/argo.js +33 -2
- package/config.yaml +269 -63
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +7 -2
- package/packages/dsh-plugin/README.md +25 -6
- package/packages/dsh-plugin/dsh/client.js +402 -47
- package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
- package/packages/dsh-plugin/dsh/index.js +135 -1876
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +3 -3
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +177 -2
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +126 -43
- package/scripts/article_meta.py +131 -0
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +119 -4
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +105 -0
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +58 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +58 -22
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +28 -3
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/http_client.py +24 -38
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +192 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/net_proxy.py +27 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery.py +7 -3
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +26 -5
- package/scripts/route_domains.py +170 -31
- package/scripts/safe_conn.py +52 -0
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +8 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +62 -195
- package/scripts/search_rank.py +6 -0
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +138 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""recovery_runner.py — 搜索主管线错误恢复执行器与 L3 策略。
|
|
3
|
+
|
|
4
|
+
拆分自 search_pipeline.py:将零结果/异常退化恢复链与有界并发对冲执行器从主管线剥离。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from bounded_run import run_bounded
|
|
13
|
+
from except_sets import OPT_IMPORT, SHAPE_BENIGN
|
|
14
|
+
from search_rank import deduplicate_by_url
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run_pipeline_recovery(
|
|
18
|
+
*,
|
|
19
|
+
query: str,
|
|
20
|
+
merged: list[dict[str, Any]],
|
|
21
|
+
raw_results: dict[str, list],
|
|
22
|
+
decision: dict[str, Any],
|
|
23
|
+
engines: list[str],
|
|
24
|
+
available_engines_fn: Any,
|
|
25
|
+
max_results: int,
|
|
26
|
+
max_rec_level: str | None,
|
|
27
|
+
echo_triggered: bool,
|
|
28
|
+
primary_grace_s: float,
|
|
29
|
+
timeout: float,
|
|
30
|
+
depth: str,
|
|
31
|
+
mode: str,
|
|
32
|
+
since_iso: str | None,
|
|
33
|
+
until_iso: str | None,
|
|
34
|
+
skip_cache: bool,
|
|
35
|
+
quota_batch: Any,
|
|
36
|
+
engine_search_fn: Any,
|
|
37
|
+
emit_usage_fn: Any = None,
|
|
38
|
+
recovery_engines_set: set[str] | None = None,
|
|
39
|
+
engine_domain_extra: dict[str, Any] | None = None,
|
|
40
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
41
|
+
"""执行恢复链(L1 放宽 / L2 变体 / L3 换引擎):有界并发对冲跑候选引擎并复检壳。"""
|
|
42
|
+
recovery_info = None
|
|
43
|
+
try:
|
|
44
|
+
from recovery import run_recovery
|
|
45
|
+
tried = list(raw_results.keys()) or list(engines)
|
|
46
|
+
fallback_engines = decision.get("engines_fallback") or []
|
|
47
|
+
|
|
48
|
+
rec_level = max_rec_level
|
|
49
|
+
if echo_triggered:
|
|
50
|
+
rec_level = "L3"
|
|
51
|
+
elif fallback_engines and decision.get("domain") not in (
|
|
52
|
+
None, "", "general", "general_search") \
|
|
53
|
+
and (rec_level is None or rec_level < "L3"):
|
|
54
|
+
rec_level = "L3"
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
enabled_set = set(available_engines_fn())
|
|
58
|
+
except Exception:
|
|
59
|
+
enabled_set = None
|
|
60
|
+
|
|
61
|
+
dk = engine_domain_extra or {}
|
|
62
|
+
|
|
63
|
+
def _fire(eng: str) -> list[dict[str, Any]]:
|
|
64
|
+
if recovery_engines_set is not None:
|
|
65
|
+
recovery_engines_set.add(eng)
|
|
66
|
+
try:
|
|
67
|
+
res = engine_search_fn(
|
|
68
|
+
query, eng, n=max_results,
|
|
69
|
+
timeout=timeout, depth=depth, mode=mode,
|
|
70
|
+
since=since_iso, until=until_iso,
|
|
71
|
+
skip_cache=skip_cache, **dk,
|
|
72
|
+
)
|
|
73
|
+
except Exception as e:
|
|
74
|
+
sys.stderr.write(f"[recovery] 恢复链异常({type(e).__name__}): {e}\n")
|
|
75
|
+
res = []
|
|
76
|
+
goods = [r for r in (res or []) if isinstance(r, dict) and "error" not in r]
|
|
77
|
+
quota_batch.add(eng, bool(goods))
|
|
78
|
+
if goods:
|
|
79
|
+
for r in goods:
|
|
80
|
+
r.setdefault("_engine", eng)
|
|
81
|
+
r.setdefault("_recovered", True)
|
|
82
|
+
try:
|
|
83
|
+
from postprocess_signals import drop_echo_items as _drop_echo
|
|
84
|
+
goods = _drop_echo(goods, query)
|
|
85
|
+
except OPT_IMPORT + SHAPE_BENIGN:
|
|
86
|
+
pass
|
|
87
|
+
return goods
|
|
88
|
+
|
|
89
|
+
def _recovery_executor(rq: str, rengines: list[str]) -> list[dict[str, Any]]:
|
|
90
|
+
if len(rengines) <= 1:
|
|
91
|
+
return _fire(rengines[0]) if rengines else []
|
|
92
|
+
grace = max(0.0, float(primary_grace_s or 0.0))
|
|
93
|
+
wait_s = grace * (len(rengines) - 1) + max(1.0, float(timeout)) + 1.0
|
|
94
|
+
|
|
95
|
+
def _enough(finished: list[tuple[str, Any]]) -> bool:
|
|
96
|
+
return any(isinstance(v, list) and v for _i, v in finished)
|
|
97
|
+
|
|
98
|
+
finished, _unfinished = run_bounded(
|
|
99
|
+
rengines, _fire, wait_s, max_workers=3,
|
|
100
|
+
enough=_enough, stagger_s=grace,
|
|
101
|
+
)
|
|
102
|
+
for _item, value in finished:
|
|
103
|
+
if isinstance(value, list) and value:
|
|
104
|
+
return value
|
|
105
|
+
return []
|
|
106
|
+
|
|
107
|
+
rec_results, rec_result = run_recovery(
|
|
108
|
+
query, tried, _recovery_executor,
|
|
109
|
+
engines_fallback=fallback_engines, enabled=enabled_set, mode=mode,
|
|
110
|
+
max_level=rec_level,
|
|
111
|
+
)
|
|
112
|
+
recovery_info = rec_result.to_dict()
|
|
113
|
+
|
|
114
|
+
if emit_usage_fn is not None:
|
|
115
|
+
try:
|
|
116
|
+
emit_usage_fn("recovery", {
|
|
117
|
+
"query": (query[:60] if query else query),
|
|
118
|
+
"triggered": recovery_info.get("triggered"),
|
|
119
|
+
"recovered": recovery_info.get("recovered"),
|
|
120
|
+
"level_used": recovery_info.get("level_used"),
|
|
121
|
+
"strategy_used": recovery_info.get("strategy_used"),
|
|
122
|
+
"steps_tried": len(recovery_info.get("steps_tried") or []),
|
|
123
|
+
"final_query": (recovery_info.get("final_query") or "")[:60],
|
|
124
|
+
"note": recovery_info.get("note", ""),
|
|
125
|
+
})
|
|
126
|
+
except SHAPE_BENIGN:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
if rec_results:
|
|
130
|
+
try:
|
|
131
|
+
from recovery_l0 import finalize_l3_results
|
|
132
|
+
merged = finalize_l3_results(rec_results, max_results)
|
|
133
|
+
except ImportError:
|
|
134
|
+
merged = deduplicate_by_url(rec_results)[:max_results]
|
|
135
|
+
|
|
136
|
+
rec_by_eng: dict[str, list] = {}
|
|
137
|
+
for r in merged:
|
|
138
|
+
eng = r.get("_engine") or r.get("source") or ""
|
|
139
|
+
if eng:
|
|
140
|
+
r.setdefault("consensus_engines", [eng])
|
|
141
|
+
rec_by_eng.setdefault(eng, []).append(r)
|
|
142
|
+
for eng_name, lst in rec_by_eng.items():
|
|
143
|
+
raw_results.setdefault(eng_name, lst)
|
|
144
|
+
|
|
145
|
+
except ImportError:
|
|
146
|
+
pass
|
|
147
|
+
except Exception as e:
|
|
148
|
+
sys.stderr.write(f"[recovery] 错误恢复跳过: {type(e).__name__}\n")
|
|
149
|
+
|
|
150
|
+
return merged, recovery_info
|
package/scripts/replay_eval.py
CHANGED
|
@@ -534,8 +534,25 @@ def _fmt_header(doc: dict) -> str:
|
|
|
534
534
|
|
|
535
535
|
# ── 命令行 ────────────────────────────────────────────────────────────────────
|
|
536
536
|
|
|
537
|
+
def _read_queries_file(path: str) -> list[str]:
|
|
538
|
+
"""从文件读查询词:每行一个,忽略空行与行首注释。
|
|
539
|
+
|
|
540
|
+
为什么要有这个入口:原来靠 shell 的 `mapfile` 拼参数,在 zsh / 空文件 /
|
|
541
|
+
只有空行三种情况下会静默退化成「录一条空查询」——2026-10-22 本地实跑
|
|
542
|
+
就录出了一条 query="" 的 case,而阈值判定只看 id 交集,于是「一条也算通过」。
|
|
543
|
+
"""
|
|
544
|
+
qs = [line.strip() for line in Path(path).read_text(encoding="utf-8").splitlines()
|
|
545
|
+
if line.strip() and not line.lstrip().startswith("#")]
|
|
546
|
+
return qs
|
|
547
|
+
|
|
548
|
+
|
|
537
549
|
def _cmd_record(args) -> int:
|
|
538
550
|
queries = args.queries or _DEFAULT_QUERIES
|
|
551
|
+
if getattr(args, "queries_file", None):
|
|
552
|
+
queries = _read_queries_file(args.queries_file)
|
|
553
|
+
if not queries:
|
|
554
|
+
print(f"查询集是空的:{args.queries_file}", file=sys.stderr)
|
|
555
|
+
return 1
|
|
539
556
|
out_path = Path(args.out) if args.out else DEFAULT_DATA_PATH
|
|
540
557
|
cases: list[dict] = []
|
|
541
558
|
for q in queries:
|
|
@@ -576,6 +593,7 @@ def main() -> int:
|
|
|
576
593
|
ap.add_argument("--save-baseline", help="把当前结果存为对照基准")
|
|
577
594
|
ap.add_argument("--record", action="store_true", help="联网录制(唯一会联网的模式)")
|
|
578
595
|
ap.add_argument("--queries", nargs="*", help="录制用的查询词")
|
|
596
|
+
ap.add_argument("--queries-file", help="从文件读查询词(每行一个,空行与 # 注释忽略)")
|
|
579
597
|
ap.add_argument("--out", help="录制结果写到哪里")
|
|
580
598
|
ap.add_argument("--mode", default="auto")
|
|
581
599
|
ap.add_argument("--depth", default="fast")
|
package/scripts/research.py
CHANGED
|
@@ -518,7 +518,8 @@ def deep_research(query: str, num_sub_queries: int = 4, max_results: int = 5,
|
|
|
518
518
|
route_strategy: str | None = None,
|
|
519
519
|
work_packages: Any = None,
|
|
520
520
|
allow_recompute: bool = False,
|
|
521
|
-
broaden: bool = False
|
|
521
|
+
broaden: bool = False,
|
|
522
|
+
cross_lingual_trace: bool = False) -> dict[str, Any]:
|
|
522
523
|
"""执行取证。有工作包则按依赖分阶段;否则扩词检索。产出 dossier。"""
|
|
523
524
|
import time as _time
|
|
524
525
|
_deadline = _time.time() + timeout * 3
|
|
@@ -569,36 +570,49 @@ def deep_research(query: str, num_sub_queries: int = 4, max_results: int = 5,
|
|
|
569
570
|
route_strategy=route_strategy,
|
|
570
571
|
)
|
|
571
572
|
|
|
572
|
-
#
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
573
|
+
# ── 跨语言长程搜索与溯源链路(Cross-Lingual Long-Range Trace & Pivot)──
|
|
574
|
+
trace_report: dict[str, Any] | None = None
|
|
575
|
+
try:
|
|
576
|
+
from cross_lingual_trace import (
|
|
577
|
+
assess_cross_lingual_need,
|
|
578
|
+
build_cross_lingual_trace_queries,
|
|
579
|
+
synthesize_cross_lingual_evidence,
|
|
580
|
+
)
|
|
581
|
+
assess = assess_cross_lingual_need(
|
|
582
|
+
original_query,
|
|
583
|
+
collection.get("merged_results", []),
|
|
584
|
+
domain=(profile or {}).get("key") or "",
|
|
585
|
+
force_trace=bool(cross_lingual_trace or broaden),
|
|
586
|
+
)
|
|
587
|
+
should_trace = cross_lingual_trace or (not packages and (broaden or assess["needed"]))
|
|
588
|
+
if should_trace and _time.time() < _deadline:
|
|
589
|
+
trace_subs = []
|
|
590
|
+
for t_lang in assess.get("target_langs", ["en"])[:2]:
|
|
591
|
+
trace_subs.extend(build_cross_lingual_trace_queries(original_query, target_lang=t_lang))
|
|
585
592
|
existing_queries = {sq["query"] for sq in sub_queries}
|
|
586
|
-
|
|
587
|
-
if
|
|
588
|
-
|
|
589
|
-
|
|
593
|
+
trace_subs = [sq for sq in trace_subs if sq["query"] not in existing_queries]
|
|
594
|
+
if trace_subs:
|
|
595
|
+
trace_collection = collect_sources(
|
|
596
|
+
trace_subs, max_results, timeout, depth, mode,
|
|
590
597
|
engines_priority=engines_priority or None,
|
|
591
598
|
profile=profile, budget=budget, route_strategy=route_strategy,
|
|
592
599
|
)
|
|
593
|
-
collection["sub_results"].extend(
|
|
594
|
-
sub_queries.extend(
|
|
595
|
-
|
|
600
|
+
collection["sub_results"].extend(trace_collection["sub_results"])
|
|
601
|
+
sub_queries.extend(trace_subs)
|
|
602
|
+
trace_report = synthesize_cross_lingual_evidence(
|
|
603
|
+
collection.get("merged_results", []),
|
|
604
|
+
trace_collection.get("merged_results", []),
|
|
605
|
+
original_query,
|
|
606
|
+
)
|
|
596
607
|
result_lists = [sr["results"] for sr in collection["sub_results"] if sr.get("results")]
|
|
597
608
|
if len(result_lists) > 1:
|
|
598
609
|
from search_rank import rrf_merge
|
|
599
610
|
collection["merged_results"] = rrf_merge(result_lists)[:max_results * 3]
|
|
600
611
|
elif result_lists:
|
|
601
612
|
collection["merged_results"] = result_lists[0]
|
|
613
|
+
except Exception as _trace_err:
|
|
614
|
+
import logging
|
|
615
|
+
logging.getLogger("unified_search").warning("跨语言长程溯源跳过: %s", _trace_err)
|
|
602
616
|
|
|
603
617
|
gaps = identify_gaps(collection["sub_results"], original_query)
|
|
604
618
|
source_grades = (profile or {}).get("source_grades") if profile else None
|
|
@@ -669,6 +683,8 @@ def deep_research(query: str, num_sub_queries: int = 4, max_results: int = 5,
|
|
|
669
683
|
plan_info, rewrite_result,
|
|
670
684
|
)
|
|
671
685
|
_attach_evidence_loop(report, collection)
|
|
686
|
+
if trace_report is not None:
|
|
687
|
+
report["cross_lingual_trace"] = trace_report
|
|
672
688
|
report["quality_gate_results"] = evaluate_dossier_gates(report)
|
|
673
689
|
report["conclusion_cap"] = report["quality_gate_results"]["conclusion_cap"]
|
|
674
690
|
return report
|
|
@@ -8,6 +8,29 @@ Agent 的 MECE 问题树走 research_work_packages,不要把扩词当研究拆
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import re
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _english_anchor(query: str) -> str:
|
|
14
|
+
"""中英混合查询里的英文锚点;单 token 不足以独立成查询时返回空串。
|
|
15
|
+
|
|
16
|
+
why(2026-10-22 实测):「Rust 异步运行时怎么选」被扩成一条独立的
|
|
17
|
+
`Rust` 子查询,抓回 Rust 游戏官网、语言首页、算法库,15 条来源里 3 条
|
|
18
|
+
与问题无关;「AI 监管 新规」同理会扩成 `AI`。单 token 丢了中文里真正的
|
|
19
|
+
限定语(异步运行时/监管),既浪费一次检索又污染引用。
|
|
20
|
+
判据用「≥2 个词 或 单 token ≥5 字符」:`kubernetes`、`Python`、`Claude Code`
|
|
21
|
+
这类本身可独立检索的保留;`Rust`/`AI`/`DSH`/纯数字年份与股票代码不单飞。
|
|
22
|
+
"""
|
|
23
|
+
tokens = [w for w in query.split()
|
|
24
|
+
if w.isascii() and len(w) > 1 and any(c.isalnum() for c in w)
|
|
25
|
+
and not w.isdigit()]
|
|
26
|
+
text = " ".join(tokens)
|
|
27
|
+
if not text:
|
|
28
|
+
return ""
|
|
29
|
+
if len(tokens) >= 2:
|
|
30
|
+
return text
|
|
31
|
+
return text if len(text) >= 5 else ""
|
|
32
|
+
|
|
33
|
+
|
|
11
34
|
def expand_query(query: str, num_sub: int = 4) -> list[dict[str, str]]:
|
|
12
35
|
"""按关键词特征扩检索词。产出的是搜索变体,不是可验证子问题。"""
|
|
13
36
|
sub_queries: list[dict[str, str]] = []
|
|
@@ -16,7 +39,7 @@ def expand_query(query: str, num_sub: int = 4) -> list[dict[str, str]]:
|
|
|
16
39
|
has_english = any(c.isascii() and c.isalpha() for c in query)
|
|
17
40
|
|
|
18
41
|
if has_chinese and has_english:
|
|
19
|
-
eng_words =
|
|
42
|
+
eng_words = _english_anchor(query)
|
|
20
43
|
if eng_words:
|
|
21
44
|
sub_queries.append({
|
|
22
45
|
"query": eng_words,
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""retry_policy.py — 失败类别与重试判据(唯一来源)。
|
|
3
|
+
|
|
4
|
+
拆出 engine_dispatch 的理由有两条:
|
|
5
|
+
|
|
6
|
+
1. **单一真源**:重试判据(该不该再发一次)与归因判据(这次算什么失败)
|
|
7
|
+
必须用同一套词表与同一个优先级。两处各写一套的下场是重试按一套词表
|
|
8
|
+
跳过、熔断按另一套记账,对同一次失败给出两个不同判断。
|
|
9
|
+
2. **行数门禁**:engine_dispatch 已接近祖父上限(tests/test_module_size_gate.
|
|
10
|
+
py),而本模块是纯策略、无 IO、无闭包状态——正好是「拆出去两边都轻」
|
|
11
|
+
的那一类。
|
|
12
|
+
|
|
13
|
+
本模块不做决定、不碰状态:只把「错误文本 → 类别 → 是否值得重试」算出来。
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from cache_guard import _QUOTA_ERROR_KEYWORDS
|
|
21
|
+
|
|
22
|
+
# 拦截页特征词(唯一来源):error 文本里出现即判 blocked。HTML 引擎的反爬
|
|
23
|
+
# 命中没有 error 文本(静默空结果),走 engines_base 的归因寄存器;
|
|
24
|
+
# 这张表兜住「error 结果里带拦截页字样」的可见路径。
|
|
25
|
+
_BLOCKED_ERROR_KEYWORDS = (
|
|
26
|
+
"just a moment", "checking your browser", "cf-browser-verification",
|
|
27
|
+
"challenge", "ddos-guard", "perimeterx", "access denied",
|
|
28
|
+
"handshake failure", "unable to handshake", "安全验证", "滑动验证",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# 确定性失败:源端状态,不是网络抖动。重试只是多烧一次调用并推迟切备选
|
|
32
|
+
#——配额耗尽重试仍是耗尽,鉴权重试仍是 401,反爬重试仍撞同一堵墙。
|
|
33
|
+
# 与 dsh-free-search 的 failure-aware fallback 同构(quota/auth 冷却到会话
|
|
34
|
+
# 结束、bot-wall 短退避,只有 transient 才重试一次)。
|
|
35
|
+
NON_RETRYABLE_STATUS = frozenset({"quota-exhausted", "auth-failed", "blocked"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def classify_error_status(message: Any) -> str:
|
|
39
|
+
"""错误文本 → 失败类别。**重试判据与归因判据的共同定义点**。
|
|
40
|
+
|
|
41
|
+
优先级顺序即语义强弱:超时最具体(描述本次调用的形态),配额/限流/
|
|
42
|
+
拦截是源端状态,鉴权最后(401/403 也可能来自拦截页)。
|
|
43
|
+
"""
|
|
44
|
+
m = str(message or "").lower()
|
|
45
|
+
if "timeout" in m or "timed out" in m:
|
|
46
|
+
return "timeout"
|
|
47
|
+
if any(k in m for k in _QUOTA_ERROR_KEYWORDS):
|
|
48
|
+
return "quota-exhausted"
|
|
49
|
+
if "rate" in m or "429" in m:
|
|
50
|
+
return "rate-limited"
|
|
51
|
+
if any(k in m for k in _BLOCKED_ERROR_KEYWORDS):
|
|
52
|
+
return "blocked"
|
|
53
|
+
if "auth" in m or "401" in m or "403" in m:
|
|
54
|
+
return "auth-failed"
|
|
55
|
+
return "error"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def is_retryable_failure(res: list[Any] | None) -> bool:
|
|
59
|
+
"""这组结果值不值得再试一次(纯函数,门禁可直接打到)。
|
|
60
|
+
|
|
61
|
+
判据取自 `classify_error_status`,与归因同源。空结果与无错误条目返回
|
|
62
|
+
True——沿用既有「重试一次」语义,本次只收紧确定性失败这一类。
|
|
63
|
+
"""
|
|
64
|
+
for r in res or []:
|
|
65
|
+
if isinstance(r, dict) and "error" in r:
|
|
66
|
+
return classify_error_status(r.get("error")) not in NON_RETRYABLE_STATUS
|
|
67
|
+
return True
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def record_attempts(outcome: dict[str, Any], attempts: dict[str, int]) -> None:
|
|
71
|
+
"""把「实际发了几次网络请求」如实写进 outcome。
|
|
72
|
+
|
|
73
|
+
单独成函数是为了让 engine_dispatch 不必为 3 行记账逻辑长到行数门禁
|
|
74
|
+
外面去;记账口径(至少 1 次)也只在这里定义一次。
|
|
75
|
+
"""
|
|
76
|
+
outcome["attempts"] = max(1, int((attempts or {}).get("n", 1)))
|
package/scripts/route.py
CHANGED
|
@@ -22,7 +22,7 @@ from __future__ import annotations
|
|
|
22
22
|
|
|
23
23
|
import sys
|
|
24
24
|
import time
|
|
25
|
-
from
|
|
25
|
+
from search_types import slot_record as dataclass
|
|
26
26
|
from typing import Any, Callable
|
|
27
27
|
from cli_io import dumps
|
|
28
28
|
|
|
@@ -128,6 +128,7 @@ from route_lang import ( # noqa: E402
|
|
|
128
128
|
_SOCIAL_ZH_GENERAL,
|
|
129
129
|
_get_registry,
|
|
130
130
|
)
|
|
131
|
+
from paid_policy import insert_hedge
|
|
131
132
|
from route_combo import ( # noqa: E402
|
|
132
133
|
_expand_local_search,
|
|
133
134
|
_general_fallback,
|
|
@@ -585,14 +586,11 @@ def _route_by_domain(ctx: _RouteCtx, domain: dict[str, Any], secondary: list[dic
|
|
|
585
586
|
is_catch_all=is_catch_all, enabled=enabled)
|
|
586
587
|
if _strong:
|
|
587
588
|
confidence = 0.9
|
|
588
|
-
# 熔断统一处理 +
|
|
589
|
-
# 判据见 backfill_after_breaker docstring)
|
|
589
|
+
# 熔断统一处理 + 槽位回填(判据见 backfill_after_breaker)
|
|
590
590
|
engines_combo = backfill_after_breaker(
|
|
591
591
|
engines_combo, enabled, features=features, query=query,
|
|
592
592
|
skip_aux=_pure_combo, refill_pool=pre_policy_pool)
|
|
593
|
-
#
|
|
594
|
-
# research 语境例外:子查询跑满 combo(no_early_stop),串行会拖垮
|
|
595
|
-
# 整条研究管线,强制并行
|
|
593
|
+
# parallel 与预算后 combo 长度对齐;research 语境强制并行
|
|
596
594
|
if (mode == "fast" and context != "research") or len(engines_combo) <= 1:
|
|
597
595
|
parallel = False
|
|
598
596
|
elif context == "research":
|
|
@@ -601,6 +599,8 @@ def _route_by_domain(ctx: _RouteCtx, domain: dict[str, Any], secondary: list[dic
|
|
|
601
599
|
# 双引擎默认串行,利于 early-stop(答案域)
|
|
602
600
|
parallel = parallel and len(engines_combo) > 2
|
|
603
601
|
|
|
602
|
+
engines_combo = insert_hedge(engines_combo, domain.get("name", ""), mode, enabled)
|
|
603
|
+
|
|
604
604
|
return _done(
|
|
605
605
|
engine=engines_combo[0],
|
|
606
606
|
engines=engines_combo,
|
package/scripts/route_cache.py
CHANGED
|
@@ -19,6 +19,7 @@ route_cache → route → route_cache 的循环导入。
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
+
import os
|
|
22
23
|
import time
|
|
23
24
|
from typing import Any
|
|
24
25
|
|
|
@@ -100,6 +101,23 @@ def _route_state_fingerprint() -> str:
|
|
|
100
101
|
parts.append("cb=" + ",".join(sorted(get_breaker().auto_disabled())))
|
|
101
102
|
except Exception:
|
|
102
103
|
return ""
|
|
104
|
+
# 路由旋钮的 env 覆盖必须参与指纹:否则「设了 ARGO_GENERAL_FALLBACK_LEAD
|
|
105
|
+
# 却因缓存命中不生效」,逃生开关与 A/B 臂都变成摆设(2026-10-21 实测踩中)。
|
|
106
|
+
# LEGACY 是同一类开关(改通用兜底链的成员),一并入指纹,避免只修一个入口。
|
|
107
|
+
# 统一经 engine_env.get_env 取值(兼顾 ~/.config/argo/env 文件与进程环境)。
|
|
108
|
+
try:
|
|
109
|
+
from engine_env import get_env as _get_env
|
|
110
|
+
except Exception:
|
|
111
|
+
_get_env = lambda k, d="": os.environ.get(k, d) # noqa: E731
|
|
112
|
+
parts.append("lead=" + (_get_env("ARGO_GENERAL_FALLBACK_LEAD") or ""))
|
|
113
|
+
parts.append("legacy=" + (_get_env("ARGO_GENERAL_FALLBACK_LEGACY") or ""))
|
|
114
|
+
# 付费对冲开关与月度预算同样改变 combo;不纳入指纹会让开关或预算变更在缓存命中时静默不生效。
|
|
115
|
+
parts.append("paid=" + (_get_env("ARGO_PAID_SEARCH") or ""))
|
|
116
|
+
try:
|
|
117
|
+
from budget import budget_signature
|
|
118
|
+
parts.append(f"bgt={budget_signature()!r}")
|
|
119
|
+
except Exception:
|
|
120
|
+
pass
|
|
103
121
|
return "|".join(parts)
|
|
104
122
|
|
|
105
123
|
|
package/scripts/route_combo.py
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
+
import os
|
|
15
16
|
from typing import Any
|
|
16
17
|
|
|
17
18
|
from config import get_engines
|
|
@@ -101,6 +102,12 @@ def _general_fallback(enabled: set[str]) -> list[str]:
|
|
|
101
102
|
except ImportError:
|
|
102
103
|
GENERAL_FREE_FALLBACK = ("anysearch", "octen", "local_bing")
|
|
103
104
|
order = list(GENERAL_FREE_FALLBACK)
|
|
105
|
+
# 领队覆盖(2026-10-21):A/B 与线上回退都走这里,避免为了复测改常量。
|
|
106
|
+
# 传引擎名即把它提到首位(例:ARGO_GENERAL_FALLBACK_LEAD=anysearch 复现
|
|
107
|
+
# 2026-10-21 之前的顺位);传空/未知值不生效。与下面的 LEGACY 开关正交。
|
|
108
|
+
lead_override = (os.environ.get("ARGO_GENERAL_FALLBACK_LEAD") or "").strip()
|
|
109
|
+
if lead_override and lead_override in order:
|
|
110
|
+
order = [lead_override] + [e for e in order if e != lead_override]
|
|
104
111
|
# 逃生开关:A/B 与线上回退都走这条,避免为了复测改常量。
|
|
105
112
|
# 2026-10-17 对拍(交错、随机序):octen 入第二顺位后,anysearch 抖动
|
|
106
113
|
# 窗口 p90 3273→1614ms、max 6266→1928ms、错误 9→0;健康窗口中位无差。
|
|
@@ -495,6 +502,20 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
495
502
|
# 学习器在此处首次解析(惰性,见模块顶部 _get_learner):走到这里说明
|
|
496
503
|
# 真的要装配 combo,这笔钱躲不掉;不进本函数的路径(缓存命中等)不付。
|
|
497
504
|
_adaptive_learner = _get_learner()
|
|
505
|
+
|
|
506
|
+
# 场景分桶取分(2026-10-24):优先用 (engine, domain, lang) 的桶分,
|
|
507
|
+
# 桶无样本时 adaptive.get_score 自动回退全局分。判据与旧口径一致,
|
|
508
|
+
# 只是多了一个维度——「引擎在 A 域强、B 域弱」现在可被路由区分。
|
|
509
|
+
_dom_name = str(domain.get("name") or "")
|
|
510
|
+
_q_lang = str((features or {}).get("primary_lang") or "")
|
|
511
|
+
|
|
512
|
+
def _score_of(e: str) -> float:
|
|
513
|
+
try:
|
|
514
|
+
return _adaptive_learner.get_score(e, _dom_name, _q_lang)
|
|
515
|
+
except TypeError:
|
|
516
|
+
# 旧版 adaptive(无 domain/lang 形参)回退全局分
|
|
517
|
+
return _adaptive_learner.get_score(e)
|
|
518
|
+
|
|
498
519
|
combo = domain.get("engines_combo", [])
|
|
499
520
|
primary = domain.get("primary", "anysearch")
|
|
500
521
|
fallback = domain.get("fallback")
|
|
@@ -574,7 +595,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
574
595
|
# 成员与 fallback,自动进入恢复链 L3 兜底。
|
|
575
596
|
if primary and (
|
|
576
597
|
_adaptive_learner is None
|
|
577
|
-
or
|
|
598
|
+
or _score_of(primary) >= _VERTICAL_FLOOR):
|
|
578
599
|
protect.add(primary)
|
|
579
600
|
# 仅 modal_card 整 combo 免 cost 裁剪(结构化路径不可被 anysearch 顶替)
|
|
580
601
|
if domain_name == "modal_card":
|
|
@@ -601,14 +622,14 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
601
622
|
# 自动回归 combo,且它仍是域 fallback(恢复链 L3 候选)。
|
|
602
623
|
if domain_name in _VERTICAL_PROTECT:
|
|
603
624
|
protect = {e for e in original
|
|
604
|
-
if
|
|
625
|
+
if _score_of(e) >= _VERTICAL_FLOOR
|
|
605
626
|
} | protect
|
|
606
627
|
filtered = [
|
|
607
628
|
e for e in filtered
|
|
608
629
|
if e in protect
|
|
609
630
|
or (e == primary
|
|
610
|
-
and
|
|
611
|
-
or
|
|
631
|
+
and _score_of(e) >= _VERTICAL_FLOOR)
|
|
632
|
+
or _score_of(e) >= 0.3
|
|
612
633
|
]
|
|
613
634
|
if not filtered:
|
|
614
635
|
filtered = original
|
|
@@ -655,7 +676,7 @@ def _get_engines_combo(domain: dict[str, Any], enabled: set[str], mode: str = "a
|
|
|
655
676
|
seen_fam.add(f)
|
|
656
677
|
members = [x for x in rest if _fam(x) == f]
|
|
657
678
|
if len(members) > 1:
|
|
658
|
-
scored = [(x,
|
|
679
|
+
scored = [(x, _score_of(x)) for x in members]
|
|
659
680
|
top_score = max(s for _, s in scored)
|
|
660
681
|
laggards = [x for x, s in scored if top_score - s >= 0.15]
|
|
661
682
|
if laggards and len(laggards) < len(scored):
|