argo-search 2.9.7 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +73 -150
- package/README.es.md +73 -150
- package/README.ja.md +79 -156
- package/README.ko.md +78 -155
- package/README.md +80 -159
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +78 -5
- package/bin/argo.js +33 -2
- package/config.yaml +269 -63
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +7 -2
- package/packages/dsh-plugin/README.md +25 -6
- package/packages/dsh-plugin/dsh/client.js +402 -47
- package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
- package/packages/dsh-plugin/dsh/index.js +135 -1876
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +3 -3
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +177 -2
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +126 -43
- package/scripts/article_meta.py +131 -0
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +119 -4
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +105 -0
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +58 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +58 -22
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +28 -3
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/http_client.py +24 -38
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +192 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/net_proxy.py +27 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery.py +7 -3
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +26 -5
- package/scripts/route_domains.py +170 -31
- package/scripts/safe_conn.py +52 -0
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +8 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +62 -195
- package/scripts/search_rank.py +6 -0
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +138 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
|
@@ -27,16 +27,23 @@ import threading
|
|
|
27
27
|
from typing import Any, Callable
|
|
28
28
|
|
|
29
29
|
from cache_guard import (
|
|
30
|
-
_QUOTA_ERROR_KEYWORDS,
|
|
31
30
|
attempt_cache_write,
|
|
32
31
|
is_state_failure_outcome,
|
|
33
32
|
)
|
|
34
33
|
from time_utils import is_time_capable
|
|
34
|
+
from budget import over_budget_error
|
|
35
|
+
from dispatch_result import DispatchResult, summarize_dispatch
|
|
35
36
|
from query_signals import (
|
|
36
37
|
cumulative_sufficient,
|
|
37
38
|
results_sufficient,
|
|
38
39
|
query_coverage_ok,
|
|
39
40
|
)
|
|
41
|
+
from retry_policy import (
|
|
42
|
+
NON_RETRYABLE_STATUS as _NON_RETRYABLE_STATUS,
|
|
43
|
+
classify_error_status,
|
|
44
|
+
is_retryable_failure,
|
|
45
|
+
record_attempts,
|
|
46
|
+
)
|
|
40
47
|
|
|
41
48
|
# 失败原因分类(engine_failure.py 的类别全集)→ 引擎结果 status 的对照表。
|
|
42
49
|
# 未列出的类别经 .get(cat, "error") 归为 error:以后新增失败类别时默认可见,
|
|
@@ -158,14 +165,8 @@ def _adaptive_grace(engine: str, base_grace_s: float) -> float:
|
|
|
158
165
|
return base_grace_s
|
|
159
166
|
return max(0.3, min(base, float(p75_ms) / 1000.0))
|
|
160
167
|
|
|
161
|
-
#
|
|
162
|
-
#
|
|
163
|
-
# 这张表兜住「error 结果里带拦截页字样」的可见路径。
|
|
164
|
-
_BLOCKED_ERROR_KEYWORDS = (
|
|
165
|
-
"just a moment", "checking your browser", "cf-browser-verification",
|
|
166
|
-
"challenge", "ddos-guard", "perimeterx", "access denied",
|
|
167
|
-
"handshake failure", "unable to handshake", "安全验证", "滑动验证",
|
|
168
|
-
)
|
|
168
|
+
# 失败类别与重试判据住在 retry_policy(唯一来源):重试判据与归因判据必须
|
|
169
|
+
# 同源,而本文件已接近行数上限——纯策略搬出去,两边都轻。
|
|
169
170
|
|
|
170
171
|
|
|
171
172
|
def classify_engine_outcome(eng: str, res: list[dict[str, Any]],
|
|
@@ -185,19 +186,7 @@ def classify_engine_outcome(eng: str, res: list[dict[str, Any]],
|
|
|
185
186
|
errors = [r for r in res if isinstance(r, dict) and "error" in r]
|
|
186
187
|
goods = [r for r in res if isinstance(r, dict) and "error" not in r]
|
|
187
188
|
if errors and not goods:
|
|
188
|
-
|
|
189
|
-
if "timeout" in msg:
|
|
190
|
-
st = "timeout"
|
|
191
|
-
elif any(k in msg for k in _QUOTA_ERROR_KEYWORDS):
|
|
192
|
-
st = "quota-exhausted"
|
|
193
|
-
elif "rate" in msg or "429" in msg:
|
|
194
|
-
st = "rate-limited"
|
|
195
|
-
elif any(k in msg for k in _BLOCKED_ERROR_KEYWORDS):
|
|
196
|
-
st = "blocked"
|
|
197
|
-
elif "auth" in msg or "401" in msg or "403" in msg:
|
|
198
|
-
st = "auth-failed"
|
|
199
|
-
else:
|
|
200
|
-
st = "error"
|
|
189
|
+
st = classify_error_status(errors[0].get("error"))
|
|
201
190
|
return {
|
|
202
191
|
"engine": eng, "status": st,
|
|
203
192
|
"results_count": 0, "latency_ms": latency_ms,
|
|
@@ -214,39 +203,6 @@ def classify_engine_outcome(eng: str, res: list[dict[str, Any]],
|
|
|
214
203
|
}
|
|
215
204
|
|
|
216
205
|
|
|
217
|
-
class DispatchResult:
|
|
218
|
-
"""编排产出 + 两个钩子(融合后仍要补搜的路径用)。
|
|
219
|
-
|
|
220
|
-
`run_one` / `ingest` 必须外露:macro_data 域的 D6 证据下限补搜在融合之后,
|
|
221
|
-
它要跑单个引擎并把结果并进同一份账(raw_results / engine_outcomes /
|
|
222
|
-
engine_latency / wasted 计数),另起一份会让 engines_used 与配额记账对不上。
|
|
223
|
-
|
|
224
|
-
`budget_used_ms` / `budget_total_ms`:本次编排的实际墙钟消耗与总预算
|
|
225
|
-
(deep 等无预算模式 total 为 None)。搜索输出把它挂进 timing.budget,
|
|
226
|
-
让「这次离预算上限还有多远」可观测,而不必反推 process_ms。
|
|
227
|
-
|
|
228
|
-
"""
|
|
229
|
-
|
|
230
|
-
__slots__ = ("raw_results", "engine_outcomes", "engine_latency",
|
|
231
|
-
"wasted_ms", "early_stopped", "run_one", "ingest",
|
|
232
|
-
"budget_used_ms", "budget_total_ms", "useful_ms")
|
|
233
|
-
|
|
234
|
-
def __init__(self, raw_results, engine_outcomes, engine_latency,
|
|
235
|
-
wasted_ms, early_stopped, run_one, ingest,
|
|
236
|
-
budget_used_ms=None, budget_total_ms=None,
|
|
237
|
-
useful_ms: int = 0) -> None:
|
|
238
|
-
self.raw_results = raw_results
|
|
239
|
-
self.engine_outcomes = engine_outcomes
|
|
240
|
-
self.engine_latency = engine_latency
|
|
241
|
-
self.wasted_ms = wasted_ms
|
|
242
|
-
self.early_stopped = early_stopped
|
|
243
|
-
self.run_one = run_one
|
|
244
|
-
self.ingest = ingest
|
|
245
|
-
self.budget_used_ms = budget_used_ms
|
|
246
|
-
self.budget_total_ms = budget_total_ms
|
|
247
|
-
self.useful_ms = useful_ms
|
|
248
|
-
|
|
249
|
-
|
|
250
206
|
def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
251
207
|
decision: dict[str, Any], parallel: bool,
|
|
252
208
|
domain: str, mode: str, depth: str,
|
|
@@ -279,6 +235,8 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
279
235
|
raw_results: dict[str, list[dict[str, Any]]] = {}
|
|
280
236
|
engine_outcomes: list[dict[str, Any]] = []
|
|
281
237
|
engine_latency: dict[str, int] = {}
|
|
238
|
+
# 每引擎实际发出的网络请求数(含重试与 balanced 补跑)——_run_one 收账用。
|
|
239
|
+
_attempt_counts: dict[str, int] = {}
|
|
282
240
|
|
|
283
241
|
# 远端配额耗尽快照(route_combo 的 F7 已在 combo 层排除 hard-down
|
|
284
242
|
# 引擎;本快照供下方 per-engine 闸口兜底主路由/恢复链/显式 engine=
|
|
@@ -399,8 +357,14 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
399
357
|
since=since_iso, until=until_iso, skip_cache=skip_cache,
|
|
400
358
|
**_engine_domain_kwargs(),
|
|
401
359
|
)
|
|
360
|
+
_attempt_counts[eng] = _attempt_counts.get(eng, 0) + 1
|
|
402
361
|
if last_result and any("error" not in r for r in last_result):
|
|
403
362
|
return last_result
|
|
363
|
+
# 确定性失败不重试:配额/鉴权/反爬是源端状态,再发一次仍是同样
|
|
364
|
+
# 的结果,只多烧一次调用并推迟切备选(见 is_retryable_failure)。
|
|
365
|
+
# transient 与空结果沿用既有「重试一次」语义。
|
|
366
|
+
if not is_retryable_failure(last_result):
|
|
367
|
+
return last_result
|
|
404
368
|
# 慢源(retries=0,超时即弃)不再用 balanced 补跑,避免超时场景双倍耗时
|
|
405
369
|
if retries > 0 and depth != "balanced":
|
|
406
370
|
_remain = _eng_budget - (_now() - _t_eng_start)
|
|
@@ -412,6 +376,7 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
412
376
|
since=since_iso, until=until_iso, skip_cache=skip_cache,
|
|
413
377
|
**_engine_domain_kwargs(),
|
|
414
378
|
)
|
|
379
|
+
_attempt_counts[eng] = _attempt_counts.get(eng, 0) + 1
|
|
415
380
|
return last_result
|
|
416
381
|
|
|
417
382
|
def _run_one(eng: str) -> tuple[str, list[dict[str, Any]], dict[str, Any], int]:
|
|
@@ -493,6 +458,14 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
493
458
|
outcome["status"] = "ok-cached" if eng_hit else "no-results-cached"
|
|
494
459
|
return eng, eng_hit, outcome, lat
|
|
495
460
|
|
|
461
|
+
# 预算:缓存命中不收费,放在缓存之后;判定/文案在 budget.py。
|
|
462
|
+
_budget_err = over_budget_error(eng)
|
|
463
|
+
if _budget_err:
|
|
464
|
+
lat = int((_now() - t_eng) * 1000)
|
|
465
|
+
outcome = classify_outcome(eng, [], lat, status_hint="skipped-budget")
|
|
466
|
+
outcome["detail"] = _budget_err
|
|
467
|
+
return eng, [], outcome, lat
|
|
468
|
+
|
|
496
469
|
# 网络调用
|
|
497
470
|
# 答案型域(early_min 存在,1 条快照即可交付)的慢源收紧超时:
|
|
498
471
|
# FRED/Eurostat 这类 timeout=10s 的源一旦挂掉就阻塞整条串行路径,
|
|
@@ -534,6 +507,9 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
534
507
|
r.setdefault("_elapsed", lat / 1000.0)
|
|
535
508
|
|
|
536
509
|
outcome = classify_outcome(eng, res, lat)
|
|
510
|
+
# 如实记账:本次实际发了几次网络请求(重试 + balanced 补跑)。
|
|
511
|
+
# 「重试了一次」此前不可见——与「早停 + wasted_ms 如实记账」同一纪律。
|
|
512
|
+
record_attempts(outcome, {"n": _attempt_counts.get(eng, 1)})
|
|
537
513
|
# 失败原因合入:引擎内部那些不报错的失败路径(反爬命中/HTTP 状态码/网络异常)
|
|
538
514
|
# 没有 error 文本,结果会落成 no-results;失败原因记录把它们还原成
|
|
539
515
|
# 真实状态,供熔断与 --json 可观测面使用。
|
|
@@ -617,19 +593,21 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
617
593
|
raw_results[eng] = res
|
|
618
594
|
engine_outcomes.append(outcome)
|
|
619
595
|
engine_latency[eng] = lat
|
|
620
|
-
# 完成时刻(相对本次编排起点):wasted 的真值来源,见函数尾部的记账
|
|
621
596
|
engine_done_ms[eng] = (_now() - _budget_base) * 1000.0
|
|
622
597
|
if outcome["status"] in _CONTRIBUTING_STATUS:
|
|
623
598
|
contributed_ms.append(engine_done_ms[eng])
|
|
624
599
|
|
|
625
600
|
engine_done_ms: dict[str, float] = {}
|
|
626
601
|
contributed_ms: list[float] = []
|
|
627
|
-
#
|
|
628
|
-
#
|
|
629
|
-
# lat_ms 记进 engine_latency 就丢掉——「答案就绪后还在等」这个口径在
|
|
630
|
-
# 有界并发下恒为 0(最后完成的引擎就是触发早停的那个,wall==useful),
|
|
631
|
-
# 仪表盘因此永远显示 0,无法据此调 tail_grace / hedge / 并发上限。
|
|
602
|
+
# 早停弃置引擎已花的墙钟(并行资源浪费,不是用户等待);记账口径见
|
|
603
|
+
# dispatch_result 模块注释。
|
|
632
604
|
abandoned_ms: list[float] = []
|
|
605
|
+
# 早停弃置且仍存活的 (engine, holder, now_fn):交 tail_completion 后台
|
|
606
|
+
# 收割(2026-10-25)。now_fn 跟 _now 走——t0_mono 传入时整套是单调钟,
|
|
607
|
+
# 收割线程不能用墙钟去减 holder["t0"]。
|
|
608
|
+
stranded: list[tuple[str, dict[str, Any], Any]] = []
|
|
609
|
+
first_result_at: float | None = None
|
|
610
|
+
first_sufficient_at: float | None = None
|
|
633
611
|
# 配额批次的构造在调用方:flush 必须发生在**最后一个 _ingest 之后**,
|
|
634
612
|
# 而那个点在融合后的 D6 补搜里(本模块之外)。在这里另建一个实例,
|
|
635
613
|
# 调用方 flush 到的就是空批次——补搜引擎的记账永远落不了盘。
|
|
@@ -725,6 +703,10 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
725
703
|
# 弃置的引擎白花的墙钟计入 wasted(见 abandoned_ms):
|
|
726
704
|
# 它是「调度多花了多少」的量,不是「多等了多久」。
|
|
727
705
|
abandoned_ms.append(lat_ms)
|
|
706
|
+
# 弃置≠丢弃交付:holder 交给后台收割(宽限内完成则记
|
|
707
|
+
# 学习器真实成绩 + 合并写缓存,见 tail_completion)。
|
|
708
|
+
# 主响应与上面这笔 cancelled 记账逐位不变。
|
|
709
|
+
stranded.append((eng, holder, _now))
|
|
728
710
|
_ingest(eng, [],
|
|
729
711
|
classify_outcome(eng, [], lat_ms, "cancelled"),
|
|
730
712
|
lat_ms)
|
|
@@ -787,6 +769,7 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
787
769
|
"""
|
|
788
770
|
if not engs:
|
|
789
771
|
return False
|
|
772
|
+
nonlocal first_result_at, first_sufficient_at
|
|
790
773
|
queue = list(engs)
|
|
791
774
|
pending: list[tuple[dict[str, Any], Any, str]] = []
|
|
792
775
|
deadline = _now() + max(0.0, wait_s)
|
|
@@ -817,7 +800,10 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
817
800
|
pending.remove(item)
|
|
818
801
|
progressed = True
|
|
819
802
|
goods = _holder_goods(holder)
|
|
803
|
+
if goods and first_result_at is None:
|
|
804
|
+
first_result_at = _now()
|
|
820
805
|
if stop is not None and stop(eng, goods):
|
|
806
|
+
first_sufficient_at = _now()
|
|
821
807
|
_settle_pending(pending, status="cancelled")
|
|
822
808
|
return True
|
|
823
809
|
if goods and tail_grace_s > 0.0:
|
|
@@ -985,35 +971,21 @@ def run_dispatch(*, query: str, retrieval_query: str, engines: list[str],
|
|
|
985
971
|
tail_grace_fn=straggler_tail_fn):
|
|
986
972
|
early_stopped = True
|
|
987
973
|
|
|
988
|
-
|
|
989
|
-
|
|
990
974
|
budget_total_ms = int(budget_s * 1000) if budget_s is not None else None
|
|
991
|
-
#
|
|
992
|
-
# 就已经在手里了)。`wasted_ms` 见下方口径说明。
|
|
993
|
-
#
|
|
994
|
-
# 此前的 wasted 是「所有非 ok 引擎的 latency 之和」——它根本不是时间量:
|
|
995
|
-
# 并行时必然大于墙钟(实测「世界杯 2026 主办国」wasted 2847ms > dispatch
|
|
996
|
-
# 2313ms),而且把「引擎如实返回了 0 条」也算成浪费。两个已知代价:
|
|
997
|
-
# ① 上一轮的优化盘点据此写下「早停与 hedge race 记账诚实、调度不用动」,
|
|
998
|
-
# 而实测调度是最大的单一浪费源;② 读者无法从它推出任何行动。
|
|
999
|
-
#
|
|
1000
|
-
# 再往后改成 `wall - useful`(useful = 最后一个**贡献结果**的引擎的完成
|
|
1001
|
-
# 时刻),并宣称口径是「答案就绪后还在等」。但有界并发下这仍是恒等式:
|
|
1002
|
-
# 触发早停的引擎自己就是最后完成的那个,返回当刻 wall == useful,于是
|
|
1003
|
-
# wasted 永远是 0——实测每轮皆 0。空转的仪表比没有仪表更糟:它让人以为
|
|
1004
|
-
# 调度无浪费可调,于是谁都不会去调 tail_grace / hedge / 并发上限。
|
|
1005
|
-
#
|
|
1006
|
-
# 现在的口径:**被编排主动弃置的引擎已经花掉的墙钟之和**。
|
|
1007
|
-
# 这些时间确实白花(没换来结果、也没拖住返回),是唯一既能算清、
|
|
1008
|
-
# 又能指向具体动作的量;与 wall 无关(弃置不阻塞返回),故不再保证
|
|
1009
|
-
# useful + wasted == wall。仍被等待的超时引擎不算浪费——那是引擎不健康,
|
|
1010
|
-
# 在 engine_latency / 熔断里已可见。
|
|
975
|
+
# 记账口径见 dispatch_result 模块注释;wasted_ms 保留为 abandoned 别名。
|
|
1011
976
|
wall_ms = int((_now() - _budget_base) * 1000)
|
|
1012
|
-
|
|
977
|
+
_acc = summarize_dispatch(
|
|
978
|
+
wall_ms=wall_ms, contributed_ms=contributed_ms,
|
|
979
|
+
first_result_at=first_result_at, first_sufficient_at=first_sufficient_at,
|
|
980
|
+
base=_budget_base, abandoned_ms=abandoned_ms)
|
|
1013
981
|
return DispatchResult(
|
|
1014
982
|
raw_results, engine_outcomes, engine_latency,
|
|
1015
|
-
|
|
983
|
+
_acc["abandoned_engine_ms"], early_stopped, _run_one, _ingest,
|
|
1016
984
|
budget_used_ms=wall_ms,
|
|
1017
985
|
budget_total_ms=budget_total_ms,
|
|
1018
|
-
useful_ms=useful_ms,
|
|
986
|
+
useful_ms=_acc["useful_ms"],
|
|
987
|
+
first_sufficient_ms=_acc["first_sufficient_ms"],
|
|
988
|
+
tail_wait_ms=_acc["tail_wait_ms"],
|
|
989
|
+
abandoned_engine_ms=_acc["abandoned_engine_ms"],
|
|
990
|
+
stranded=stranded,
|
|
1019
991
|
)
|
package/scripts/engine_env.py
CHANGED
|
@@ -186,6 +186,10 @@ KNOWN_ENV_ALIASES: dict[str, list[str]] = {
|
|
|
186
186
|
"firecrawl": ["ARGO_FIRECRAWL_API_KEY", "FIRECRAWL_API_KEY"], # 可选(keyless 免费层)
|
|
187
187
|
"weread": ["ARGO_WEREAD_API_KEY", "WEREAD_API_KEY"], # 微信读书 Agent Gateway
|
|
188
188
|
"em_miaoxiang": ["ARGO_EASTMONEY_APIKEY", "EASTMONEY_APIKEY"], # 东财妙想(可选)
|
|
189
|
+
# 腾讯元宝搜索:官方文档用的名字是 TENCENTCLOUD_WSA_APIKEY,
|
|
190
|
+
# argo 推荐名在前;两个都认,便于照文档配的用户直接可用。
|
|
191
|
+
"yuanbao": ["ARGO_YUANBAO_API_KEY", "TENCENTCLOUD_WSA_APIKEY",
|
|
192
|
+
"YUANBAO_API_KEY"],
|
|
189
193
|
}
|
|
190
194
|
|
|
191
195
|
# 占位符名 → 候选 env(用于 config 中 {TAVILY_API_KEY} 展开)
|
|
@@ -215,6 +219,7 @@ _NON_SECRET_PLACEHOLDERS = {"QUERY", "N", "TIMESTAMP", "MODE", "DEPTH", "UUID"}
|
|
|
215
219
|
|
|
216
220
|
# 可选密钥:有则更好,缺失不阻断自动路由
|
|
217
221
|
OPTIONAL_ENV_ENGINES: set[str] = {
|
|
222
|
+
"exa", # 无 key 走官方 MCP 匿名通道(engines_builders_tech._exa_mcp_keyless)
|
|
218
223
|
"github", # GITHUB_TOKEN 仅提高限频
|
|
219
224
|
"anysearch", # ANYSEARCH_API_KEY 可选
|
|
220
225
|
"firecrawl", # FIRECRAWL_API_KEY 可选:官方免费层匿名(keyless)即可搜索
|
|
@@ -112,12 +112,17 @@ _ENGINE_FAMILY_OVERRIDES: dict[str, str] = {
|
|
|
112
112
|
"eastmoney": "finance_market",
|
|
113
113
|
"cninfo": "finance_market",
|
|
114
114
|
"seeking_alpha": "finance_market",
|
|
115
|
+
# 2026-10-21 批次十四:一手行情/公告
|
|
116
|
+
"hkex": "finance_market", # 港交所披露易公告
|
|
117
|
+
"shfe": "finance_market", # 上期所商品日行情
|
|
118
|
+
"defillama": "finance_market", # 协议 TVL(链上资金规模)
|
|
115
119
|
# 宏观
|
|
116
120
|
"fred": "finance_macro",
|
|
117
121
|
"worldbank": "finance_macro",
|
|
118
122
|
"nbs_stats": "finance_macro",
|
|
119
123
|
"eurostat": "finance_macro",
|
|
120
124
|
"fx_rate": "finance_macro",
|
|
125
|
+
"cfets": "finance_macro", # 人民币中间价(官方在岸口径)
|
|
121
126
|
# 快讯
|
|
122
127
|
"cls_telegraph": "news_flash",
|
|
123
128
|
"em_global_news": "news_flash",
|
|
@@ -250,6 +255,7 @@ _ENGINE_FAMILY_OVERRIDES: dict[str, str] = {
|
|
|
250
255
|
# 百科/实体
|
|
251
256
|
"ror": "knowledge", # 研究机构标识
|
|
252
257
|
"opencorporates": "knowledge",
|
|
258
|
+
"gleif": "knowledge", # 全球法人 LEI 注册库
|
|
253
259
|
"wikisource": "knowledge",
|
|
254
260
|
"zdic": "knowledge", # 汉典:字义/音韵/字源
|
|
255
261
|
# 媒体/图书/艺术
|
package/scripts/engine_policy.py
CHANGED
|
@@ -51,13 +51,13 @@ FORCE_DAILY_CORE: frozenset[str] = frozenset({
|
|
|
51
51
|
# 排在通用检索源末位 + spec 的 tier=daily_support,日常 fast/auto 的 combo 预算
|
|
52
52
|
# (2/3 个)够不到它,只在 deep/research 无截断时参战,共享免费层不被烧穿。
|
|
53
53
|
GENERAL_FREE_FALLBACK: tuple[str, ...] = (
|
|
54
|
-
# 2026-10-
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
"
|
|
60
|
-
"parallel_free", "wikipedia",
|
|
54
|
+
# 2026-10-21 A/B:octen 顶到第一位。交错对拍(ab_eval,4 对/查询、
|
|
55
|
+
# --no-cache)里 octen 对 anysearch 8:0 全胜:露营 天幕 p90 6313→835ms、
|
|
56
|
+
# asyncio p90 2769→963ms,结果数中位同为 8。生产日志同向:general_search
|
|
57
|
+
# 346 条查询里 324 条的最后一棒是 anysearch(p50 1932ms),octen 同域
|
|
58
|
+
# p50 776ms。anysearch 退到对冲位,能力面不变(仍在 combo 内)。
|
|
59
|
+
"octen", "anysearch", "local_bing", "uapi", "local_baidu", "firecrawl",
|
|
60
|
+
"parallel_free", "tavily_free", "keenable_free", "wikipedia",
|
|
61
61
|
)
|
|
62
62
|
|
|
63
63
|
|
package/scripts/engines.py
CHANGED
|
@@ -60,6 +60,7 @@ __all__ = [
|
|
|
60
60
|
"_parse_semantic_scholar",
|
|
61
61
|
"_ensure_engine_source",
|
|
62
62
|
]
|
|
63
|
+
from engines_builders_keyless import _build_keenable_free_engine
|
|
63
64
|
from engines_builders import (
|
|
64
65
|
_build_who_don_engine,
|
|
65
66
|
_build_who_gho_engine,
|
|
@@ -169,6 +170,11 @@ from engines_builders import (
|
|
|
169
170
|
_build_yahoo_finance_engine,
|
|
170
171
|
_build_sec_companyfacts_engine,
|
|
171
172
|
_build_frankfurter_engine,
|
|
173
|
+
_build_hkex_engine,
|
|
174
|
+
_build_shfe_engine,
|
|
175
|
+
_build_cfets_engine,
|
|
176
|
+
_build_defillama_engine,
|
|
177
|
+
_build_yuanbao_engine,
|
|
172
178
|
_build_seeking_alpha_engine,
|
|
173
179
|
_build_qweather_engine,
|
|
174
180
|
_build_wenshu_engine,
|
|
@@ -393,6 +399,7 @@ _BUILDERS = {
|
|
|
393
399
|
"html": _build_html_engine,
|
|
394
400
|
"local_search": _build_local_search_engine,
|
|
395
401
|
"exa": _build_exa_engine,
|
|
402
|
+
"keenable_free": _build_keenable_free_engine,
|
|
396
403
|
"anysearch": _build_anysearch_engine,
|
|
397
404
|
"em_miaoxiang": _build_em_miaoxiang_engine,
|
|
398
405
|
"cninfo": _build_cninfo_engine,
|
|
@@ -471,6 +478,11 @@ _BUILDERS = {
|
|
|
471
478
|
"yahoo_finance": _build_yahoo_finance_engine,
|
|
472
479
|
"sec_companyfacts": _build_sec_companyfacts_engine,
|
|
473
480
|
"frankfurter": _build_frankfurter_engine,
|
|
481
|
+
"hkex": _build_hkex_engine,
|
|
482
|
+
"shfe": _build_shfe_engine,
|
|
483
|
+
"cfets": _build_cfets_engine,
|
|
484
|
+
"defillama": _build_defillama_engine,
|
|
485
|
+
"yuanbao": _build_yuanbao_engine,
|
|
474
486
|
"seeking_alpha": _build_seeking_alpha_engine,
|
|
475
487
|
"qweather": _build_qweather_engine,
|
|
476
488
|
"wenshu": _build_wenshu_engine,
|
|
@@ -845,6 +857,15 @@ def search(query: str, engine: str, n: int = 5, timeout: float = 8, depth: str =
|
|
|
845
857
|
except Exception:
|
|
846
858
|
results = []
|
|
847
859
|
results = results or []
|
|
860
|
+
# SERP 相关度守卫统一在引擎执行出口判定。放在 builder 里会漏掉
|
|
861
|
+
# people_daily 这类模糊搜索 API:它匹配不到时回整页默认 feed,形态是
|
|
862
|
+
# 正常结果,只有按 query 覆盖率才判得出来(issue #18 复现 A)。
|
|
863
|
+
# builder 已经判过垃圾的 local_* 这里拿到空列表,按早退语义不重复记账。
|
|
864
|
+
try:
|
|
865
|
+
from serp_kill_log import serp_guard_apply
|
|
866
|
+
results = serp_guard_apply(engine, query, results)
|
|
867
|
+
except ImportError:
|
|
868
|
+
pass
|
|
848
869
|
# 桶化后可能多于请求数:调用方按请求 n 截断语义由 execute_search 融合层负责;
|
|
849
870
|
# 此处保留全量(缓存键按 eff_n 存,命中时同样可截)。
|
|
850
871
|
return results
|
package/scripts/engines_base.py
CHANGED
|
@@ -384,11 +384,11 @@ def _build_cli_engine(spec: dict[str, Any]) -> Any:
|
|
|
384
384
|
args = _resolve(search_args, query, n, mode=mode, **kwargs)
|
|
385
385
|
if not cmd:
|
|
386
386
|
return []
|
|
387
|
-
#
|
|
387
|
+
# 跨平台解释器:优先 sys.executable(bin/argo 已验证它带 PyYAML);
|
|
388
|
+
# PATH 上的 python3 未必是同一解释器,见 tests/test_cli_engine_interpreter.py
|
|
388
389
|
if isinstance(cmd, list) and cmd and cmd[0] == "python3":
|
|
389
390
|
import shutil # 仅 CLI 引擎解析解释器时需要,不进模块级导入链
|
|
390
|
-
|
|
391
|
-
cmd[0] = py3
|
|
391
|
+
cmd[0] = sys.executable or shutil.which("python3") or shutil.which("python")
|
|
392
392
|
for key, tmpl in filter_args.items():
|
|
393
393
|
if kwargs.get(key) not in (None, ""):
|
|
394
394
|
args += _resolve(tmpl, query, n, mode=mode, **kwargs)
|
|
@@ -28,6 +28,9 @@ from engines_builders_batch9 import (
|
|
|
28
28
|
_build_tle_mirror_engine,
|
|
29
29
|
_build_nhtsa_vpic_engine,
|
|
30
30
|
)
|
|
31
|
+
from engines_builders_keyless import (
|
|
32
|
+
_build_keenable_free_engine,
|
|
33
|
+
)
|
|
31
34
|
from engines_builders_tech import (
|
|
32
35
|
_build_exa_engine,
|
|
33
36
|
_build_anysearch_engine,
|
|
@@ -85,6 +88,15 @@ from engines_builders_batch13 import (
|
|
|
85
88
|
_build_sec_companyfacts_engine,
|
|
86
89
|
_build_frankfurter_engine,
|
|
87
90
|
)
|
|
91
|
+
from engines_builders_batch14 import (
|
|
92
|
+
_build_hkex_engine,
|
|
93
|
+
_build_shfe_engine,
|
|
94
|
+
_build_cfets_engine,
|
|
95
|
+
_build_defillama_engine,
|
|
96
|
+
)
|
|
97
|
+
from engines_builders_batch15 import (
|
|
98
|
+
_build_yuanbao_engine,
|
|
99
|
+
)
|
|
88
100
|
from engines_builders_data import (
|
|
89
101
|
_build_open_library_engine,
|
|
90
102
|
_build_weread_engine,
|
|
@@ -20,6 +20,7 @@ import time
|
|
|
20
20
|
import urllib.error
|
|
21
21
|
import urllib.parse
|
|
22
22
|
import urllib.request
|
|
23
|
+
from datetime import datetime
|
|
23
24
|
from typing import Any
|
|
24
25
|
|
|
25
26
|
from engine_env import get_env
|
|
@@ -69,6 +70,42 @@ def _zhihu_error_item(code: Any, message: Any) -> dict[str, Any]:
|
|
|
69
70
|
return {"error": text, "source": "zhihu_global"}
|
|
70
71
|
|
|
71
72
|
|
|
73
|
+
def _zhihu_epoch(value: Any) -> int | None:
|
|
74
|
+
"""ISO 时间窗 → 秒级时间戳;空/不可解析返回 None(不阻断检索)。"""
|
|
75
|
+
raw = str(value or "").strip()
|
|
76
|
+
if not raw:
|
|
77
|
+
return None
|
|
78
|
+
try:
|
|
79
|
+
return int(datetime.fromisoformat(raw.replace("Z", "+00:00")).timestamp())
|
|
80
|
+
except (TypeError, ValueError):
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_DB_OPERATOR_RE = re.compile(r"(?:^|\s)db:(all|realtime|static)(?=\s|$)", re.I)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _parse_db_operator(query: str) -> tuple[str, str]:
|
|
88
|
+
"""解析 `db:realtime|static|all` 查询操作符,返回 (db, 剩余查询)。"""
|
|
89
|
+
match = _DB_OPERATOR_RE.search(query or "")
|
|
90
|
+
if not match:
|
|
91
|
+
return "all", query
|
|
92
|
+
db = match.group(1).lower()
|
|
93
|
+
rest = _DB_OPERATOR_RE.sub(" ", query, count=1).strip()
|
|
94
|
+
return db, rest or (query or "")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _build_zhihu_publish_filter(since: Any, until: Any) -> str:
|
|
98
|
+
"""把 --since/--until 下推成官方 Filter(publish_time 支持 >=/<=)。"""
|
|
99
|
+
since_ts = _zhihu_epoch(since)
|
|
100
|
+
until_ts = _zhihu_epoch(until)
|
|
101
|
+
parts = []
|
|
102
|
+
if since_ts is not None:
|
|
103
|
+
parts.append(f"publish_time>={since_ts}")
|
|
104
|
+
if until_ts is not None:
|
|
105
|
+
parts.append(f"publish_time<={until_ts}")
|
|
106
|
+
return " AND ".join(parts)
|
|
107
|
+
|
|
108
|
+
|
|
72
109
|
def _build_zhihu_global_engine(spec: dict[str, Any]) -> Any:
|
|
73
110
|
"""知乎开放平台全网搜索(developer.zhihu.com global_search)。
|
|
74
111
|
|
|
@@ -92,12 +129,20 @@ def _build_zhihu_global_engine(spec: dict[str, Any]) -> Any:
|
|
|
92
129
|
|
|
93
130
|
# 解析 site:/host: 站点限定语法 → Filter: host=="..."(实现单点在 cn)
|
|
94
131
|
filter_expr, search_query = _parse_site_filter(query)
|
|
95
|
-
|
|
96
|
-
|
|
132
|
+
# db:realtime|static|all:官方 SearchDB 选择(默认 all)
|
|
133
|
+
db, search_query = _parse_db_operator(search_query or query)
|
|
134
|
+
# --since/--until → 官方 publish_time 过滤(文档 2026-10 正式定义)
|
|
135
|
+
_time_filter = _build_zhihu_publish_filter(kwargs.get("since"),
|
|
136
|
+
kwargs.get("until"))
|
|
137
|
+
if _time_filter:
|
|
138
|
+
filter_expr = (f"{filter_expr} AND {_time_filter}"
|
|
139
|
+
if filter_expr else _time_filter)
|
|
140
|
+
|
|
141
|
+
filtered = bool(filter_expr or db != "all")
|
|
97
142
|
params: dict[str, Any] = {
|
|
98
143
|
"Query": search_query or query,
|
|
99
144
|
"Count": str(_ZHIHU_COUNT_MAX if filtered else min(int(n or 5), _ZHIHU_COUNT_MAX)),
|
|
100
|
-
"SearchDB":
|
|
145
|
+
"SearchDB": db,
|
|
101
146
|
}
|
|
102
147
|
if filter_expr:
|
|
103
148
|
params["Filter"] = filter_expr
|