argo-search 2.9.7 → 2.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +58 -145
- package/README.es.md +58 -145
- package/README.ja.md +64 -151
- package/README.ko.md +63 -150
- package/README.md +65 -154
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +74 -5
- package/bin/argo.js +33 -2
- package/config.yaml +219 -60
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +6 -2
- package/packages/dsh-plugin/README.md +23 -5
- package/packages/dsh-plugin/dsh/client.js +304 -15
- package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
- package/packages/dsh-plugin/dsh/index.js +112 -1875
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +2 -2
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +175 -1
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +24 -1
- package/scripts/article_meta.py +131 -0
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +98 -0
- package/scripts/engine_dispatch.py +49 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +54 -22
- package/scripts/fetch_v3.py +25 -0
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +82 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +7 -0
- package/scripts/route_domains.py +32 -19
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +7 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +42 -193
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +115 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/local_engines.py +0 -321
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""output_transparency.py — 搜索输出的可信度评估、结构化失败清单与精简档透明修饰。
|
|
3
|
+
|
|
4
|
+
从 search_output.py 拆出(2026-10-21):
|
|
5
|
+
1. 单一职责:search_output 管「结果成形与人读格式」,本模块管「可信度/失败分类/
|
|
6
|
+
不可信边界」三件透明化信号;拆开后 search_output 保持在 ~650 行,远离千行硬限。
|
|
7
|
+
2. 口径合一:`_FAILURE_NON_ERROR` 与 `_NON_ERROR_OUTCOME` 同源派生,不再维护
|
|
8
|
+
两份独立 frozenset。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
# 本次调用不算失败的基础状态(引擎正常交付、无结果、或被早停主动取消)
|
|
17
|
+
_FAILURE_NON_ERROR: frozenset[str] = frozenset({
|
|
18
|
+
"ok", "ok-cached", "partial", "no-results", "no-results-cached", "cancelled",
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
# errors[] 与 uncertainty 排除的 outcome 状态:在基础非错误集之上再排除
|
|
22
|
+
# skipped-quota-exhausted(持续一天的配额跳过状态,不作为单次检索错误刷屏)
|
|
23
|
+
_NON_ERROR_OUTCOME: frozenset[str] = _FAILURE_NON_ERROR | {"skipped-quota-exhausted"}
|
|
24
|
+
|
|
25
|
+
# 二手/聚合平台(转载与洗稿高发地)。**刻意不复用 evidence 的词表**:
|
|
26
|
+
# evidence 属热路径禁入名单(tests/test_hotpath_import_lint.py 的 HEAVY_TOP,
|
|
27
|
+
# 它会拖进 content_signals/stylometry/burstiness 约 9ms),而本函数在每次
|
|
28
|
+
# 搜索的收口处无条件执行。两处词表需同步——改这里时看 evidence.
|
|
29
|
+
# _SECONDARY_PLATFORM_PATTERNS。
|
|
30
|
+
_AGGREGATOR_HOSTS: frozenset[str] = frozenset({
|
|
31
|
+
"baijiahao.baidu.com", # 百家号(自媒体洗稿重灾区)
|
|
32
|
+
"toutiao.com", # 头条号(聚合+洗稿)
|
|
33
|
+
"k.sina.com.cn", # 新浪博客/聚合
|
|
34
|
+
"guba.eastmoney.com", # 股吧(UGC 讨论,非事实来源)
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
# 可信度等级(与 dsh-free-search 的 meta.uncertainty 同形)
|
|
38
|
+
_UNCERTAINTY_HIGH = "high"
|
|
39
|
+
_UNCERTAINTY_MEDIUM = "medium"
|
|
40
|
+
_UNCERTAINTY_LOW = "low"
|
|
41
|
+
# 严重度序:用于「多信号取最差」——等级只能降不升。
|
|
42
|
+
_SEVERITY = {_UNCERTAINTY_HIGH: 0, _UNCERTAINTY_MEDIUM: 1, _UNCERTAINTY_LOW: 2}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _hostname(url: Any) -> str:
|
|
46
|
+
"""URL → 归一化主机名(小写、去 www.);解析失败返回空串。"""
|
|
47
|
+
try:
|
|
48
|
+
from urllib.parse import urlparse
|
|
49
|
+
host = urlparse(str(url or "")).hostname or ""
|
|
50
|
+
except Exception:
|
|
51
|
+
return ""
|
|
52
|
+
return host.lower().removeprefix("www.")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# 失败类别 → (归一类别, 可执行建议, 是否可重试)。类别名与 dsh-free-search
|
|
56
|
+
# 的 failureClass 对齐(quota/auth/bot-wall/transient/rate-limit/unknown),
|
|
57
|
+
# 但归类依据是 argo 自己的 outcome.status(retry_policy.classify_error_status
|
|
58
|
+
# 的唯一真源),不另写一套错误词表。
|
|
59
|
+
_FAILURE_META: dict[str, tuple[str, str, bool]] = {
|
|
60
|
+
"timeout": ("transient", "超时;可重试或换源", True),
|
|
61
|
+
"quota-exhausted": ("quota", "配额耗尽;换源或等窗口重置", False),
|
|
62
|
+
"rate-limited": ("rate-limit", "被限流;稍后重试或换源", True),
|
|
63
|
+
"blocked": ("bot-wall", "被反爬拦截;稍后重试或换源", True),
|
|
64
|
+
"auth-failed": ("auth", "检查 API Key 是否有效、是否有该接口权限", False),
|
|
65
|
+
"error": ("unknown", "源端错误;可重试或换源", True),
|
|
66
|
+
"skipped-missing-env": ("missing-key", "未配置密钥;配置后即可参与", False),
|
|
67
|
+
"skipped-quota-exhausted": ("quota", "配额耗尽,本轮跳过", False),
|
|
68
|
+
"skipped-circuit-open": ("cooldown", "熔断冷却中,稍后自动恢复", True),
|
|
69
|
+
"skipped-budget": ("budget", "本轮预算不足,未执行", True),
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def build_failures(result: dict[str, Any], limit: int = 6) -> list[dict[str, Any]]:
|
|
74
|
+
"""把引擎 outcome 压成 Agent 可行动的结构化失败清单。
|
|
75
|
+
|
|
76
|
+
与 errors 的分工:errors 是给人看的整行文本(兼容旧契约),本字段是
|
|
77
|
+
给模型/宿主判断「要不要重试、换哪个源、是不是缺 key」的分类结果。
|
|
78
|
+
只收录没交出结果的 outcome;ok/no-results 不算失败(后者是「查了没有」,
|
|
79
|
+
不是「没查成」),与 uncertainty 的降级判据同源。
|
|
80
|
+
"""
|
|
81
|
+
out: list[dict[str, Any]] = []
|
|
82
|
+
seen: set[tuple[str, str]] = set()
|
|
83
|
+
for o in (result.get("engine_outcomes") or []):
|
|
84
|
+
if not isinstance(o, dict):
|
|
85
|
+
continue
|
|
86
|
+
engine = str(o.get("engine") or "")
|
|
87
|
+
status = str(o.get("status") or "")
|
|
88
|
+
if not engine or status in _FAILURE_NON_ERROR:
|
|
89
|
+
continue
|
|
90
|
+
key = (engine, status)
|
|
91
|
+
if key in seen:
|
|
92
|
+
continue
|
|
93
|
+
seen.add(key)
|
|
94
|
+
category, action, retryable = _FAILURE_META.get(
|
|
95
|
+
status, ("unknown", "源端异常;可重试或换源", True))
|
|
96
|
+
item: dict[str, Any] = {
|
|
97
|
+
"engine": engine,
|
|
98
|
+
"status": status,
|
|
99
|
+
"category": category,
|
|
100
|
+
"retryable": retryable,
|
|
101
|
+
"action": action,
|
|
102
|
+
}
|
|
103
|
+
# skipped-* 是「持续状态」而不是本次尝试失败:配额窗口/熔断冷却在
|
|
104
|
+
# 周期边界前不会因重试改变。标出来,避免 Agent 立刻重试同一个源。
|
|
105
|
+
if status.startswith("skipped-"):
|
|
106
|
+
item["persistent"] = True
|
|
107
|
+
detail = str(o.get("detail") or "").strip()
|
|
108
|
+
if detail:
|
|
109
|
+
item["detail"] = detail[:160]
|
|
110
|
+
out.append(item)
|
|
111
|
+
if len(out) >= max(1, limit):
|
|
112
|
+
break
|
|
113
|
+
return out
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def decorate_compact_payload(out: dict[str, Any],
|
|
117
|
+
result: dict[str, Any]) -> None:
|
|
118
|
+
"""精简响应(MCP/Agent 档)的收尾修饰:边界清洗 + 三件透明信息 + sources 投影。"""
|
|
119
|
+
try:
|
|
120
|
+
from content_boundary import (CONTENT_TRUST_UNTRUSTED,
|
|
121
|
+
sanitize_results_boundary)
|
|
122
|
+
sanitize_results_boundary(out.get("results"))
|
|
123
|
+
out["content_trust"] = CONTENT_TRUST_UNTRUSTED
|
|
124
|
+
except Exception:
|
|
125
|
+
pass
|
|
126
|
+
failures = result.get("failures")
|
|
127
|
+
if isinstance(failures, list) and failures:
|
|
128
|
+
out["failures"] = failures[:6]
|
|
129
|
+
meta = result.get("meta")
|
|
130
|
+
if isinstance(meta, dict) and meta.get("uncertainty"):
|
|
131
|
+
out["meta"] = {"uncertainty": meta["uncertainty"]}
|
|
132
|
+
sources = result.get("sources")
|
|
133
|
+
if sources:
|
|
134
|
+
out["sources"] = [
|
|
135
|
+
{
|
|
136
|
+
"ref": s.get("ref"),
|
|
137
|
+
"title": s.get("title"),
|
|
138
|
+
"url": s.get("url"),
|
|
139
|
+
"engine": s.get("engine") or s.get("source"),
|
|
140
|
+
}
|
|
141
|
+
for s in sources
|
|
142
|
+
if isinstance(s, dict) and s.get("url")
|
|
143
|
+
][:20]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def build_uncertainty(result: dict[str, Any]) -> dict[str, Any]:
|
|
147
|
+
"""一次调用的综合可信度:`{"level": ..., "notes": [...]}`。
|
|
148
|
+
|
|
149
|
+
与 `limitations` 的分工:limitations 是**逐条**的局限声明(早停、降级、
|
|
150
|
+
缓存复用、站点过滤),本字段是**整体**等级——消费者拿到一个 level 就能
|
|
151
|
+
决定「直接引用」还是「先核验」,不必自己交叉比对五六个字段。
|
|
152
|
+
"""
|
|
153
|
+
notes: list[str] = []
|
|
154
|
+
results = [r for r in (result.get("results") or []) if isinstance(r, dict)]
|
|
155
|
+
|
|
156
|
+
if not results:
|
|
157
|
+
return {"level": _UNCERTAINTY_LOW, "notes": ["没有返回结果"]}
|
|
158
|
+
|
|
159
|
+
level = _UNCERTAINTY_HIGH
|
|
160
|
+
|
|
161
|
+
def _worse(candidate: str) -> None:
|
|
162
|
+
nonlocal level
|
|
163
|
+
if _SEVERITY[candidate] > _SEVERITY[level]:
|
|
164
|
+
level = candidate
|
|
165
|
+
|
|
166
|
+
if len(results) == 1:
|
|
167
|
+
_worse(_UNCERTAINTY_LOW)
|
|
168
|
+
notes.append("只有 1 个来源,无法交叉验证")
|
|
169
|
+
elif len(results) == 2:
|
|
170
|
+
_worse(_UNCERTAINTY_MEDIUM)
|
|
171
|
+
notes.append("只有 2 个来源")
|
|
172
|
+
|
|
173
|
+
hosts = {_hostname(r.get("url")) for r in results}
|
|
174
|
+
hosts.discard("")
|
|
175
|
+
if 0 < len(hosts) == 1:
|
|
176
|
+
_worse(_UNCERTAINTY_MEDIUM)
|
|
177
|
+
notes.append("所有结果来自同一站点")
|
|
178
|
+
|
|
179
|
+
# 尊重 search_output._AGGREGATOR_HOSTS 的猴子补丁(test_output_uncertainty 门禁锁)
|
|
180
|
+
so_mod = sys.modules.get("search_output")
|
|
181
|
+
agg_hosts = getattr(so_mod, "_AGGREGATOR_HOSTS", _AGGREGATOR_HOSTS)
|
|
182
|
+
aggregators = sorted(h for h in hosts if h in agg_hosts
|
|
183
|
+
or any(h.endswith("." + a) for a in agg_hosts))
|
|
184
|
+
if aggregators:
|
|
185
|
+
_worse(_UNCERTAINTY_MEDIUM)
|
|
186
|
+
notes.append(f"含聚合站内容({'/'.join(aggregators)}),建议二次核对")
|
|
187
|
+
|
|
188
|
+
if result.get("time_filter_warning"):
|
|
189
|
+
_worse(_UNCERTAINTY_MEDIUM)
|
|
190
|
+
notes.append("时间窗未实际过滤,结果可能不在请求的时间范围内")
|
|
191
|
+
|
|
192
|
+
outcomes = [o for o in (result.get("engine_outcomes") or [])
|
|
193
|
+
if isinstance(o, dict)]
|
|
194
|
+
failed = [o for o in outcomes
|
|
195
|
+
if str(o.get("status") or "") not in _NON_ERROR_OUTCOME]
|
|
196
|
+
if failed:
|
|
197
|
+
_worse(_UNCERTAINTY_LOW)
|
|
198
|
+
notes.append(f"{len(failed)} 个引擎未交出结果,覆盖可能比计划的窄")
|
|
199
|
+
|
|
200
|
+
return {"level": level, "notes": notes}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""paid_policy.py — 付费搜索的自动对冲位(2026-10-21,方案一)。
|
|
3
|
+
|
|
4
|
+
只解决一件事:在**已证明有优势的域**里,把付费引擎放到 combo 第二位当对冲,
|
|
5
|
+
而不是让它进通用候选池(那会被 TF-IDF/兜底带到任意查询并扣费)。
|
|
6
|
+
|
|
7
|
+
策略:
|
|
8
|
+
- `ARGO_PAID_SEARCH=off` → 完全关闭;缺省 `auto`。
|
|
9
|
+
- 该域在 `PAID_DOMAIN_HEDGES` 里,且引擎 key 就绪、月度预算未用完
|
|
10
|
+
→ 返回该引擎;预算用完/无 key → 回落 `tinyfish`(免费、带日期),
|
|
11
|
+
再不行返回空,由原 combo 继续。
|
|
12
|
+
- `fast`/`budget` 模式不参与(与 quota 的成本纪律一致)。
|
|
13
|
+
|
|
14
|
+
当前只开两个域:`english_tech`、`intl_news_flash`。依据 2026-10-21 多引擎
|
|
15
|
+
实测:DeepSeek 在英文技术/英文新闻上 p50 515ms、日期覆盖 23/39,明显优于
|
|
16
|
+
anysearch/Exa/Tavily;中文宽泛新闻与金融原文反而更差,故不纳入。
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
|
|
23
|
+
__all__ = ["PAID_DOMAIN_HEDGES", "hedge_for", "insert_hedge"]
|
|
24
|
+
|
|
25
|
+
PAID_DOMAIN_HEDGES: dict[str, str] = {
|
|
26
|
+
"english_tech": "deepseek_search",
|
|
27
|
+
"intl_news_flash": "deepseek_search",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
_FREE_FALLBACK = "tinyfish"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _enabled() -> bool:
|
|
34
|
+
try:
|
|
35
|
+
from engine_env import get_env
|
|
36
|
+
raw = get_env("ARGO_PAID_SEARCH", "auto")
|
|
37
|
+
except Exception:
|
|
38
|
+
raw = os.environ.get("ARGO_PAID_SEARCH", "auto")
|
|
39
|
+
return raw.strip().lower() not in ("off", "0", "false", "no")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _key_ready(engine: str) -> bool:
|
|
43
|
+
try:
|
|
44
|
+
from engine_env import missing_env_for
|
|
45
|
+
return not missing_env_for(engine)
|
|
46
|
+
except Exception:
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _budget_ok(engine: str) -> bool:
|
|
51
|
+
try:
|
|
52
|
+
from quota import get_quota_manager
|
|
53
|
+
return not get_quota_manager().is_hard_down(engine)
|
|
54
|
+
except Exception:
|
|
55
|
+
# 预算层读失败时 fail-closed:宁可不自动花钱
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _healthy_and_enabled(engine: str, enabled: set[str] | None) -> bool:
|
|
60
|
+
if enabled is not None and engine not in enabled:
|
|
61
|
+
return False
|
|
62
|
+
try:
|
|
63
|
+
from route_combo import _breaker_blocked
|
|
64
|
+
if _breaker_blocked(engine):
|
|
65
|
+
return False
|
|
66
|
+
except Exception:
|
|
67
|
+
pass
|
|
68
|
+
return True
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def hedge_for(domain: str, mode: str = "auto",
|
|
72
|
+
enabled: set[str] | None = None) -> list[str]:
|
|
73
|
+
"""返回该域应插入的付费/回落引擎(0 或 1 个)。永不抛异常。"""
|
|
74
|
+
if not _enabled() or mode in ("fast", "budget"):
|
|
75
|
+
return []
|
|
76
|
+
engine = PAID_DOMAIN_HEDGES.get(str(domain or ""))
|
|
77
|
+
if not engine:
|
|
78
|
+
return []
|
|
79
|
+
try:
|
|
80
|
+
if (_healthy_and_enabled(engine, enabled)
|
|
81
|
+
and _key_ready(engine) and _budget_ok(engine)):
|
|
82
|
+
return [engine]
|
|
83
|
+
# 付费位不可用:TinyFish 是免费且带发布日期的回落位
|
|
84
|
+
if (_healthy_and_enabled(_FREE_FALLBACK, enabled)
|
|
85
|
+
and _key_ready(_FREE_FALLBACK) and _budget_ok(_FREE_FALLBACK)):
|
|
86
|
+
return [_FREE_FALLBACK]
|
|
87
|
+
except Exception:
|
|
88
|
+
return []
|
|
89
|
+
return []
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def insert_hedge(combo: list[str], domain: str, mode: str = "auto",
|
|
93
|
+
enabled: set[str] | None = None) -> list[str]:
|
|
94
|
+
"""把付费/回落引擎插到 combo 第二位(原地改并返回,便于链式调用)。"""
|
|
95
|
+
for engine in reversed(hedge_for(domain, mode, enabled=enabled)):
|
|
96
|
+
if engine not in combo:
|
|
97
|
+
combo.insert(min(1, len(combo)), engine)
|
|
98
|
+
return combo
|
package/scripts/quota.py
CHANGED
|
@@ -14,6 +14,7 @@ quota.py — Unified Search v2 配额管理器
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import json
|
|
17
|
+
import os
|
|
17
18
|
import time
|
|
18
19
|
import threading
|
|
19
20
|
from pathlib import Path
|
|
@@ -28,6 +29,22 @@ SKILL_DIR = Path(__file__).parent.parent
|
|
|
28
29
|
BACKENDS_DIR = SKILL_DIR / "backends"
|
|
29
30
|
QUOTA_PROFILES_PATH = BACKENDS_DIR / "quota_profiles.json"
|
|
30
31
|
|
|
32
|
+
# 次数额度的环境覆盖白名单(DSH 设置页「付费搜索」分组写入)。
|
|
33
|
+
#
|
|
34
|
+
# 为什么按引擎列名而不是一个 JSON:面板是「一行一个 KEY=VALUE」的形态,
|
|
35
|
+
# 每个数字字段要独占一行才能被现有的 envMap/白名单门禁识别;一个 JSON
|
|
36
|
+
# 大字段既无法在面板上分引擎显示,也会让 env gate 失去可判定的锚点。
|
|
37
|
+
#
|
|
38
|
+
# 语义:值 = 该引擎配额周期内的次数上限;0 = 不限制(None);空/非法 = 保持
|
|
39
|
+
# config.yaml 的声明值。周期(day/month)不在面板改——它决定的是「这条限制
|
|
40
|
+
# 按天还是按月」,与引擎的商业额度形态绑定,换周期需要同时换文案与默认值。
|
|
41
|
+
_QUOTA_LIMIT_ENV: tuple[tuple[str, str], ...] = (
|
|
42
|
+
("ARGO_QUOTA_ANYSEARCH_LIMIT", "anysearch"), # 日
|
|
43
|
+
("ARGO_QUOTA_ZHIHU_LIMIT", "zhihu"), # 日
|
|
44
|
+
("ARGO_QUOTA_YUANBAO_LIMIT", "yuanbao"), # 月
|
|
45
|
+
("ARGO_QUOTA_TAVILY_LIMIT", "tavily"), # 月
|
|
46
|
+
)
|
|
47
|
+
|
|
31
48
|
|
|
32
49
|
_STATE_DIR_CACHE: tuple[str, Path] | None = None
|
|
33
50
|
|
|
@@ -118,6 +135,16 @@ def _prune_foreign_engines(state: dict, protect: set[str]) -> dict:
|
|
|
118
135
|
if name in known or name in protect}
|
|
119
136
|
|
|
120
137
|
|
|
138
|
+
def _at_or_over_budget(cost_used: float, budget: float) -> bool:
|
|
139
|
+
"""预算是否用完:按 6 位小数比较,消掉单价累加的浮点漂移。
|
|
140
|
+
|
|
141
|
+
实测:Exa $0.01 × 1000 次 = 9.999999999999831,直接 `>= 10` 会判「还没
|
|
142
|
+
用完」,额度显示 10.0 却还能再打一次。预算是对外承诺的数字,比较口径
|
|
143
|
+
必须与展示口径一致(budget_status 也是 round 6)。
|
|
144
|
+
"""
|
|
145
|
+
return round(float(cost_used), 6) >= round(float(budget), 6)
|
|
146
|
+
|
|
147
|
+
|
|
121
148
|
class QuotaManager:
|
|
122
149
|
"""配额追踪与消耗速率计算(v2)。"""
|
|
123
150
|
|
|
@@ -151,10 +178,29 @@ class QuotaManager:
|
|
|
151
178
|
self._profiles_hot = None
|
|
152
179
|
self._state_hot = None
|
|
153
180
|
|
|
154
|
-
def _fresh_locked(self) -> None:
|
|
155
|
-
"""带锁调用:磁盘文件签名变化即重读(调用方须已持锁)。
|
|
181
|
+
def _fresh_locked(self, engine: str | None = None) -> None:
|
|
182
|
+
"""带锁调用:磁盘文件签名变化即重读(调用方须已持锁)。
|
|
183
|
+
|
|
184
|
+
同一次路由遍历(~0.3ms 内对 230+ 个不同 engine 逐一查询)共用首轮
|
|
185
|
+
stat 结果,避免单次 route_query 产生 900+ 次重复 stat;同一引擎再次
|
|
186
|
+
查询或超过 2ms 立即重新 stat,保持跨进程热读立即可见(test_hot_state)。
|
|
187
|
+
"""
|
|
188
|
+
now = time.monotonic()
|
|
189
|
+
seen = getattr(self, "_fresh_seen", None)
|
|
190
|
+
if (engine is not None and seen is not None and engine not in seen
|
|
191
|
+
and (now - getattr(self, "_fresh_ts", 0.0)) < 0.002):
|
|
192
|
+
seen.add(engine)
|
|
193
|
+
return
|
|
194
|
+
self._fresh_ts = now
|
|
195
|
+
self._fresh_seen = {engine} if engine is not None else set()
|
|
196
|
+
try:
|
|
197
|
+
import budget as _budget
|
|
198
|
+
b_sig = _budget.budget_signature()
|
|
199
|
+
except Exception:
|
|
200
|
+
b_sig = None
|
|
156
201
|
try:
|
|
157
|
-
if self._profiles_hot is not None and self._profiles_hot.changed()
|
|
202
|
+
if (self._profiles_hot is not None and self._profiles_hot.changed()) \
|
|
203
|
+
or (b_sig is not None and b_sig != getattr(self, "_budget_sig", None)):
|
|
158
204
|
self._load_profiles()
|
|
159
205
|
if self._state_hot is not None and self._state_hot.changed():
|
|
160
206
|
self._load_state()
|
|
@@ -169,6 +215,38 @@ class QuotaManager:
|
|
|
169
215
|
self._profiles = json.loads(QUOTA_PROFILES_PATH.read_bytes())
|
|
170
216
|
except (json.JSONDecodeError, OSError):
|
|
171
217
|
self._profiles = {}
|
|
218
|
+
# 月度预算覆盖(2026-10-21):只改 profile 的 budget_* 与 period,
|
|
219
|
+
# 不碰 quota_profiles.json(那是随包发版的派生件)。拼错/无预算的引擎
|
|
220
|
+
# 不受影响,符合「其他默认不改」。
|
|
221
|
+
try:
|
|
222
|
+
import budget as _budget
|
|
223
|
+
_budget.apply_to_profiles(self._profiles)
|
|
224
|
+
self._budget_sig = _budget.budget_signature()
|
|
225
|
+
except Exception:
|
|
226
|
+
self._budget_sig = None
|
|
227
|
+
self._apply_quota_env_overrides()
|
|
228
|
+
|
|
229
|
+
def _apply_quota_env_overrides(self) -> None:
|
|
230
|
+
"""把面板写入的 ARGO_QUOTA_<ENGINE>_LIMIT 叠到次数额度上。
|
|
231
|
+
|
|
232
|
+
只改内存里的 profile,不写 quota_profiles.json(派生件随包发版)。
|
|
233
|
+
非法/空值静默忽略:面板与环境变量都可能被手工编辑,配额读取失败
|
|
234
|
+
不应让整次路由崩掉。0 = 显式不限制。
|
|
235
|
+
"""
|
|
236
|
+
for env_name, engine in _QUOTA_LIMIT_ENV:
|
|
237
|
+
raw = (os.environ.get(env_name) or "").strip()
|
|
238
|
+
if not raw:
|
|
239
|
+
continue
|
|
240
|
+
try:
|
|
241
|
+
value = int(raw)
|
|
242
|
+
except ValueError:
|
|
243
|
+
continue
|
|
244
|
+
if value < 0:
|
|
245
|
+
continue
|
|
246
|
+
profile = self._profiles.get(engine)
|
|
247
|
+
if not isinstance(profile, dict):
|
|
248
|
+
continue
|
|
249
|
+
profile["limit"] = value if value > 0 else None
|
|
172
250
|
|
|
173
251
|
def _load_state(self) -> None:
|
|
174
252
|
if _state_path().exists():
|
|
@@ -256,12 +334,18 @@ class QuotaManager:
|
|
|
256
334
|
st = self._state.setdefault(engine, {
|
|
257
335
|
"used": 0, "limit": 0, "calls": [],
|
|
258
336
|
"errors": 0, "last_reset": now, "total_cost": 0.0,
|
|
337
|
+
"cost_used": 0.0,
|
|
259
338
|
})
|
|
260
339
|
st["used"] = st.get("used", 0) + credits
|
|
261
340
|
st.setdefault("calls", []).append(now)
|
|
262
341
|
if not success:
|
|
263
342
|
st["errors"] = st.get("errors", 0) + 1
|
|
264
|
-
|
|
343
|
+
else:
|
|
344
|
+
# 只对成功调用计费:DeepSeek 等按成功请求收费,失败也计入
|
|
345
|
+
# 会让预算被网络抖动假消耗(2026-10-21 修)。
|
|
346
|
+
_cost = self.get_cost_per_call(engine)
|
|
347
|
+
st["cost_used"] = st.get("cost_used", 0.0) + _cost
|
|
348
|
+
st["total_cost"] = st.get("total_cost", 0.0) + _cost
|
|
265
349
|
self._prune_locked(st, now)
|
|
266
350
|
|
|
267
351
|
self._mutate_locked(_apply)
|
|
@@ -291,13 +375,14 @@ class QuotaManager:
|
|
|
291
375
|
if not isinstance(state, dict):
|
|
292
376
|
return
|
|
293
377
|
state["used"] = 0
|
|
378
|
+
state["cost_used"] = 0.0
|
|
294
379
|
state["last_reset"] = now
|
|
295
380
|
argo_paths.atomic_write_json(_state_path(), self._state)
|
|
296
381
|
|
|
297
382
|
def get_remaining_ratio(self, engine: str) -> float:
|
|
298
383
|
"""获取配额剩余比例。无限配额返回 1.0。"""
|
|
299
384
|
with self._lock:
|
|
300
|
-
self._fresh_locked()
|
|
385
|
+
self._fresh_locked(engine)
|
|
301
386
|
# 按周期重置(先于 limit 判空:null 引擎计数也要按周期归零,
|
|
302
387
|
# 否则用量计数永久累计、无周期语义)。重置在文件锁内落盘。
|
|
303
388
|
if self._period_elapsed(engine, time.time()):
|
|
@@ -305,9 +390,18 @@ class QuotaManager:
|
|
|
305
390
|
profile = self._profiles.get(engine, {})
|
|
306
391
|
state = self._state.get(engine, {})
|
|
307
392
|
limit = profile.get("limit")
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
393
|
+
ratios: list[float] = []
|
|
394
|
+
if limit is not None:
|
|
395
|
+
ratios.append(max(0.0, (limit - state.get("used", 0)) / limit))
|
|
396
|
+
budget = profile.get("budget_amount")
|
|
397
|
+
if budget:
|
|
398
|
+
cost_used = float(state.get("cost_used", 0.0))
|
|
399
|
+
if _at_or_over_budget(cost_used, budget):
|
|
400
|
+
ratios.append(0.0)
|
|
401
|
+
else:
|
|
402
|
+
ratios.append(max(0.0,
|
|
403
|
+
(float(budget) - cost_used) / float(budget)))
|
|
404
|
+
return min(ratios) if ratios else 1.0
|
|
311
405
|
|
|
312
406
|
@staticmethod
|
|
313
407
|
def _engine_key_fp(engine: str) -> str:
|
|
@@ -394,7 +488,7 @@ class QuotaManager:
|
|
|
394
488
|
|
|
395
489
|
def is_remote_exhausted(self, engine: str) -> bool:
|
|
396
490
|
with self._lock:
|
|
397
|
-
self._fresh_locked()
|
|
491
|
+
self._fresh_locked(engine)
|
|
398
492
|
self._refresh_remote_state_locked(engine, time.time())
|
|
399
493
|
return "remote_exhausted" in (self._state.get(engine) or {})
|
|
400
494
|
|
|
@@ -436,9 +530,51 @@ class QuotaManager:
|
|
|
436
530
|
与限频/预算无关——路由组合层用它做全模式排除;
|
|
437
531
|
is_available 的限频/付费判断不在此列。
|
|
438
532
|
"""
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
533
|
+
with self._lock:
|
|
534
|
+
self._fresh_locked(engine)
|
|
535
|
+
now = time.time()
|
|
536
|
+
self._refresh_remote_state_locked(engine, now)
|
|
537
|
+
st = self._state.get(engine) or {}
|
|
538
|
+
if "remote_exhausted" in st:
|
|
539
|
+
return True
|
|
540
|
+
if self._period_elapsed(engine, now):
|
|
541
|
+
self._reset_period_locked(engine, now)
|
|
542
|
+
st = self._state.get(engine) or {}
|
|
543
|
+
limit = (self._profiles.get(engine) or {}).get("limit")
|
|
544
|
+
if limit is not None and (limit - st.get("used", 0)) <= 0:
|
|
545
|
+
return True
|
|
546
|
+
budget = (self._profiles.get(engine) or {}).get("budget_amount")
|
|
547
|
+
return budget is not None and float(st.get("cost_used", 0.0)) >= float(budget)
|
|
548
|
+
|
|
549
|
+
def is_over_budget(self, engine: str) -> bool:
|
|
550
|
+
"""只判「月度预算是否用完」,不看次数/限频/远端配额。
|
|
551
|
+
|
|
552
|
+
调度层用它给显式调用一条可行动的错误(「本月预算已用完 ¥5.00/¥5.00」),
|
|
553
|
+
与 is_hard_down 的「配额意义上不可用」分开,避免把限流也报成预算。
|
|
554
|
+
"""
|
|
555
|
+
with self._lock:
|
|
556
|
+
self._fresh_locked(engine)
|
|
557
|
+
now = time.time()
|
|
558
|
+
if self._period_elapsed(engine, now):
|
|
559
|
+
self._reset_period_locked(engine, now)
|
|
560
|
+
profile = self._profiles.get(engine) or {}
|
|
561
|
+
budget = profile.get("budget_amount")
|
|
562
|
+
if budget is None:
|
|
563
|
+
return False
|
|
564
|
+
st = self._state.get(engine) or {}
|
|
565
|
+
return _at_or_over_budget(st.get("cost_used", 0.0), budget)
|
|
566
|
+
|
|
567
|
+
def budget_status(self, engine: str) -> dict[str, Any]:
|
|
568
|
+
"""单引擎预算快照(O(1),避免 over_budget_error 为拼错误文案扫全表 280 引擎)。"""
|
|
569
|
+
with self._lock:
|
|
570
|
+
self._fresh_locked(engine)
|
|
571
|
+
profile = self._profiles.get(engine) or {}
|
|
572
|
+
st = self._state.get(engine) or {}
|
|
573
|
+
return {
|
|
574
|
+
"budget": profile.get("budget_amount"),
|
|
575
|
+
"budget_currency": profile.get("budget_currency"),
|
|
576
|
+
"cost_used": round(float(st.get("cost_used", 0.0)), 6),
|
|
577
|
+
}
|
|
442
578
|
|
|
443
579
|
def get_current_rpm(self, engine: str) -> float:
|
|
444
580
|
"""获取最近 1 分钟的调用速率。"""
|
|
@@ -475,7 +611,7 @@ class QuotaManager:
|
|
|
475
611
|
将 is_remote_exhausted / get_remaining_ratio / get_current_rpm 的
|
|
476
612
|
检查合并到单次持锁中,消除 TOCTOU 间隙。
|
|
477
613
|
"""
|
|
478
|
-
self._fresh_locked()
|
|
614
|
+
self._fresh_locked(engine)
|
|
479
615
|
# is_remote_exhausted 的内联版本(不重复获取锁)
|
|
480
616
|
self._refresh_remote_state_locked(engine, time.time())
|
|
481
617
|
if "remote_exhausted" in (self._state.get(engine) or {}):
|
|
@@ -493,6 +629,10 @@ class QuotaManager:
|
|
|
493
629
|
qr = max(0.0, (limit - used) / limit)
|
|
494
630
|
if qr <= 0:
|
|
495
631
|
return False
|
|
632
|
+
budget = profile.get("budget_amount")
|
|
633
|
+
if budget is not None and _at_or_over_budget(
|
|
634
|
+
state.get("cost_used", 0.0), budget):
|
|
635
|
+
return False
|
|
496
636
|
# get_current_rpm 的内联版本(不重复获取锁)
|
|
497
637
|
qps = profile.get("qps")
|
|
498
638
|
if qps is not None:
|
|
@@ -538,6 +678,11 @@ class QuotaManager:
|
|
|
538
678
|
"total_cost": round(self.get_total_cost(engine), 6),
|
|
539
679
|
"used": self._state.get(engine, {}).get("used", 0),
|
|
540
680
|
"limit": profile.get("limit", "∞"),
|
|
681
|
+
"period": profile.get("period"),
|
|
682
|
+
# 月度预算(原生币种):amount=None 表示未设预算、沿用次数额度
|
|
683
|
+
"budget": profile.get("budget_amount"),
|
|
684
|
+
"budget_currency": profile.get("budget_currency"),
|
|
685
|
+
"cost_used": round((self._state.get(engine) or {}).get("cost_used", 0.0), 6),
|
|
541
686
|
"cost_tier": profile.get("cost_tier", "free"),
|
|
542
687
|
"remote_exhausted_until": (
|
|
543
688
|
round(float(mark["until"])) if mark else None),
|