argo-search 2.9.7 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +73 -150
- package/README.es.md +73 -150
- package/README.ja.md +79 -156
- package/README.ko.md +78 -155
- package/README.md +80 -159
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +78 -5
- package/bin/argo.js +33 -2
- package/config.yaml +269 -63
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +7 -2
- package/packages/dsh-plugin/README.md +25 -6
- package/packages/dsh-plugin/dsh/client.js +402 -47
- package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
- package/packages/dsh-plugin/dsh/index.js +135 -1876
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +3 -3
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +177 -2
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +126 -43
- package/scripts/article_meta.py +131 -0
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +119 -4
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +105 -0
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +58 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +58 -22
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +28 -3
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/http_client.py +24 -38
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +192 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/net_proxy.py +27 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery.py +7 -3
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +26 -5
- package/scripts/route_domains.py +170 -31
- package/scripts/safe_conn.py +52 -0
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +8 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +62 -195
- package/scripts/search_rank.py +6 -0
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +138 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""budget.py — 按引擎的月度预算(原生币种)。
|
|
3
|
+
|
|
4
|
+
为什么需要(2026-10-21):`quota.py` 的额度是**次数**(limit/used),而付费引擎
|
|
5
|
+
(DeepSeek、Tavily 等)的真实约束是**钱**。按次数换算会在单价变化时失真,也会
|
|
6
|
+
把「失败调用」算成花费(DeepSeek 只对成功请求计费)。本模块只做三件事:
|
|
7
|
+
|
|
8
|
+
1. 读取预算(默认 → 共享文件 → env 覆盖),不碰配额状态;
|
|
9
|
+
2. 把预算挂到 `quota.py` 的 profile 上(budget_amount / budget_currency / period);
|
|
10
|
+
3. 提供 `argo budget set|list|unset` 的存储原语。
|
|
11
|
+
|
|
12
|
+
存储位置:`~/.config/argo/budgets.json`(POSIX)或 `%APPDATA%\\argo\\budgets.json`
|
|
13
|
+
(Windows),与 `~/.config/argo/env` 同根;`ARGO_BUDGET_FILE` 可覆盖(测试/多环境)。
|
|
14
|
+
格式:
|
|
15
|
+
|
|
16
|
+
{"deepseek_search": {"amount": 5.0, "currency": "CNY", "period": "month"}}
|
|
17
|
+
|
|
18
|
+
env 覆盖(优先级高于文件,便于插件/临时实验):
|
|
19
|
+
- `ARGO_BUDGETS_JSON`:整份 JSON(最高)
|
|
20
|
+
- `ARGO_BUDGET_<ENGINE>`:单引擎金额(币种按引擎 cost_unit,缺省 CNY;周期 month)
|
|
21
|
+
|
|
22
|
+
默认只给 DeepSeek 一个月度体验额度(¥5 ≈ 1000 次成功调用);其他引擎默认不设,
|
|
23
|
+
沿用各自 quota_profiles.json 里的既有额度,符合「其他默认不改」。
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"DEFAULT_BUDGETS",
|
|
35
|
+
"budget_file",
|
|
36
|
+
"budget_signature",
|
|
37
|
+
"load_budgets",
|
|
38
|
+
"budget_for",
|
|
39
|
+
"apply_to_profiles",
|
|
40
|
+
"set_budget",
|
|
41
|
+
"unset_budget",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
# 默认预算:只覆盖 DeepSeek。金额按人民币;折算依据是官方 5 元/千次成功请求。
|
|
45
|
+
DEFAULT_BUDGETS: dict[str, dict[str, Any]] = {
|
|
46
|
+
"deepseek_search": {"amount": 5.0, "currency": "CNY", "period": "month"},
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
_ENV_JSON = "ARGO_BUDGETS_JSON"
|
|
50
|
+
_ENV_FILE = "ARGO_BUDGET_FILE"
|
|
51
|
+
_ENV_PREFIX = "ARGO_" + "BUDGET_"
|
|
52
|
+
|
|
53
|
+
# 单引擎 env 覆盖的默认币种(金额单位必须与引擎 cost_unit 对齐,否则预算
|
|
54
|
+
# 与 cost_used 是两个币种在比大小)。未列出的引擎按 CNY,与历史行为一致。
|
|
55
|
+
_ENV_CURRENCY = {
|
|
56
|
+
"deepseek_search": "CNY",
|
|
57
|
+
"exa": "USD",
|
|
58
|
+
"tavily": "USD",
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def budget_file() -> Path:
|
|
63
|
+
"""预算文件路径(不保证存在)。"""
|
|
64
|
+
override = (os.environ.get(_ENV_FILE) or "").strip()
|
|
65
|
+
if override:
|
|
66
|
+
return Path(os.path.expanduser(override))
|
|
67
|
+
if os.name == "nt":
|
|
68
|
+
base = (os.environ.get("APPDATA") or "").strip() or os.path.expanduser("~")
|
|
69
|
+
else:
|
|
70
|
+
base = (os.environ.get("XDG_CONFIG_HOME") or "").strip() or os.path.expanduser("~/.config")
|
|
71
|
+
return Path(base) / "argo" / "budgets.json"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _valid(entry: Any) -> dict[str, Any] | None:
|
|
75
|
+
"""校验并归一一条预算;坏条目返回 None(不阻断其他引擎)。"""
|
|
76
|
+
if not isinstance(entry, dict):
|
|
77
|
+
return None
|
|
78
|
+
try:
|
|
79
|
+
amount = float(entry.get("amount"))
|
|
80
|
+
except (TypeError, ValueError):
|
|
81
|
+
return None
|
|
82
|
+
if amount < 0:
|
|
83
|
+
return None
|
|
84
|
+
currency = str(entry.get("currency") or "CNY").strip().upper() or "CNY"
|
|
85
|
+
period = str(entry.get("period") or "month").strip().lower()
|
|
86
|
+
if period not in ("day", "month"):
|
|
87
|
+
period = "month"
|
|
88
|
+
return {"amount": amount, "currency": currency, "period": period}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _read_file() -> dict[str, dict[str, Any]]:
|
|
92
|
+
try:
|
|
93
|
+
raw = json.loads(budget_file().read_text(encoding="utf-8"))
|
|
94
|
+
except (OSError, ValueError):
|
|
95
|
+
return {}
|
|
96
|
+
if not isinstance(raw, dict):
|
|
97
|
+
return {}
|
|
98
|
+
out: dict[str, dict[str, Any]] = {}
|
|
99
|
+
for engine, entry in raw.items():
|
|
100
|
+
if str(engine).startswith("_"):
|
|
101
|
+
continue
|
|
102
|
+
norm = _valid(entry)
|
|
103
|
+
if norm is not None:
|
|
104
|
+
out[str(engine)] = norm
|
|
105
|
+
return out
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _read_env() -> dict[str, dict[str, Any]]:
|
|
109
|
+
out: dict[str, dict[str, Any]] = {}
|
|
110
|
+
raw = (os.environ.get(_ENV_JSON) or "").strip()
|
|
111
|
+
if raw:
|
|
112
|
+
try:
|
|
113
|
+
parsed = json.loads(raw)
|
|
114
|
+
except ValueError:
|
|
115
|
+
parsed = None
|
|
116
|
+
if isinstance(parsed, dict):
|
|
117
|
+
for engine, entry in parsed.items():
|
|
118
|
+
norm = _valid(entry)
|
|
119
|
+
if norm is not None:
|
|
120
|
+
out[str(engine)] = norm
|
|
121
|
+
# 单引擎便捷覆盖:"ARGO_BUDGET_DEEPSEEK_SEARCH"=5 → deepseek_search 5 CNY/月。
|
|
122
|
+
# 同族命名 ARGO_BUDGET_<ENGINE> 由下面按前缀解析;字面量保留给插件门禁对拍。
|
|
123
|
+
# 只认「大写化后能在默认/文件里找到的引擎」太窄;这里按环境变量名反推引擎名,
|
|
124
|
+
# 由调用方(quota)决定是否真的存在该引擎,避免拼错静默生效。
|
|
125
|
+
for key, value in os.environ.items():
|
|
126
|
+
if not key.startswith(_ENV_PREFIX) or key in (_ENV_JSON, _ENV_FILE):
|
|
127
|
+
continue
|
|
128
|
+
engine = key[len(_ENV_PREFIX):].strip().lower()
|
|
129
|
+
if not engine:
|
|
130
|
+
continue
|
|
131
|
+
try:
|
|
132
|
+
amount = float(str(value).strip())
|
|
133
|
+
except (TypeError, ValueError):
|
|
134
|
+
continue
|
|
135
|
+
if amount < 0:
|
|
136
|
+
continue
|
|
137
|
+
currency = (out.get(engine) or {}).get("currency") \
|
|
138
|
+
or _ENV_CURRENCY.get(engine, "CNY")
|
|
139
|
+
entry = {"amount": amount, "currency": currency, "period": "month"}
|
|
140
|
+
out[engine] = entry
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def budget_signature() -> tuple[Any, ...]:
|
|
145
|
+
"""预算配置源的轻量签名(文件 mtime/size + 环境变量覆盖),供热读与路由缓存换键。"""
|
|
146
|
+
p = budget_file()
|
|
147
|
+
try:
|
|
148
|
+
st = p.stat()
|
|
149
|
+
f_sig: tuple[Any, ...] = (st.st_mtime_ns, st.st_size)
|
|
150
|
+
except OSError:
|
|
151
|
+
f_sig = (0, 0)
|
|
152
|
+
return (
|
|
153
|
+
f_sig,
|
|
154
|
+
os.environ.get(_ENV_JSON, ""),
|
|
155
|
+
os.environ.get("ARGO_BUDGET_DEEPSEEK_SEARCH", ""),
|
|
156
|
+
os.environ.get("ARGO_BUDGET_EXA", ""),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def load_budgets() -> dict[str, dict[str, Any]]:
|
|
161
|
+
"""默认 → 文件 → env,逐层覆盖。"""
|
|
162
|
+
merged: dict[str, dict[str, Any]] = {k: dict(v) for k, v in DEFAULT_BUDGETS.items()}
|
|
163
|
+
merged.update(_read_file())
|
|
164
|
+
merged.update(_read_env())
|
|
165
|
+
return merged
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def budget_for(engine: str) -> dict[str, Any] | None:
|
|
169
|
+
return load_budgets().get(str(engine))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def apply_to_profiles(profiles: dict[str, dict[str, Any]]) -> None:
|
|
173
|
+
"""把预算挂到 quota profile(就地)。只覆盖存在的引擎,拼错不生效。"""
|
|
174
|
+
active = load_budgets()
|
|
175
|
+
for engine, spec in profiles.items():
|
|
176
|
+
if not isinstance(spec, dict) or str(engine).startswith("_"):
|
|
177
|
+
continue
|
|
178
|
+
entry = active.get(engine)
|
|
179
|
+
if entry is not None:
|
|
180
|
+
spec["budget_amount"] = entry["amount"]
|
|
181
|
+
spec["budget_currency"] = entry["currency"]
|
|
182
|
+
# 预算周期即重置周期;与 limit 共用同一条 `period`,避免两套时钟。
|
|
183
|
+
spec["period"] = entry["period"]
|
|
184
|
+
else:
|
|
185
|
+
spec.pop("budget_amount", None)
|
|
186
|
+
spec.pop("budget_currency", None)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _write(data: dict[str, Any]) -> None:
|
|
190
|
+
path = budget_file()
|
|
191
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
193
|
+
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
194
|
+
encoding="utf-8")
|
|
195
|
+
os.replace(tmp, path)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def over_budget_error(engine: str) -> str | None:
|
|
199
|
+
"""预算耗尽时返回可行动错误文本;未耗尽/未配预算/读失败返回 None。
|
|
200
|
+
|
|
201
|
+
放在 budget.py 而不是 dispatch:engine_dispatch 有行数门禁,预算检查是
|
|
202
|
+
「读状态 + 拼一句话」的纯策略,搬出来也让显式/自动路径共用同一文案。
|
|
203
|
+
"""
|
|
204
|
+
try:
|
|
205
|
+
from engine_env import env_flag
|
|
206
|
+
if env_flag("ARGO_PAID_BUDGET_BYPASS", default=False):
|
|
207
|
+
return None
|
|
208
|
+
except Exception:
|
|
209
|
+
if (os.environ.get("ARGO_PAID_BUDGET_BYPASS", "").strip().lower()
|
|
210
|
+
in ("1", "true", "yes", "on")):
|
|
211
|
+
return None
|
|
212
|
+
try:
|
|
213
|
+
from quota import get_quota_manager
|
|
214
|
+
qm = get_quota_manager()
|
|
215
|
+
# 只认真正的 True:测试常把 quota manager 打成 MagicMock,属性调用
|
|
216
|
+
# 会返回真值 mock——不能把「mock 存在」当成预算耗尽。
|
|
217
|
+
if qm.is_over_budget(engine) is not True:
|
|
218
|
+
return None
|
|
219
|
+
st = (qm.budget_status(engine) if hasattr(qm, "budget_status")
|
|
220
|
+
else (qm.get_stats().get(engine) or {}))
|
|
221
|
+
return (f"本月预算已用完({st.get('cost_used', 0)}/{st.get('budget')} "
|
|
222
|
+
f"{st.get('budget_currency') or ''});"
|
|
223
|
+
f"可用 ARGO_PAID_BUDGET_BYPASS=1 单次放行")
|
|
224
|
+
except Exception:
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def set_budget(engine: str, amount: float, *, currency: str = "CNY",
|
|
229
|
+
period: str = "month") -> dict[str, Any]:
|
|
230
|
+
"""写入一条预算(保留文件里其他引擎)。返回归一后的条目。"""
|
|
231
|
+
norm = _valid({"amount": amount, "currency": currency, "period": period})
|
|
232
|
+
if norm is None:
|
|
233
|
+
raise ValueError("预算金额必须是非负数")
|
|
234
|
+
data = _read_file()
|
|
235
|
+
data[str(engine)] = norm
|
|
236
|
+
_write(data)
|
|
237
|
+
return norm
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def unset_budget(engine: str) -> bool:
|
|
241
|
+
"""删除一条预算;返回是否真的删了。"""
|
|
242
|
+
data = _read_file()
|
|
243
|
+
if engine not in data:
|
|
244
|
+
return False
|
|
245
|
+
data.pop(engine, None)
|
|
246
|
+
_write(data)
|
|
247
|
+
return True
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""budget_cli.py — `argo budget`:按引擎的月度预算读写口。
|
|
3
|
+
|
|
4
|
+
用法:
|
|
5
|
+
argo budget list [--json]
|
|
6
|
+
argo budget set <engine> <amount> [--currency CNY|USD] [--period month|day]
|
|
7
|
+
argo budget unset <engine>
|
|
8
|
+
|
|
9
|
+
预算存在 ~/.config/argo/budgets.json(`ARGO_BUDGET_FILE` 可覆盖);`quota.py`
|
|
10
|
+
启动时读取,按引擎原生币种与调用成本比较。默认只有 DeepSeek ¥5/月。
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
import budget
|
|
19
|
+
from cli_io import dumps
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _cmd_list(args: argparse.Namespace) -> int:
|
|
23
|
+
items = budget.load_budgets()
|
|
24
|
+
try:
|
|
25
|
+
from quota import get_quota_manager
|
|
26
|
+
stats = get_quota_manager().get_stats()
|
|
27
|
+
except Exception:
|
|
28
|
+
stats = {}
|
|
29
|
+
rows = []
|
|
30
|
+
for engine, entry in sorted(items.items()):
|
|
31
|
+
st = stats.get(engine) or {}
|
|
32
|
+
rows.append({
|
|
33
|
+
"engine": engine,
|
|
34
|
+
"amount": entry["amount"],
|
|
35
|
+
"currency": entry["currency"],
|
|
36
|
+
"period": entry["period"],
|
|
37
|
+
"cost_used": st.get("cost_used"),
|
|
38
|
+
"cost_per_call": st.get("cost_per_call"),
|
|
39
|
+
"available": st.get("available"),
|
|
40
|
+
})
|
|
41
|
+
if args.json:
|
|
42
|
+
print(dumps({"file": str(budget.budget_file()), "budgets": rows}))
|
|
43
|
+
else:
|
|
44
|
+
print(f"预算文件:{budget.budget_file()}")
|
|
45
|
+
if not rows:
|
|
46
|
+
print("(无预算;引擎沿用 quota_profiles.json 的次数额度)")
|
|
47
|
+
for r in rows:
|
|
48
|
+
print(f" {r['engine']:<20} {r['amount']:>8.2f} {r['currency']}/"
|
|
49
|
+
f"{r['period']} 已用 {r['cost_used']} 可用={r['available']}")
|
|
50
|
+
return 0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _cmd_set(args: argparse.Namespace) -> int:
|
|
54
|
+
try:
|
|
55
|
+
entry = budget.set_budget(args.engine, args.amount,
|
|
56
|
+
currency=args.currency,
|
|
57
|
+
period=args.period)
|
|
58
|
+
except ValueError as exc:
|
|
59
|
+
print(f"设置失败:{exc}", file=sys.stderr)
|
|
60
|
+
return 2
|
|
61
|
+
print(f"已设置 {args.engine}: {entry['amount']} {entry['currency']}/{entry['period']}"
|
|
62
|
+
f"({budget.budget_file()})")
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _cmd_unset(args: argparse.Namespace) -> int:
|
|
67
|
+
ok = budget.unset_budget(args.engine)
|
|
68
|
+
print(("已删除 " if ok else "未找到 ") + args.engine)
|
|
69
|
+
return 0 if ok else 1
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def main(argv: list[str] | None = None) -> int:
|
|
73
|
+
ap = argparse.ArgumentParser(prog="argo budget",
|
|
74
|
+
description="按引擎的月度预算(原生币种)")
|
|
75
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
76
|
+
p_list = sub.add_parser("list", help="列出预算与本月已用")
|
|
77
|
+
p_list.add_argument("--json", action="store_true")
|
|
78
|
+
p_list.set_defaults(func=_cmd_list)
|
|
79
|
+
p_set = sub.add_parser("set", help="设置某引擎预算")
|
|
80
|
+
p_set.add_argument("engine")
|
|
81
|
+
p_set.add_argument("amount", type=float)
|
|
82
|
+
p_set.add_argument("--currency", default="CNY")
|
|
83
|
+
p_set.add_argument("--period", default="month", choices=["day", "month"])
|
|
84
|
+
p_set.set_defaults(func=_cmd_set)
|
|
85
|
+
p_unset = sub.add_parser("unset", help="删除某引擎预算")
|
|
86
|
+
p_unset.add_argument("engine")
|
|
87
|
+
p_unset.set_defaults(func=_cmd_unset)
|
|
88
|
+
args = ap.parse_args(argv)
|
|
89
|
+
return args.func(args)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
if __name__ == "__main__":
|
|
93
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# 构建 Argo Release 资产包(tar.gz、wheel 与 sha256 校验和)
|
|
3
|
+
#
|
|
4
|
+
# 用法:
|
|
5
|
+
# bash scripts/build_release_assets.sh
|
|
6
|
+
# 产物输出至 dist/ 目录。
|
|
7
|
+
|
|
8
|
+
set -euo pipefail
|
|
9
|
+
|
|
10
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
11
|
+
cd "$ROOT_DIR"
|
|
12
|
+
|
|
13
|
+
VERSION=$(python3 -c 'import json; print(json.load(open("package.json"))["version"])')
|
|
14
|
+
DIST_DIR="$ROOT_DIR/dist"
|
|
15
|
+
STAGE_DIR="$DIST_DIR/argo-$VERSION"
|
|
16
|
+
|
|
17
|
+
echo "==> 开始构建 Argo Release 资产包 (v$VERSION)..."
|
|
18
|
+
rm -rf "$DIST_DIR"
|
|
19
|
+
mkdir -p "$DIST_DIR" "$STAGE_DIR"
|
|
20
|
+
|
|
21
|
+
# 1. 拷贝运行时必需核心文件至 staging 目录(排除测试/缓存/私有数据)
|
|
22
|
+
echo "--> 组织源码资产..."
|
|
23
|
+
cp -r bin config.yaml engines backends data mcp scripts sub-skills README.md SKILL.md LICENSE package.json pyproject.toml "$STAGE_DIR/"
|
|
24
|
+
|
|
25
|
+
# 清理 staging 内的多余缓存与测试
|
|
26
|
+
find "$STAGE_DIR" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
|
27
|
+
find "$STAGE_DIR" -name "*.pyc" -delete 2>/dev/null || true
|
|
28
|
+
rm -rf "$STAGE_DIR/data/jobs" 2>/dev/null || true
|
|
29
|
+
find "$STAGE_DIR/sub-skills" -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true
|
|
30
|
+
find "$STAGE_DIR/sub-skills" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
|
31
|
+
|
|
32
|
+
# 2. 打包 tar.gz
|
|
33
|
+
TARBALL="$DIST_DIR/argo-v$VERSION.tar.gz"
|
|
34
|
+
echo "--> 生成源码压缩包: $TARBALL"
|
|
35
|
+
tar -czf "$TARBALL" -C "$DIST_DIR" "argo-$VERSION"
|
|
36
|
+
|
|
37
|
+
# 3. 如果环境已安装 build 模块,打出标准 Python wheel。
|
|
38
|
+
# wheel 的运行树来自 staging:构建前复制到 argo_search/_data,构建后清理。
|
|
39
|
+
# 这样 pip/uvx 安装后 console script 仍能找到 config/engines/sub-skills。
|
|
40
|
+
if python3 -c 'import build' 2>/dev/null; then
|
|
41
|
+
echo "--> 构建 Python Wheel 与 Sdist..."
|
|
42
|
+
PKG_DATA="$ROOT_DIR/argo_search/_data"
|
|
43
|
+
rm -rf "$PKG_DATA"
|
|
44
|
+
cp -r "$STAGE_DIR" "$PKG_DATA"
|
|
45
|
+
python3 -m build --outdir "$DIST_DIR"
|
|
46
|
+
rm -rf "$PKG_DATA"
|
|
47
|
+
elif python3 -c 'import setuptools, wheel' 2>/dev/null; then
|
|
48
|
+
echo "--> 使用 setuptools 构建 Wheel..."
|
|
49
|
+
python3 setup.py bdist_wheel --dist-dir "$DIST_DIR" 2>/dev/null || true
|
|
50
|
+
fi
|
|
51
|
+
rm -rf "$STAGE_DIR"
|
|
52
|
+
|
|
53
|
+
# 4. 生成 SHA256 校验和清单
|
|
54
|
+
echo "--> 计算 SHA256 校验和..."
|
|
55
|
+
(
|
|
56
|
+
cd "$DIST_DIR"
|
|
57
|
+
if command -v sha256sum >/dev/null 2>&1; then
|
|
58
|
+
sha256sum * > SHA256SUMS.txt
|
|
59
|
+
elif command -v shasum >/dev/null 2>&1; then
|
|
60
|
+
shasum -a 256 * > SHA256SUMS.txt
|
|
61
|
+
fi
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
echo "==> 构建完成!Release 资产清单:"
|
|
65
|
+
ls -lh "$DIST_DIR"
|
package/scripts/cache.py
CHANGED
|
@@ -354,7 +354,17 @@ class SQLiteCache:
|
|
|
354
354
|
pass
|
|
355
355
|
|
|
356
356
|
def _init_db(self):
|
|
357
|
+
if (self._db_path not in (":memory:", "")
|
|
358
|
+
and not self._db_path.startswith("file:")
|
|
359
|
+
and os.path.exists(self._db_path)
|
|
360
|
+
and not os.access(self._db_path, os.W_OK)):
|
|
361
|
+
raise _get_sqlite3().OperationalError("readonly database")
|
|
357
362
|
with self._conn() as conn:
|
|
363
|
+
try:
|
|
364
|
+
if conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == (str(self.SCHEMA_VERSION),):
|
|
365
|
+
return
|
|
366
|
+
except _get_sqlite3().Error:
|
|
367
|
+
pass
|
|
358
368
|
conn.executescript("""
|
|
359
369
|
CREATE TABLE IF NOT EXISTS search_cache (
|
|
360
370
|
key TEXT PRIMARY KEY,
|
|
@@ -463,6 +473,20 @@ class SQLiteCache:
|
|
|
463
473
|
self._hits += 1
|
|
464
474
|
return self._deserialize(value_blob, compressed)
|
|
465
475
|
|
|
476
|
+
def get_many(self, keys: list[str]) -> dict[str, dict]:
|
|
477
|
+
if not keys or self._degraded_reason is not None:
|
|
478
|
+
return {}
|
|
479
|
+
out: dict[str, dict] = {}
|
|
480
|
+
with self._lock, self._conn() as conn:
|
|
481
|
+
ph = ",".join("?" for _ in keys)
|
|
482
|
+
for k, blob, comp, ts, ttl in conn.execute(
|
|
483
|
+
f"SELECT key, value_blob, compressed, created_at, ttl "
|
|
484
|
+
f"FROM search_cache WHERE key IN ({ph})", tuple(keys)
|
|
485
|
+
):
|
|
486
|
+
if not self._is_expired(ts, ttl):
|
|
487
|
+
out[k] = self._deserialize(blob, comp)
|
|
488
|
+
return out
|
|
489
|
+
|
|
466
490
|
def set(self, key: str, query: str, engine: str, max_results: int,
|
|
467
491
|
value: dict, domain: str = "general", ttl: int | None = None,
|
|
468
492
|
mode: str = "", depth: str = ""):
|
|
@@ -516,35 +540,9 @@ class SQLiteCache:
|
|
|
516
540
|
pass
|
|
517
541
|
|
|
518
542
|
def _evict_if_needed(self):
|
|
519
|
-
"""
|
|
520
|
-
|
|
521
|
-
**历史 bug(2026-09-19 复现)**:判定读的是文件页数
|
|
522
|
-
(page_count × page_size),扣减的却是 payload 净长。而本库
|
|
523
|
-
`auto_vacuum=0` 且全仓无 VACUUM,页数**永不下降**——库一旦超过
|
|
524
|
-
MAX_DB_SIZE_MB,`while total_bytes > target_bytes` 就永不收敛:
|
|
525
|
-
每次 set() 把整库(含刚写入的那一行)删空,此后每次写入重复清空,
|
|
526
|
-
缓存永久 100% miss。修法是让扣减与判定同源——进入淘汰循环后一律
|
|
527
|
-
用 payload 口径,删完再 VACUUM 把文件真正缩回去(不 VACUUM 的话
|
|
528
|
-
下一轮 set() 会看到同样的页数并重删一遍,正是这个 bug 的另一半)。
|
|
529
|
-
|
|
530
|
-
另一半是**过期行**:它们对任何调用方都不可见(get() 命中即删),
|
|
531
|
-
实测线上 5521 行里 5444 行(98.6%)已过期、占 payload 的 98.3%。
|
|
532
|
-
不回收不只是占盘——find_similar 与淘汰扫描都要扫过这些永不命中的行。
|
|
533
|
-
按节流间隔回收,稳态每小时一次。
|
|
534
|
-
|
|
535
|
-
阈值口径(2026-10-15):limit 取 `self._max_size_mb`(构造时定下),
|
|
536
|
-
不再直接读模块常量 MAX_DB_SIZE_MB——生产侧的上限现在来自 config.yaml
|
|
537
|
-
的 `cache.max_size_mb`,改配置真的会改行为。
|
|
538
|
-
"""
|
|
543
|
+
"""空间回收三步:过期行清扫(每小时节流)→ LRU 淘汰(按 payload 口径)→ VACUUM。"""
|
|
539
544
|
with self._conn() as conn:
|
|
540
545
|
limit_bytes = self._max_size_mb * 1024 * 1024
|
|
541
|
-
|
|
542
|
-
# 三条触发各自独立,代价都是 O(1):
|
|
543
|
-
# 过期清扫(节流到每小时一次,因为它要全表扫)
|
|
544
|
-
# LRU 淘汰(文件超限时)
|
|
545
|
-
# 纯回收(空闲页够多时——清扫刚删完一大片就会命中这条,
|
|
546
|
-
# 不把它独立出来,回收就会被节流挡到下一个清扫点,
|
|
547
|
-
# 文件白停在高水位一小时)
|
|
548
546
|
if self._expiry_sweep_due(conn):
|
|
549
547
|
conn.execute(
|
|
550
548
|
"DELETE FROM search_cache WHERE created_at + ttl < ?",
|
|
@@ -557,7 +555,6 @@ class SQLiteCache:
|
|
|
557
555
|
" * (SELECT page_size FROM pragma_page_size)"
|
|
558
556
|
).fetchone()[0] or 0
|
|
559
557
|
if page_bytes > limit_bytes:
|
|
560
|
-
# 按最近访问时间淘汰最旧行;扣减口径与判定同源,循环必然收敛
|
|
561
558
|
target_bytes = int(limit_bytes * 0.8)
|
|
562
559
|
payload = conn.execute(
|
|
563
560
|
"SELECT COALESCE(SUM(LENGTH(value_blob)), 0) FROM search_cache"
|
|
@@ -574,9 +571,6 @@ class SQLiteCache:
|
|
|
574
571
|
payload -= sum((sz or 0) for _, sz in rows)
|
|
575
572
|
conn.commit()
|
|
576
573
|
|
|
577
|
-
# 删行不会让文件变小:页进 freelist,文件停在高水位。不回收的话
|
|
578
|
-
# 稳态下会留着一整块「只有空闲页」的库——实测线上 10.1MB 文件里
|
|
579
|
-
# 活数据只有 0.12MB。判据用空闲页占比而非固定字节:小库才会回收。
|
|
580
574
|
if self._free_bytes(conn) > self._RECLAIM_FREE_RATIO * (page_bytes or 1):
|
|
581
575
|
conn.execute("VACUUM")
|
|
582
576
|
|
|
@@ -603,42 +597,11 @@ class SQLiteCache:
|
|
|
603
597
|
mode: str = "auto", depth: str = "fast") -> list[dict]:
|
|
604
598
|
"""近重复查询软命中:扫描最近缓存,minhash 相似度 ≥ threshold 的条目。
|
|
605
599
|
|
|
606
|
-
返回 [{key, query, similarity}]
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
单字差异相似度约 0.75,0.7 可捕捉近重复且排除无关查询(≈0)。
|
|
612
|
-
|
|
613
|
-
engine 隔离(v2.4.2 修复):SQL 必须按 engine 列过滤。此前形参收了
|
|
614
|
-
engine 但 WHERE 子句只用 domain,软命中因此跨引擎串味——实测
|
|
615
|
-
`--engine v2ex` 可命中 bilibili 缓存;fetch 与 evidence 同 domain 下
|
|
616
|
-
URL 词面相似(如 x 与 x.md,相似度 0.875)会互相串正文。
|
|
617
|
-
组合键(多引擎拼接的 `a+b`)不参与软命中:组合结果集是融合产物,
|
|
618
|
-
与任何单引擎缓存都不可互换。
|
|
619
|
-
|
|
620
|
-
**engine 现在是精确匹配,不再是「auto 通配」**(2026-09-19 随之调整):
|
|
621
|
-
engine 列从「路由出来的引擎组合」改存「请求侧身份」(用户点名的引擎
|
|
622
|
-
或 auto,见 search.execute_search 的 cache_engine_key 说明)。语义变了,
|
|
623
|
-
通配的含义也跟着变——以前 `auto` 不是列里会出现的值,通配分支其实
|
|
624
|
-
从未被走到(组合列要么是 `a+b` 要么是单引擎名);现在 `auto` 是真实的
|
|
625
|
-
请求身份,再通配就等于让「自动路由」的请求去吃「显式指定 pypi」的
|
|
626
|
-
缓存,正是 v2.4.2 要挡的跨引擎串味。请求不同 → 结果不可互换。
|
|
627
|
-
|
|
628
|
-
限定符隔离(2026-09-18 修复):限定符是过滤器而非内容,整串相似度
|
|
629
|
-
会被它主导——`keywords:pi-package mcp` 与 `keywords:pi-package
|
|
630
|
-
memory` 整串相似度 0.875,判别词只占几个字符,于是软命中把另一条
|
|
631
|
-
查询的结果交了出去(实测 `--engine npm` 查 mcp 拿到 memory 那批包)。
|
|
632
|
-
现在先要求限定符集合相同,再比**载荷**的相似度;无限定符的查询载荷
|
|
633
|
-
即整串,行为不变。
|
|
634
|
-
|
|
635
|
-
mode/depth 隔离(2026-09-19 修复):这是同一类串味的第三处,前两处
|
|
636
|
-
(engine、限定符)修的时候漏了它。`_key` 刻意把 mode/depth 编进键、
|
|
637
|
-
类 docstring 也承诺「depth / mode 隔离,防 fast/deep、budget 污染」,
|
|
638
|
-
但软命中的 WHERE 只有 domain+engine:`--depth deep` 请求会软命中一条
|
|
639
|
-
按 fast 写的条目(或反过来),拿到的结果集与请求档位不匹配,而结果里
|
|
640
|
-
还报 `cached: true`。现在 WHERE 带上 mode/depth 精确匹配——它们本来
|
|
641
|
-
就是「结果集全集的组成部分」,与限定符同理:档位不同,结果就不可互换。
|
|
600
|
+
返回 [{key, query, similarity}],按相似度降序。
|
|
601
|
+
隔离契约:
|
|
602
|
+
- engine 精确匹配(多引擎组合 a+b / a,b 不参与软命中)
|
|
603
|
+
- 限定符集合必须一致,仅对载荷计算相似度
|
|
604
|
+
- domain / mode / depth 精确匹配
|
|
642
605
|
"""
|
|
643
606
|
nq = normalize_query(query)
|
|
644
607
|
nq_quals, nq_payload = split_qualifiers(nq)
|
|
@@ -847,6 +810,7 @@ class SearchCache:
|
|
|
847
810
|
self._l1 = LRUCache(max_size=MAX_MEMORY_ITEMS)
|
|
848
811
|
self._l2 = SQLiteCache(db_path=self._db_path, ttl=self._ttl,
|
|
849
812
|
max_size_mb=self._max_size_mb)
|
|
813
|
+
self._miss_memo: tuple[float, frozenset[str]] = (0.0, frozenset())
|
|
850
814
|
|
|
851
815
|
@staticmethod
|
|
852
816
|
def _key(query: str, engine: str, max_results: int = 0, domain: str = "general",
|
|
@@ -931,6 +895,8 @@ class SearchCache:
|
|
|
931
895
|
out["_cache_level"] = "L1"
|
|
932
896
|
return out
|
|
933
897
|
self._l1.remove(key)
|
|
898
|
+
if time.monotonic() - self._miss_memo[0] < 0.05 and key in self._miss_memo[1]:
|
|
899
|
+
return None
|
|
934
900
|
|
|
935
901
|
hit = self._l2.get(key)
|
|
936
902
|
if hit is not None:
|
|
@@ -942,11 +908,32 @@ class SearchCache:
|
|
|
942
908
|
return out
|
|
943
909
|
return None
|
|
944
910
|
|
|
911
|
+
def _read_many(self, keys: list[str]) -> dict[str, dict]:
|
|
912
|
+
if not keys or self._disabled():
|
|
913
|
+
return {}
|
|
914
|
+
out, miss, now = {}, [], time.time()
|
|
915
|
+
for k in dict.fromkeys(keys):
|
|
916
|
+
hit = self._l1.get(k)
|
|
917
|
+
if hit is not None and hit.get("_ttl", 0) > 0 and now - hit.get("_ts", 0) < hit.get("_ttl", 0):
|
|
918
|
+
out[k] = hit
|
|
919
|
+
else:
|
|
920
|
+
if hit is not None:
|
|
921
|
+
self._l1.remove(k)
|
|
922
|
+
miss.append(k)
|
|
923
|
+
if miss:
|
|
924
|
+
l2_hits = self._l2.get_many(miss)
|
|
925
|
+
for k, hit in l2_hits.items():
|
|
926
|
+
self._l1.set(k, hit)
|
|
927
|
+
out[k] = hit
|
|
928
|
+
self._miss_memo = (time.monotonic(), frozenset(k for k in miss if k not in l2_hits))
|
|
929
|
+
return out
|
|
930
|
+
|
|
945
931
|
def _write(self, key: str, query: str, engine: str, max_results: int,
|
|
946
932
|
value: dict, domain: str, ttl: int,
|
|
947
933
|
mode: str = "", depth: str = "") -> None:
|
|
948
934
|
if self._disabled():
|
|
949
935
|
return
|
|
936
|
+
self._miss_memo = (0.0, frozenset())
|
|
950
937
|
payload = {**value, "_domain": domain, "_ttl": ttl, "_ts": time.time(),
|
|
951
938
|
"_max_results": max_results}
|
|
952
939
|
self._l1.set(key, payload)
|
|
@@ -1173,7 +1160,7 @@ class SearchCache:
|
|
|
1173
1160
|
返回 {url: {"body": {...}|None, "retrieval": {...}}},只含本地有记录的 URL。
|
|
1174
1161
|
"""
|
|
1175
1162
|
out: dict[str, dict] = {}
|
|
1176
|
-
if not urls:
|
|
1163
|
+
if not urls or self._disabled():
|
|
1177
1164
|
return out
|
|
1178
1165
|
try:
|
|
1179
1166
|
from fulltext_store import path_for as _ft_path
|
|
@@ -1183,14 +1170,15 @@ class SearchCache:
|
|
|
1183
1170
|
from robots_guard import known_blocked as _known_blocked
|
|
1184
1171
|
except ImportError:
|
|
1185
1172
|
_known_blocked = None
|
|
1173
|
+
try:
|
|
1174
|
+
hits = self._read_many([self._fetch_key(u) for u in urls if u])
|
|
1175
|
+
except IO_BENIGN + SHAPE_BENIGN:
|
|
1176
|
+
hits = {}
|
|
1186
1177
|
|
|
1187
1178
|
for url in urls:
|
|
1188
1179
|
if not url:
|
|
1189
1180
|
continue
|
|
1190
|
-
|
|
1191
|
-
hit = self._read(self._fetch_key(url))
|
|
1192
|
-
except IO_BENIGN + SHAPE_BENIGN:
|
|
1193
|
-
hit = None
|
|
1181
|
+
hit = hits.get(self._fetch_key(url))
|
|
1194
1182
|
entry: dict = {}
|
|
1195
1183
|
|
|
1196
1184
|
# ① 反面记录:以前抓过、结果不可用
|