argo-search 2.9.8 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +30 -20
- package/README.es.md +30 -20
- package/README.ja.md +30 -20
- package/README.ko.md +30 -20
- package/README.md +32 -22
- package/SKILL.md +1 -1
- package/bin/argo +4 -0
- package/bin/argo.js +1 -1
- package/config.yaml +52 -5
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/package.json +3 -2
- package/packages/dsh-plugin/README.md +3 -2
- package/packages/dsh-plugin/dsh/client.js +98 -32
- package/packages/dsh-plugin/dsh/diagnostics.js +43 -0
- package/packages/dsh-plugin/dsh/index.js +23 -1
- package/packages/dsh-plugin/package.json +2 -2
- package/packages/dsh-plugin/test/client.test.js +2 -1
- package/scripts/adaptive.py +102 -42
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_security.py +119 -4
- package/scripts/dispatch_result.py +9 -2
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +9 -0
- package/scripts/evidence_loop.py +4 -0
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +3 -3
- package/scripts/http_client.py +24 -38
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/matrix_search_eval.py +111 -1
- package/scripts/mcp_transport.py +1 -1
- package/scripts/net_proxy.py +27 -0
- package/scripts/recovery.py +7 -3
- package/scripts/route_combo.py +19 -5
- package/scripts/route_domains.py +138 -12
- package/scripts/safe_conn.py +52 -0
- package/scripts/search_entry.py +1 -0
- package/scripts/search_pipeline.py +20 -2
- package/scripts/search_rank.py +6 -0
- package/scripts/serp_guard.py +23 -0
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
package/scripts/fetch_v3.py
CHANGED
|
@@ -53,7 +53,7 @@ if SCRIPT_DIR not in sys.path:
|
|
|
53
53
|
|
|
54
54
|
# 渲染层(第二级A):TinyFish 直连渲染,独立模块以保持本文件聚焦主链调度
|
|
55
55
|
import fetch_render_tinyfish as _render_tinyfish
|
|
56
|
-
|
|
56
|
+
from clock_bridge import clock # 统一时钟(测试=虚拟;生产=monotonic)
|
|
57
57
|
# 质量信号层(第三级):来源分类 / 页面类型 / 质量分 / 内容安全
|
|
58
58
|
import fetch_quality as _quality
|
|
59
59
|
|
|
@@ -1353,14 +1353,14 @@ def fetch_v3(url: str, max_chars: int = 8000, timeout: float = 8.0,
|
|
|
1353
1353
|
deadline_s = float(os.environ.get("ARGO_FETCH_DEADLINE_S", "60") or 60)
|
|
1354
1354
|
except ValueError:
|
|
1355
1355
|
deadline_s = 60.0
|
|
1356
|
-
t_chain0 =
|
|
1356
|
+
t_chain0 = clock()
|
|
1357
1357
|
deadline_hit = {"flag": False}
|
|
1358
1358
|
|
|
1359
1359
|
def _budget_left() -> float:
|
|
1360
1360
|
"""剩余降级预算;耗尽时置标记并返回 -1(调用方停止升级)。"""
|
|
1361
1361
|
if deadline_s <= 0:
|
|
1362
1362
|
return 1.0
|
|
1363
|
-
left = deadline_s - (
|
|
1363
|
+
left = deadline_s - (clock() - t_chain0)
|
|
1364
1364
|
if left <= 0:
|
|
1365
1365
|
deadline_hit["flag"] = True
|
|
1366
1366
|
return -1.0
|
package/scripts/http_client.py
CHANGED
|
@@ -36,11 +36,22 @@ from contextlib import contextmanager
|
|
|
36
36
|
from typing import Any
|
|
37
37
|
|
|
38
38
|
try:
|
|
39
|
-
from url_safety import check_url
|
|
39
|
+
from url_safety import check_url, check_redirect_target
|
|
40
|
+
from url_safety import max_redirects as _max_redirects
|
|
41
|
+
from url_safety import resolve_redirect_chain
|
|
40
42
|
except ImportError: # pragma: no cover
|
|
41
43
|
def check_url(url: str) -> tuple[bool, str]:
|
|
42
44
|
return True, ""
|
|
43
45
|
|
|
46
|
+
def check_redirect_target(url: str, start: float) -> dict | None:
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
def _max_redirects() -> int:
|
|
50
|
+
return 5
|
|
51
|
+
|
|
52
|
+
def resolve_redirect_chain(url, headers, timeout, limit=None):
|
|
53
|
+
return None
|
|
54
|
+
|
|
44
55
|
|
|
45
56
|
# ─── User-Agent 轮换池 ───────────────────────────────────────────────────────
|
|
46
57
|
|
|
@@ -525,7 +536,7 @@ class HttpClient:
|
|
|
525
536
|
"""实际执行 GET 请求(使用 http.client,不自动解压)。"""
|
|
526
537
|
start = time.time()
|
|
527
538
|
current_url = url
|
|
528
|
-
redirects_left =
|
|
539
|
+
redirects_left = _max_redirects() if follow_redirects else 0
|
|
529
540
|
|
|
530
541
|
while True:
|
|
531
542
|
# 构建请求头
|
|
@@ -593,6 +604,11 @@ class HttpClient:
|
|
|
593
604
|
ch if ord(ch) < 128 else urllib.parse.quote(ch)
|
|
594
605
|
for ch in current_url
|
|
595
606
|
)
|
|
607
|
+
# 每跳重定向目标过 SSRF 校验(与 _safe_follow_redirects 对齐):
|
|
608
|
+
# 旧实现只校验首跳,重定向到内网即可绕过(SSRF)
|
|
609
|
+
blocked = check_redirect_target(current_url, start)
|
|
610
|
+
if blocked:
|
|
611
|
+
return blocked
|
|
596
612
|
redirects_left -= 1
|
|
597
613
|
continue
|
|
598
614
|
|
|
@@ -664,7 +680,7 @@ class HttpClient:
|
|
|
664
680
|
start = time.time()
|
|
665
681
|
headers = _random_headers(extra_headers)
|
|
666
682
|
|
|
667
|
-
final_url =
|
|
683
|
+
final_url = resolve_redirect_chain(url, headers, self.timeout)
|
|
668
684
|
if final_url is None:
|
|
669
685
|
return {"status": 0, "headers": {}, "text": "", "url": url,
|
|
670
686
|
"elapsed_ms": int((time.time() - start) * 1000),
|
|
@@ -868,7 +884,7 @@ class HttpClient:
|
|
|
868
884
|
"""
|
|
869
885
|
start = time.time()
|
|
870
886
|
current_url = url
|
|
871
|
-
redirects_left =
|
|
887
|
+
redirects_left = _max_redirects() if follow_redirects else 0
|
|
872
888
|
while True:
|
|
873
889
|
headers = _random_headers(extra_headers)
|
|
874
890
|
cookie_str = self._cookies.get_cookie_header(current_url)
|
|
@@ -914,6 +930,10 @@ class HttpClient:
|
|
|
914
930
|
if not loc:
|
|
915
931
|
break
|
|
916
932
|
current_url = urllib.parse.urljoin(current_url, loc)
|
|
933
|
+
# 每跳重定向目标过 SSRF 校验(与 _do_get/_safe_follow_redirects 对齐)
|
|
934
|
+
blocked = check_redirect_target(current_url, start)
|
|
935
|
+
if blocked:
|
|
936
|
+
return blocked
|
|
917
937
|
redirects_left -= 1
|
|
918
938
|
continue
|
|
919
939
|
raw_body = resp.read()
|
|
@@ -978,40 +998,6 @@ class HttpClient:
|
|
|
978
998
|
return {"status": 0, "headers": {}, "text": "", "url": url,
|
|
979
999
|
"elapsed_ms": 0, "error": str(e)[:200]}
|
|
980
1000
|
|
|
981
|
-
def _safe_follow_redirects(self, url: str, headers: dict,
|
|
982
|
-
max_redirects: int = 5) -> str | None:
|
|
983
|
-
"""逐跳安全跟随重定向:每跳目标都过 SSRF 校验。
|
|
984
|
-
|
|
985
|
-
返回最终 URL;任一跳被拦截或超过跳数返回 None。
|
|
986
|
-
"""
|
|
987
|
-
current = url
|
|
988
|
-
for _ in range(max_redirects):
|
|
989
|
-
ok, _reason = check_url(current)
|
|
990
|
-
if not ok:
|
|
991
|
-
return None
|
|
992
|
-
parsed = urllib.parse.urlparse(current)
|
|
993
|
-
if parsed.scheme not in ("http", "https"):
|
|
994
|
-
return None
|
|
995
|
-
try:
|
|
996
|
-
from net_proxy import open_connection, request_selector, resolve_proxy
|
|
997
|
-
proxy_url = resolve_proxy(current)
|
|
998
|
-
conn, via_proxy = open_connection(parsed, self.timeout, proxy_url)
|
|
999
|
-
path = parsed.path or "/"
|
|
1000
|
-
if parsed.query:
|
|
1001
|
-
path += "?" + parsed.query
|
|
1002
|
-
conn.request("HEAD", request_selector(parsed, path, via_proxy),
|
|
1003
|
-
headers=headers)
|
|
1004
|
-
resp = conn.getresponse()
|
|
1005
|
-
status = resp.status
|
|
1006
|
-
location = resp.getheader("Location")
|
|
1007
|
-
conn.close()
|
|
1008
|
-
except Exception:
|
|
1009
|
-
return None
|
|
1010
|
-
if status not in (301, 302, 303, 307, 308) or not location:
|
|
1011
|
-
return current
|
|
1012
|
-
current = urllib.parse.urljoin(current, location)
|
|
1013
|
-
return None
|
|
1014
|
-
|
|
1015
1001
|
|
|
1016
1002
|
# ─── 便捷函数 ────────────────────────────────────────────────────────────────
|
|
1017
1003
|
|
|
Binary file
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""本地个性化引擎插件 — 不随公开仓库提交。
|
|
3
|
+
|
|
4
|
+
包含漏洞查询类引擎(OSV.dev + CISA KEV),因启发式杀毒检测
|
|
5
|
+
(HEUR:HackTool/VulnScan)从公开构建器中移出。
|
|
6
|
+
本地文件,不提交 git;删除后对应引擎自动消失,不影响其他功能。
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
import urllib.request
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from engines_base import safe_search, rank_score, http_open, note_failure
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# ── OSV.dev 开源漏洞库(免 key,必须 POST)─────────────────────────────────
|
|
21
|
+
|
|
22
|
+
_OSV_ECOSYSTEM_PREFIX: dict[str, str] = {
|
|
23
|
+
"pypi": "PyPI", "npm": "npm", "go": "Go", "crates": "crates.io",
|
|
24
|
+
"cargo": "crates.io", "maven": "Maven", "nuget": "NuGet",
|
|
25
|
+
"packagist": "Packagist", "composer": "Packagist",
|
|
26
|
+
"rubygems": "RubyGems", "gem": "RubyGems", "hex": "Hex",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
_OSV_NPM_HINTS: frozenset[str] = frozenset({
|
|
30
|
+
"lodash", "react", "vue", "express", "axios", "next", "webpack",
|
|
31
|
+
"typescript", "eslint", "vite", "jest", "moment", "jquery", "angular",
|
|
32
|
+
"svelte", "nuxt", "rollup", "babel", "prettier", "cheerio", "socket.io",
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
_OSV_GO_HINTS: frozenset[str] = frozenset({
|
|
36
|
+
"golang.org/x/net", "golang.org/x/crypto", "golang.org/x/text",
|
|
37
|
+
"github.com/gin-gonic/gin", "github.com/gorilla/websocket",
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _osv_resolve_ecosystem(query: str) -> tuple[str, str]:
|
|
42
|
+
q = (query or "").strip()
|
|
43
|
+
if ":" in q:
|
|
44
|
+
head, _, tail = q.partition(":")
|
|
45
|
+
eco = _OSV_ECOSYSTEM_PREFIX.get(head.strip().lower())
|
|
46
|
+
if eco and tail.strip():
|
|
47
|
+
return tail.strip(), eco
|
|
48
|
+
name = q
|
|
49
|
+
if name.startswith("@") or "/" in name:
|
|
50
|
+
if name.startswith("@"):
|
|
51
|
+
return name, "npm"
|
|
52
|
+
if name.startswith(("github.com/", "gitlab.com/", "golang.org/")):
|
|
53
|
+
return name, "Go"
|
|
54
|
+
return name, "npm"
|
|
55
|
+
if name.lower() in _OSV_NPM_HINTS:
|
|
56
|
+
return name, "npm"
|
|
57
|
+
if name.lower() in _OSV_GO_HINTS:
|
|
58
|
+
return name, "Go"
|
|
59
|
+
return name, "PyPI"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _build_osv_engine(spec: dict[str, Any]) -> Any:
|
|
63
|
+
"""OSV.dev 漏洞查询(按包名,POST /v1/query;免 key、免限次)。"""
|
|
64
|
+
timeout = spec.get("timeout", 15)
|
|
65
|
+
|
|
66
|
+
@safe_search
|
|
67
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None,
|
|
68
|
+
**kwargs) -> list[dict[str, Any]]:
|
|
69
|
+
to = _timeout or timeout
|
|
70
|
+
q = (query or "").strip()
|
|
71
|
+
if not q:
|
|
72
|
+
return []
|
|
73
|
+
if re.match(r"^CVE-\d{4}-\d{4,}$", q, re.I):
|
|
74
|
+
return []
|
|
75
|
+
pkg, eco = _osv_resolve_ecosystem(q)
|
|
76
|
+
body = json.dumps({"package": {"name": pkg, "ecosystem": eco}}).encode("utf-8")
|
|
77
|
+
req = urllib.request.Request(
|
|
78
|
+
"https://api.osv.dev/v1/query", data=body,
|
|
79
|
+
headers={"Content-Type": "application/json"})
|
|
80
|
+
eng_name = spec.get("_name") or "osv"
|
|
81
|
+
try:
|
|
82
|
+
with http_open(req, timeout=to, engine=eng_name) as resp:
|
|
83
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
84
|
+
except Exception as e:
|
|
85
|
+
if isinstance(e, (json.JSONDecodeError, UnicodeDecodeError)):
|
|
86
|
+
note_failure(eng_name, "parse", "json_decode", str(e))
|
|
87
|
+
import logging
|
|
88
|
+
logging.getLogger("unified_search.engines").warning(f"OSV 失败: {e}")
|
|
89
|
+
return []
|
|
90
|
+
results = []
|
|
91
|
+
for i, v in enumerate((data.get("vulns") or [])[:n]):
|
|
92
|
+
vid = str(v.get("id") or "").strip()
|
|
93
|
+
if not vid:
|
|
94
|
+
continue
|
|
95
|
+
summary = str(v.get("summary") or "").strip()
|
|
96
|
+
affected_bits = []
|
|
97
|
+
for aff in (v.get("affected") or [])[:2]:
|
|
98
|
+
if not isinstance(aff, dict):
|
|
99
|
+
continue
|
|
100
|
+
for rng in (aff.get("ranges") or [])[:2]:
|
|
101
|
+
if not isinstance(rng, dict):
|
|
102
|
+
continue
|
|
103
|
+
events = rng.get("events") or []
|
|
104
|
+
introduced = next((e.get("introduced") for e in events
|
|
105
|
+
if isinstance(e, dict) and e.get("introduced")), "")
|
|
106
|
+
fixed = next((e.get("fixed") for e in events
|
|
107
|
+
if isinstance(e, dict) and e.get("fixed")), "")
|
|
108
|
+
if introduced or fixed:
|
|
109
|
+
affected_bits.append(
|
|
110
|
+
f"{introduced or '?'} ≤ 受影响 < {fixed or '未修复'}")
|
|
111
|
+
aliases = [a for a in (v.get("aliases") or []) if a][:3]
|
|
112
|
+
snippet = " · ".join(p for p in (
|
|
113
|
+
summary, f"生态 {eco}",
|
|
114
|
+
affected_bits[0] if affected_bits else "",
|
|
115
|
+
" ".join(aliases),
|
|
116
|
+
) if p)[:300]
|
|
117
|
+
results.append({
|
|
118
|
+
"title": f"{vid} [{eco}] {summary}"[:500] or vid,
|
|
119
|
+
"url": f"https://osv.dev/vulnerability/{vid}",
|
|
120
|
+
"snippet": snippet,
|
|
121
|
+
"source": "osv",
|
|
122
|
+
"published_at": str(v.get("published") or "")[:10],
|
|
123
|
+
"score": rank_score(0.85, i),
|
|
124
|
+
})
|
|
125
|
+
return results
|
|
126
|
+
return _engine
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ── CISA KEV 已知在野利用漏洞(免 key,全量文件 + 本地过滤)──────────────
|
|
130
|
+
|
|
131
|
+
_KEV_CACHE: dict[str, Any] = {"at": 0.0, "items": None}
|
|
132
|
+
_KEV_TTL_S = 6 * 3600
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _kev_disk_path():
|
|
136
|
+
try:
|
|
137
|
+
from argo_paths import state_path
|
|
138
|
+
return state_path("cisa_kev_cache.json")
|
|
139
|
+
except Exception:
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _kev_load(timeout: float) -> list[dict[str, Any]]:
|
|
144
|
+
now = time.time()
|
|
145
|
+
cached = _KEV_CACHE.get("items")
|
|
146
|
+
if cached and (now - float(_KEV_CACHE.get("at") or 0)) < _KEV_TTL_S:
|
|
147
|
+
return cached
|
|
148
|
+
disk_p = _kev_disk_path()
|
|
149
|
+
if disk_p is not None and disk_p.exists():
|
|
150
|
+
try:
|
|
151
|
+
st = disk_p.stat()
|
|
152
|
+
if (now - st.st_mtime) < _KEV_TTL_S:
|
|
153
|
+
disk_items = json.loads(disk_p.read_text(encoding="utf-8"))
|
|
154
|
+
if isinstance(disk_items, list) and disk_items:
|
|
155
|
+
_KEV_CACHE["items"] = disk_items
|
|
156
|
+
_KEV_CACHE["at"] = st.st_mtime
|
|
157
|
+
return disk_items
|
|
158
|
+
except Exception:
|
|
159
|
+
pass
|
|
160
|
+
req = urllib.request.Request(
|
|
161
|
+
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
|
|
162
|
+
headers={"User-Agent": "argo-search/1.0 (+cisa-kev)", "Accept": "application/json"})
|
|
163
|
+
try:
|
|
164
|
+
with http_open(req, timeout=timeout, engine="cisa_kev") as resp:
|
|
165
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
166
|
+
except Exception as e:
|
|
167
|
+
if isinstance(e, (json.JSONDecodeError, UnicodeDecodeError)):
|
|
168
|
+
note_failure("cisa_kev", "parse", "json_decode", str(e))
|
|
169
|
+
import logging
|
|
170
|
+
logging.getLogger("unified_search.engines").warning(f"CISA KEV 失败: {e}")
|
|
171
|
+
return cached or []
|
|
172
|
+
items = data.get("vulnerabilities") or []
|
|
173
|
+
if not isinstance(items, list):
|
|
174
|
+
return cached or []
|
|
175
|
+
_KEV_CACHE["items"] = items
|
|
176
|
+
_KEV_CACHE["at"] = now
|
|
177
|
+
if disk_p is not None and items:
|
|
178
|
+
try:
|
|
179
|
+
from argo_paths import atomic_write_json
|
|
180
|
+
atomic_write_json(disk_p, items)
|
|
181
|
+
except Exception:
|
|
182
|
+
pass
|
|
183
|
+
return items
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _build_cisa_kev_engine(spec: dict[str, Any]) -> Any:
|
|
187
|
+
"""CISA KEV 在野利用漏洞目录(本地过滤全量目录)。"""
|
|
188
|
+
timeout = spec.get("timeout", 30)
|
|
189
|
+
|
|
190
|
+
@safe_search
|
|
191
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None,
|
|
192
|
+
**kwargs) -> list[dict[str, Any]]:
|
|
193
|
+
to = _timeout or timeout
|
|
194
|
+
q = (query or "").strip()
|
|
195
|
+
if not q:
|
|
196
|
+
return []
|
|
197
|
+
items = _kev_load(to)
|
|
198
|
+
if not items:
|
|
199
|
+
return []
|
|
200
|
+
cve_q = q.upper() if re.match(r"^CVE-\d{4}-\d{4,}$", q.strip(), re.I) else ""
|
|
201
|
+
matched = []
|
|
202
|
+
for v in items:
|
|
203
|
+
if not isinstance(v, dict):
|
|
204
|
+
continue
|
|
205
|
+
if cve_q:
|
|
206
|
+
if str(v.get("cveID") or "").strip().upper() == cve_q:
|
|
207
|
+
matched.append(v)
|
|
208
|
+
continue
|
|
209
|
+
terms = [t for t in re.split(r"[\s,]+", q.lower()) if t]
|
|
210
|
+
hay = " ".join(str(v.get(k) or "") for k in (
|
|
211
|
+
"cveID", "vendorProject", "product", "vulnerabilityName",
|
|
212
|
+
"shortDescription", "requiredAction", "cwes")).lower()
|
|
213
|
+
if all(t in hay for t in terms):
|
|
214
|
+
matched.append(v)
|
|
215
|
+
matched.sort(key=lambda x: str(x.get("dateAdded") or ""), reverse=True)
|
|
216
|
+
results = []
|
|
217
|
+
for i, v in enumerate(matched[:n]):
|
|
218
|
+
cve = str(v.get("cveID") or "").strip()
|
|
219
|
+
if not cve:
|
|
220
|
+
continue
|
|
221
|
+
vendor = str(v.get("vendorProject") or "").strip()
|
|
222
|
+
product = str(v.get("product") or "").strip()
|
|
223
|
+
name = str(v.get("vulnerabilityName") or "").strip()
|
|
224
|
+
desc = str(v.get("shortDescription") or "").strip()
|
|
225
|
+
due = str(v.get("dueDate") or "").strip()
|
|
226
|
+
ransom = str(v.get("knownRansomwareCampaignUse") or "").strip()
|
|
227
|
+
bits = [b for b in (
|
|
228
|
+
f"{vendor} {product}".strip(),
|
|
229
|
+
f"修复期限 {due}" if due else "",
|
|
230
|
+
"已知勒索软件利用" if ransom.lower() == "known" else "",
|
|
231
|
+
desc,
|
|
232
|
+
) if b]
|
|
233
|
+
results.append({
|
|
234
|
+
"title": f"{cve} — {name or product or vendor}"[:500],
|
|
235
|
+
"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog",
|
|
236
|
+
"snippet": " · ".join(bits)[:300],
|
|
237
|
+
"source": "cisa_kev",
|
|
238
|
+
"published_at": str(v.get("dateAdded") or "")[:10],
|
|
239
|
+
"score": rank_score(0.9, i),
|
|
240
|
+
})
|
|
241
|
+
return results
|
|
242
|
+
return _engine
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# ── 引擎声明(本地个性化配置,不提交公开仓库)────────────────────────────────
|
|
246
|
+
#
|
|
247
|
+
# 字段与原 engines/specs/osv.yaml、engines/specs/cisa_kev.yaml 逐项对齐——
|
|
248
|
+
# 那两个声明已加 _ 前缀改名停用(公开仓每次启动刷两行「未知引擎类型」warning),
|
|
249
|
+
# 元数据随之并入本文件,本机路由行为(explicit_only / family / priority / qps
|
|
250
|
+
# 等)与改名前完全一致。
|
|
251
|
+
|
|
252
|
+
_LOCAL_ENGINE_SPECS: dict[str, dict[str, Any]] = {
|
|
253
|
+
"osv": {
|
|
254
|
+
"type": "osv",
|
|
255
|
+
"enabled": True,
|
|
256
|
+
"explicit_only": True,
|
|
257
|
+
"cost_tier": "free",
|
|
258
|
+
"label": "OSV 开源漏洞库",
|
|
259
|
+
"family": "security",
|
|
260
|
+
"priority": 44,
|
|
261
|
+
"qps": 2,
|
|
262
|
+
"period": "second",
|
|
263
|
+
"cost_per_call": 0.0,
|
|
264
|
+
"cost_unit": "free",
|
|
265
|
+
"method": "POST",
|
|
266
|
+
"url": "https://api.osv.dev/v1/query",
|
|
267
|
+
"timeout": 15,
|
|
268
|
+
"canary_query": "requests",
|
|
269
|
+
"desc": "OSV.dev 开源软件漏洞库(按包查已知漏洞与影响版本区间,免 key;生态 PyPI/npm/Go/crates 等)",
|
|
270
|
+
"description": "OSV.dev 开源漏洞查询(按包名)",
|
|
271
|
+
"coverage": ["security", "package"],
|
|
272
|
+
"recommended": False,
|
|
273
|
+
"latency_ms": 1500,
|
|
274
|
+
"headers": {"Content-Type": "application/json"},
|
|
275
|
+
"quality_queries": [
|
|
276
|
+
{"query": "requests"},
|
|
277
|
+
{"query": "lodash"},
|
|
278
|
+
{"query": "log4j-core"},
|
|
279
|
+
],
|
|
280
|
+
},
|
|
281
|
+
"cisa_kev": {
|
|
282
|
+
"type": "cisa_kev",
|
|
283
|
+
"enabled": True,
|
|
284
|
+
"explicit_only": True,
|
|
285
|
+
"cost_tier": "free",
|
|
286
|
+
"label": "CISA KEV 在野利用漏洞",
|
|
287
|
+
"family": "security",
|
|
288
|
+
"priority": 47,
|
|
289
|
+
"qps": 1,
|
|
290
|
+
"period": "second",
|
|
291
|
+
"cost_per_call": 0.0,
|
|
292
|
+
"cost_unit": "free",
|
|
293
|
+
"timeout": 30,
|
|
294
|
+
"canary_query": "chrome",
|
|
295
|
+
"desc": "CISA 已知被在野利用漏洞目录(官方确认已被实际攻击利用的 CVE;全量 1.35MB 本地过滤)",
|
|
296
|
+
"description": "CISA KEV 在野利用漏洞目录",
|
|
297
|
+
"coverage": ["security"],
|
|
298
|
+
"recommended": False,
|
|
299
|
+
"latency_ms": 4000,
|
|
300
|
+
"quality_queries": [
|
|
301
|
+
{"query": "chrome"},
|
|
302
|
+
{"query": "CVE-2021-44228"},
|
|
303
|
+
],
|
|
304
|
+
},
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def register(builders: dict[str, Any], engine_specs: dict[str, Any]) -> None:
|
|
309
|
+
"""将本地引擎注册到全局 builders 和 engine_specs 中。
|
|
310
|
+
|
|
311
|
+
在 engines.py 的 _load_registry() 中、构建 registry 之前调用。
|
|
312
|
+
文件不存在时 engines.py 跳过 import,不影响公开仓库。
|
|
313
|
+
外置 spec(engines/specs/*.yaml)已有的字段保留,不抹除 explicit_only/family 等元数据。
|
|
314
|
+
"""
|
|
315
|
+
builders["osv"] = _build_osv_engine
|
|
316
|
+
builders["cisa_kev"] = _build_cisa_kev_engine
|
|
317
|
+
for name, spec in _LOCAL_ENGINE_SPECS.items():
|
|
318
|
+
base = dict(engine_specs.get(name) or {})
|
|
319
|
+
for k, v in spec.items():
|
|
320
|
+
base.setdefault(k, v)
|
|
321
|
+
engine_specs[name] = base
|
|
@@ -269,6 +269,50 @@ ROUTE_MATRIX: list[dict[str, Any]] = [
|
|
|
269
269
|
"domain": "company_search", "primary": "opencorporates", "scenario": "company"},
|
|
270
270
|
{"id": "R_zh_company_regcap", "q": "字节跳动 注册资本 变更", "lang": "zh",
|
|
271
271
|
"domain": "company_search", "primary": "opencorporates", "scenario": "company"},
|
|
272
|
+
# 2026-10-25 补(企业主体查询 MECE 类目):尽调风险(注销/经营异常)、开票
|
|
273
|
+
# (开票信息/税号)、人查公司(担任法定代表人/名下公司)三类此前落
|
|
274
|
+
# org_entity 的 wikidata 组合——wikidata 对中文企业 7 天空结果率 82%,
|
|
275
|
+
# company_search 的元宝才是交付源。config.yaml 两域词表成对改动
|
|
276
|
+
# (org_entity 让位 unless + company_search 收口 match/unless),改一处
|
|
277
|
+
# 必须同步另一处。企业名/人名一律合成名(对外脱敏纪律,v2.9.8 同款)。
|
|
278
|
+
{"id": "R_zh_company_risk", "q": "晨曦合成科技有限公司 注销了吗", "lang": "zh",
|
|
279
|
+
"domain": "company_search", "primary": "opencorporates", "scenario": "company"},
|
|
280
|
+
{"id": "R_zh_company_invoice", "q": "晨曦合成科技有限公司 开票信息 税号", "lang": "zh",
|
|
281
|
+
"domain": "company_search", "primary": "opencorporates", "scenario": "company"},
|
|
282
|
+
{"id": "R_zh_company_person", "q": "李合成 担任法定代表人的公司", "lang": "zh",
|
|
283
|
+
"domain": "company_search", "primary": "opencorporates", "scenario": "company"},
|
|
284
|
+
# 双向锁(v2.9.8 纪律):新词表不得误吸非主体查询——办事指南是 howto、
|
|
285
|
+
# 招聘是求职,都不该进尽调域
|
|
286
|
+
{"id": "R_zh_company_not_howto", "q": "公司注销流程 怎么办", "lang": "zh",
|
|
287
|
+
"domain_not": ["company_search", "org_entity"], "scenario": "company"},
|
|
288
|
+
{"id": "R_zh_company_not_job", "q": "晨曦合成科技有限公司 招聘", "lang": "zh",
|
|
289
|
+
"domain_not": ["company_search"], "scenario": "company"},
|
|
290
|
+
# ── 2026-10-25 六族 MECE 扫描(学术/金融财务/法律/医药)──
|
|
291
|
+
# 全部为真实场景类目,企业名/人名保持合成名(脱敏纪律)。每处 config 改动
|
|
292
|
+
# 都有成对的 unless 让位,反例见 tests/test_domain_rule_schema.py。
|
|
293
|
+
{"id": "R_zh_stock_financials", "q": "贵州茅台 2025 年报 营收", "lang": "zh",
|
|
294
|
+
"domain": "stock_query", "primary": "sina_quote", "scenario": "finance"},
|
|
295
|
+
{"id": "R_zh_stock_valuation", "q": "贵州茅台 市盈率 估值", "lang": "zh",
|
|
296
|
+
"domain": "stock_query", "primary": "sina_quote", "scenario": "finance"},
|
|
297
|
+
# zh 财务词形不得落美股域(sec_companyfacts 只认 CIK,A 股全废)
|
|
298
|
+
{"id": "R_zh_usfund_yields_cn", "q": "宁德时代 毛利率 现金流", "lang": "zh",
|
|
299
|
+
"domain_not": ["us_fundamentals"], "scenario": "finance"},
|
|
300
|
+
# 民事实务(劳动/借贷)此前落 chinese_general,flk_law 才是交付源
|
|
301
|
+
{"id": "R_zh_legal_labor", "q": "劳动合同解除 赔偿 N+1", "lang": "zh",
|
|
302
|
+
"domain": "legal", "primary": "anysearch", "scenario": "legal"},
|
|
303
|
+
{"id": "R_zh_legal_lending", "q": "民间借贷 利率 司法保护上限", "lang": "zh",
|
|
304
|
+
"domain": "legal", "primary": "anysearch", "scenario": "legal"},
|
|
305
|
+
# 药品说明书此前被 code_search 的「用法」子串抢走(GitHub/MDN 答不了药)
|
|
306
|
+
{"id": "R_zh_medical_drug_label", "q": "布洛芬 说明书 用法用量", "lang": "zh",
|
|
307
|
+
"domain": "medical", "primary": "clinicaltrials", "scenario": "medical"},
|
|
308
|
+
# 综述语境不被模型目录抢(models_dev 答不了综述)
|
|
309
|
+
{"id": "R_zh_survey_not_modelcard", "q": "大模型 长上下文 综述", "lang": "zh",
|
|
310
|
+
"domain_not": ["ai_model"], "scenario": "academic"},
|
|
311
|
+
# 论文写作 howto / 年报模板 howto 不进学术域与行情域
|
|
312
|
+
{"id": "R_zh_academic_not_howto", "q": "怎么写论文 查重 降重", "lang": "zh",
|
|
313
|
+
"domain_not": ["academic"], "scenario": "academic"},
|
|
314
|
+
{"id": "R_zh_stock_not_howto", "q": "年报 模板 怎么写", "lang": "zh",
|
|
315
|
+
"domain_not": ["stock_query", "us_fundamentals"], "scenario": "finance"},
|
|
272
316
|
|
|
273
317
|
# ── 媒体/音乐 ──
|
|
274
318
|
{"id": "R_en_media", "q": "Taylor Swift album", "lang": "en",
|
|
@@ -449,7 +493,15 @@ ROUTE_MATRIX: list[dict[str, Any]] = [
|
|
|
449
493
|
{"id": "R_stackoverflow_search", "q": "stackoverflow 编程问题 代码问题", "lang": "zh", "domain": "stackoverflow_search", "primary": "stackoverflow", "scenario": "code"},
|
|
450
494
|
{"id": "R_standards", "q": "国家标准 国标 标准号", "lang": "zh", "domain": "standards", "primary": "std_samr", "scenario": "academic"},
|
|
451
495
|
{"id": "R_tech_deep", "q": "deep dive 架构 实践", "lang": "zh", "domain": "tech_deep", "primary": "arxiv", "scenario": "tech"},
|
|
452
|
-
|
|
496
|
+
# 2026-10-25 约定更新:zh 词形财务查询统一走 stock_query(sina/eastmoney
|
|
497
|
+
# 双地覆盖,与「XX 股价」分流同构);本域收英文词形与美股语境。代价是
|
|
498
|
+
# zh 美股财务查询损失 sec 结构化精度,换来 A 股企业不再落只认 CIK 的
|
|
499
|
+
# sec_companyfacts 全空——两害取其轻,成对改动注释见 config us_fundamentals。
|
|
500
|
+
# 英文方向的域锁定另见 R_us_fundamentals_en。
|
|
501
|
+
{"id": "R_us_fundamentals", "q": "营收 净利润 财务数据 苹果", "lang": "zh",
|
|
502
|
+
"domain": "stock_query", "primary": "sina_quote", "scenario": "finance"},
|
|
503
|
+
{"id": "R_us_fundamentals_en", "q": "apple annual revenue net income", "lang": "en",
|
|
504
|
+
"domain": "us_fundamentals", "primary": "sec_companyfacts", "scenario": "finance"},
|
|
453
505
|
{"id": "R_us_legal", "q": "supreme court 判例 precedent", "lang": "en", "domain": "us_legal", "primary": "federal_register", "scenario": "legal"},
|
|
454
506
|
{"id": "R_web_archive", "q": "网页快照 历史网页 wayback", "lang": "zh", "domain": "web_archive", "primary": "wikipedia", "scenario": "general"},
|
|
455
507
|
{"id": "R_web_docs", "q": "mozilla developer fetch 属性", "lang": "en", "domain": "web_docs", "primary": "mdn", "scenario": "code"},
|
|
@@ -1131,7 +1183,65 @@ def _summarize(c: Checker) -> dict[str, Any]:
|
|
|
1131
1183
|
}
|
|
1132
1184
|
|
|
1133
1185
|
|
|
1186
|
+
def _reexec_with_yaml() -> bool:
|
|
1187
|
+
"""缺 PyYAML 时改用带依赖的解释器重入;exec 成功则不返回。
|
|
1188
|
+
|
|
1189
|
+
why:config 在无 yaml 时静默回落内置默认引擎集,route 段整片假红
|
|
1190
|
+
(2026-10-25 实测:同机无 yaml 的解释器跑出 94 FAIL、带 yaml 的全绿)。
|
|
1191
|
+
bin/argo 启动器的 _pick_python 会自动挑解释器,但分享场景里用户会直接
|
|
1192
|
+
`python3 scripts/matrix_search_eval.py`——解释器版本不该卡住人。候选序:
|
|
1193
|
+
ARGO_PYTHON(与启动器同一权威覆盖)→ PATH 上的 python3.14/3.13/3.15/
|
|
1194
|
+
3.12 → python3;每个候选真跑一次 `import yaml` 验证(探测约 100ms,
|
|
1195
|
+
评测不是热路径)。已重入过仍缺(_ARGO_MATRIX_REEXEC 标记)则放弃,
|
|
1196
|
+
避免套娃。
|
|
1197
|
+
"""
|
|
1198
|
+
import os
|
|
1199
|
+
import shutil
|
|
1200
|
+
import subprocess
|
|
1201
|
+
|
|
1202
|
+
if os.environ.get("_ARGO_MATRIX_REEXEC") == "1":
|
|
1203
|
+
return False
|
|
1204
|
+
cands = []
|
|
1205
|
+
env_py = os.environ.get("ARGO_PYTHON", "").strip()
|
|
1206
|
+
if env_py:
|
|
1207
|
+
cands.append(env_py)
|
|
1208
|
+
cands += [shutil.which("python3.%d" % m) or "" for m in (14, 13, 15, 12)]
|
|
1209
|
+
cands.append(shutil.which("python3") or "")
|
|
1210
|
+
for cand in cands:
|
|
1211
|
+
if not cand:
|
|
1212
|
+
continue
|
|
1213
|
+
if os.path.realpath(cand) == os.path.realpath(sys.executable):
|
|
1214
|
+
continue # 当前解释器已证明缺 yaml,跳过
|
|
1215
|
+
try:
|
|
1216
|
+
probe = subprocess.run([cand, "-c", "import yaml"],
|
|
1217
|
+
capture_output=True, timeout=30)
|
|
1218
|
+
except Exception:
|
|
1219
|
+
continue
|
|
1220
|
+
if probe.returncode == 0:
|
|
1221
|
+
os.environ["_ARGO_MATRIX_REEXEC"] = "1"
|
|
1222
|
+
print("[matrix] 当前解释器缺 PyYAML,改用 %s 重入" % cand,
|
|
1223
|
+
file=sys.stderr)
|
|
1224
|
+
os.execv(cand, [cand, sys.argv[0], *sys.argv[1:]])
|
|
1225
|
+
return False
|
|
1226
|
+
|
|
1227
|
+
|
|
1134
1228
|
def main() -> int:
|
|
1229
|
+
# PyYAML 缺失时 config 会**静默**回落到内置默认引擎集(垂直域引擎全不在),
|
|
1230
|
+
# route 段整片假红(2026-10-25 实测:无 yaml 的解释器跑出 94 FAIL,带 yaml
|
|
1231
|
+
# 重跑 197 全绿——错误根因是解释器不是路由)。先自愈换解释器重入
|
|
1232
|
+
# (3.15/3.14 等版本差异不再卡用户),换无可换才硬失败退出。
|
|
1233
|
+
try:
|
|
1234
|
+
import yaml # noqa: F401
|
|
1235
|
+
except ImportError as exc:
|
|
1236
|
+
_reexec_with_yaml()
|
|
1237
|
+
print(
|
|
1238
|
+
f"[matrix] PyYAML 不可用({exc})且候选解释器(ARGO_PYTHON 与 PATH "
|
|
1239
|
+
"上的 python3.x)均缺:config 将静默降级到内置默认引擎集,route 段"
|
|
1240
|
+
"会整片假红。请 pip install pyyaml,或设 ARGO_PYTHON 指向带依赖的"
|
|
1241
|
+
"解释器。",
|
|
1242
|
+
file=sys.stderr,
|
|
1243
|
+
)
|
|
1244
|
+
return 2
|
|
1135
1245
|
ap = argparse.ArgumentParser(description="多语言×引擎×场景 矩阵测试")
|
|
1136
1246
|
ap.add_argument("--offline", action="store_true")
|
|
1137
1247
|
ap.add_argument("--live", action="store_true")
|
package/scripts/mcp_transport.py
CHANGED
package/scripts/net_proxy.py
CHANGED
|
@@ -217,6 +217,19 @@ def open_url(req: Any, timeout: float = 10.0):
|
|
|
217
217
|
return opener.open(req, timeout=timeout)
|
|
218
218
|
|
|
219
219
|
|
|
220
|
+
def _verified_tls_context():
|
|
221
|
+
"""显式构造「校验开启」的 TLS 上下文(check_hostname + CERT_REQUIRED)。
|
|
222
|
+
|
|
223
|
+
默认 http.client 已用校验上下文;这里显式化,让「TLS 校验不可被静默关掉」
|
|
224
|
+
成为可测试的不变式(而不是依赖库默认)。
|
|
225
|
+
"""
|
|
226
|
+
import ssl
|
|
227
|
+
ctx = ssl.create_default_context()
|
|
228
|
+
ctx.check_hostname = True
|
|
229
|
+
ctx.verify_mode = ssl.CERT_REQUIRED
|
|
230
|
+
return ctx
|
|
231
|
+
|
|
232
|
+
|
|
220
233
|
def open_connection(parsed: ParseResult, timeout: float,
|
|
221
234
|
proxy_url: str | None) -> tuple[HTTPConnection, bool]:
|
|
222
235
|
"""按是否走代理构造 http.client 连接。返回 (conn, via_proxy)。
|
|
@@ -224,12 +237,26 @@ def open_connection(parsed: ParseResult, timeout: float,
|
|
|
224
237
|
https 目标 + 代理 → 到代理的 HTTPSConnection + set_tunnel(CONNECT 隧道);
|
|
225
238
|
http 目标 + 代理 → 到代理的 HTTPConnection,request 须用绝对 URL
|
|
226
239
|
(见 request_selector)。
|
|
240
|
+
|
|
241
|
+
直连(无代理):用「解析即锁定」的连接类(url_safety.Safe*Connection),
|
|
242
|
+
堵 check_url 与 connect 之间的 DNS 重绑定窗口;HTTPS 显式用校验上下文。
|
|
227
243
|
"""
|
|
228
244
|
import http.client
|
|
229
245
|
import urllib.parse
|
|
230
246
|
|
|
231
247
|
target_port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
232
248
|
if not proxy_url:
|
|
249
|
+
# 直连:解析即锁定 + TLS 校验(详见 safe_conn 与 _verified_tls_context)
|
|
250
|
+
try:
|
|
251
|
+
from safe_conn import SafeHTTPConnection, SafeHTTPSConnection
|
|
252
|
+
if parsed.scheme == "https":
|
|
253
|
+
return SafeHTTPSConnection(
|
|
254
|
+
parsed.hostname, target_port, timeout=timeout,
|
|
255
|
+
context=_verified_tls_context()), False
|
|
256
|
+
return SafeHTTPConnection(parsed.hostname, target_port,
|
|
257
|
+
timeout=timeout), False
|
|
258
|
+
except ImportError: # pragma: no cover
|
|
259
|
+
pass
|
|
233
260
|
cls = (http.client.HTTPSConnection if parsed.scheme == "https"
|
|
234
261
|
else http.client.HTTPConnection)
|
|
235
262
|
return cls(parsed.hostname, target_port, timeout=timeout), False
|