argo-search 2.9.7 → 2.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/README.en.md +73 -150
  2. package/README.es.md +73 -150
  3. package/README.ja.md +79 -156
  4. package/README.ko.md +78 -155
  5. package/README.md +80 -159
  6. package/SKILL.md +3 -3
  7. package/assets/readme/hero.svg +2 -2
  8. package/backends/domain_profiles.json +48 -0
  9. package/backends/engine_registry.yaml +118 -4
  10. package/backends/quota_profiles.json +92 -12
  11. package/backends/source_types_cn.json +27 -1
  12. package/bin/argo +78 -5
  13. package/bin/argo.js +33 -2
  14. package/config.yaml +269 -63
  15. package/docs/ARGO_INTRO.md +5 -3
  16. package/docs/ENGINE_CATALOG.md +50 -39
  17. package/docs/MCP_SETUP.md +3 -1
  18. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  19. package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
  20. package/engines/specs/cfets.yaml +41 -0
  21. package/engines/specs/defillama.yaml +38 -0
  22. package/engines/specs/firecrawl.yaml +1 -1
  23. package/engines/specs/fxtwitter.yaml +5 -0
  24. package/engines/specs/gleif.yaml +55 -0
  25. package/engines/specs/hkex.yaml +40 -0
  26. package/engines/specs/keenable_free.yaml +36 -0
  27. package/engines/specs/shfe.yaml +39 -0
  28. package/engines/specs/stackexchange.yaml +2 -2
  29. package/engines/specs/tavily_free.yaml +56 -0
  30. package/engines/specs/yuanbao.yaml +93 -0
  31. package/mcp/clients.yaml +32 -0
  32. package/package.json +7 -2
  33. package/packages/dsh-plugin/README.md +25 -6
  34. package/packages/dsh-plugin/dsh/client.js +402 -47
  35. package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
  36. package/packages/dsh-plugin/dsh/index.js +135 -1876
  37. package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
  38. package/packages/dsh-plugin/dsh/runner.js +666 -0
  39. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  40. package/packages/dsh-plugin/package.json +3 -3
  41. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  42. package/packages/dsh-plugin/test/client.test.js +177 -2
  43. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  44. package/packages/dsh-plugin/test/index.test.js +44 -0
  45. package/scripts/adaptive.py +126 -43
  46. package/scripts/article_meta.py +131 -0
  47. package/scripts/bench_tail_ab.py +281 -0
  48. package/scripts/bounded_run.py +91 -49
  49. package/scripts/budget.py +247 -0
  50. package/scripts/budget_cli.py +93 -0
  51. package/scripts/build_release_assets.sh +65 -0
  52. package/scripts/cache.py +60 -72
  53. package/scripts/ci_drift_gate.py +90 -0
  54. package/scripts/clock_bridge.py +22 -0
  55. package/scripts/content_boundary.py +46 -0
  56. package/scripts/content_security.py +119 -4
  57. package/scripts/content_signals.py +26 -5
  58. package/scripts/cross_lingual_trace.py +293 -0
  59. package/scripts/dispatch_result.py +105 -0
  60. package/scripts/engine_add.py +602 -0
  61. package/scripts/engine_dispatch.py +58 -86
  62. package/scripts/engine_env.py +5 -0
  63. package/scripts/engine_families.py +6 -0
  64. package/scripts/engine_policy.py +7 -7
  65. package/scripts/engines.py +21 -0
  66. package/scripts/engines_base.py +3 -3
  67. package/scripts/engines_builders.py +12 -0
  68. package/scripts/engines_builders_batch10.py +48 -3
  69. package/scripts/engines_builders_batch14.py +656 -0
  70. package/scripts/engines_builders_batch15.py +249 -0
  71. package/scripts/engines_builders_keyless.py +111 -0
  72. package/scripts/engines_builders_tech.py +0 -1
  73. package/scripts/evidence.py +31 -3
  74. package/scripts/evidence_loop.py +58 -22
  75. package/scripts/fetch_quality.py +13 -0
  76. package/scripts/fetch_v3.py +28 -3
  77. package/scripts/gen_routing_snapshot.py +11 -0
  78. package/scripts/http_client.py +24 -38
  79. package/scripts/install.ps1 +18 -0
  80. package/scripts/install.sh +16 -1
  81. package/scripts/job.py +3 -3
  82. package/scripts/local_seek.py +41 -5
  83. package/scripts/macro_countries.py +15 -0
  84. package/scripts/matrix_search_eval.py +192 -0
  85. package/scripts/mcp_compact.py +234 -0
  86. package/scripts/mcp_handlers.py +60 -642
  87. package/scripts/mcp_launch.sh +10 -8
  88. package/scripts/mcp_local.py +241 -0
  89. package/scripts/mcp_server.py +19 -14
  90. package/scripts/mcp_setup.py +33 -9
  91. package/scripts/mcp_tools.py +1 -0
  92. package/scripts/mcp_transport.py +1 -1
  93. package/scripts/mcp_warm.py +127 -0
  94. package/scripts/net_proxy.py +27 -0
  95. package/scripts/output_transparency.py +200 -0
  96. package/scripts/paid_policy.py +98 -0
  97. package/scripts/quota.py +158 -13
  98. package/scripts/recovery.py +7 -3
  99. package/scripts/recovery_runner.py +150 -0
  100. package/scripts/replay_eval.py +18 -0
  101. package/scripts/research.py +37 -21
  102. package/scripts/research_expand.py +24 -1
  103. package/scripts/retry_policy.py +76 -0
  104. package/scripts/route.py +6 -6
  105. package/scripts/route_cache.py +18 -0
  106. package/scripts/route_combo.py +26 -5
  107. package/scripts/route_domains.py +170 -31
  108. package/scripts/safe_conn.py +52 -0
  109. package/scripts/search.py +15 -27
  110. package/scripts/search_attribution.py +100 -3
  111. package/scripts/search_cli.py +24 -11
  112. package/scripts/search_entry.py +8 -1
  113. package/scripts/search_output.py +99 -15
  114. package/scripts/search_pipeline.py +62 -195
  115. package/scripts/search_rank.py +6 -0
  116. package/scripts/search_types.py +107 -9
  117. package/scripts/semantic_evidence.py +179 -3
  118. package/scripts/serp_guard.py +138 -7
  119. package/scripts/serp_kill_log.py +25 -2
  120. package/scripts/stats_cli.py +13 -1
  121. package/scripts/tail_completion.py +212 -0
  122. package/scripts/url_safety.py +126 -3
  123. package/scripts/usage_log.py +45 -0
  124. package/scripts/wide_research.py +45 -0
  125. package/scripts/zhihu_quota.py +139 -0
  126. package/sub-skills/local-search/bm25.py +27 -3
  127. package/sub-skills/local-search/cli_runner.py +331 -0
  128. package/sub-skills/local-search/parsers.py +328 -0
  129. package/sub-skills/local-search/search_v3.py +31 -674
  130. package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
  131. package/assets/readme/settings-keys.png +0 -0
  132. package/assets/readme/settings-tools.png +0 -0
  133. package/sub-skills/ego-search/references/browser-runtime.md +0 -43
  134. package/sub-skills/ego-search/references/install.md +0 -67
  135. package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
  136. package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
  137. package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
  138. package/sub-skills/js-run/tests/test_jsrun.py +0 -165
  139. package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
  140. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
  141. package/sub-skills/local-seek/references/git-integration.md +0 -54
  142. package/sub-skills/local-seek/references/remote-handoff.md +0 -46
  143. package/sub-skills/local-seek/references/rg-recipes.md +0 -96
  144. package/sub-skills/local-seek/references/strategies.md +0 -52
  145. package/sub-skills/local-seek/references/structural-search.md +0 -59
  146. package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
  147. package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
  148. package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
  149. package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
  150. package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
  151. package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
  152. package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
  153. package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
  154. package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
@@ -0,0 +1,212 @@
1
+ """tail_completion.py — 早停后台补全:慢源交付从「丢弃」变「补齐」。
2
+
3
+ ## 为什么(2026-10-25)
4
+
5
+ 七天窗口 1185 条真实查询里 74% 发生 early-stop。被放弃的交付不是低质货:
6
+ 离线回放(bench_tail_ab --quality)显示被放弃交付 81% 来自 anysearch,相关度
7
+ 代理比全库均线高 0.125;五条对照查询里四条「唯一有结果的就是慢源」(证据与
8
+ 口径见 docs/QUALITY_OF_SLOW_SOURCES.md)。早停省的是**用户等待**,不该连
9
+ **交付**一起丢——那是在拿准确性换速度。缩尾宽已被实证否决,本模块是拍板的
10
+ 正确方向:慢源不拖住快源(早交付),但慢源的交付进缓存(后台补全)。
11
+
12
+ ## 做什么
13
+
14
+ engine_dispatch 在早停弃置引擎时把 (engine, holder, now_fn) 交给本模块——
15
+ 主响应与既有记账逐位不变(cancelled outcome / abandoned_ms 照记)。本模块
16
+ 起一个 daemon 收割线程,在主流程返回后继续等被弃置的引擎,宽限内完成的:
17
+
18
+ 1. 学习器记真实成绩(success / latency / quality / 场景分桶标签)——此前
19
+ cancelled 被 cache_guard.should_record_adaptive 跳过,慢源的真实表现
20
+ 从没进过学习器,场景分桶(engine×domain×lang)也少一笔真数据;
21
+ 2. 把交付合并写回 L2 combo 缓存(URL 去重、新条目追加、缺省 score 用词面
22
+ 相关度)——同查询第二次免费拿到「快源+慢源」的全量覆盖。
23
+
24
+ timeout 弃置不收:那是引擎确实超窗(甚至卡死),补全等于鼓励慢源;
25
+ cancelled 是「编排收工决定,引擎没失败」——两者语义不同(见
26
+ engine_dispatch._settle_pending 的注释)。
27
+
28
+ ## 进程语义
29
+
30
+ 收割线程是 daemon;CLI 进程退出前由 atexit 宽限等待(hand_off 首次调用时
31
+ 注册,无 stranded 的进程零成本、不注册)。MCP/daemon 常驻进程不退出,
32
+ 自然收割全部。宽限从**入队时刻**起算整批(job 级 deadline),不做逐引擎
33
+ 累加——实测被放弃交付从早停时刻起的剩余等待 p50 2306ms / p90 3883ms。
34
+
35
+ ## 开关
36
+
37
+ ARGO_TAIL_COMPLETE_S:宽限秒数(默认 3.0;0=禁用整个机制,行为回到
38
+ 纯丢弃)。改动记得同步 references/usage.md 的开关总表(usage_doc_gates)。
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import atexit
44
+ import os
45
+ import threading
46
+
47
+ _LOCK = threading.Lock()
48
+ _JOBS: list[dict] = []
49
+ _WORKER: threading.Thread | None = None
50
+ _ATEXIT_REGISTERED = False
51
+ _STATS = {"salvaged": 0, "expired": 0, "recorded": 0}
52
+ _POLL_INTERVAL_S = 0.05
53
+
54
+
55
+ def grace_seconds() -> float:
56
+ """收割宽限(ARGO_TAIL_COMPLETE_S,默认 3.0;<=0 禁用)。"""
57
+ try:
58
+ return float(os.environ.get("ARGO_TAIL_COMPLETE_S", "3.0") or 3.0)
59
+ except ValueError:
60
+ return 3.0
61
+
62
+
63
+ def hand_off(stranded, *, query, cache_engine_key, max_results,
64
+ domain, mode, depth, decision, skip_cache=False,
65
+ cache_instance=None) -> None:
66
+ """把早停弃置的引擎交给后台收割。
67
+
68
+ ``stranded`` 是 engine_dispatch 产出的 ``[(engine, holder, now_fn), ...]``:
69
+ holder["r"] 出现即完成(与 _ingest_holder 同判据),now_fn 用于算真实
70
+ latency(dispatch 可能跑在单调钟上,收割线程不能用墙钟去减)。
71
+ ``cache_instance`` 是本次请求的缓存实例(req.cache):补全读写的必须是
72
+ 同一实例(L1 内存态 + L2 落盘一致),模块级没有全局缓存可拿。
73
+ 其余参数与 finalize 的 cache.set / learner.record 同源同口径。
74
+ 任何异常 fail-open:补全失败只丢「锦上添花」,不影响主响应。
75
+ """
76
+ grace = grace_seconds()
77
+ if grace <= 0 or not stranded:
78
+ return
79
+ global _ATEXIT_REGISTERED
80
+ with _LOCK:
81
+ if not _ATEXIT_REGISTERED:
82
+ atexit.register(wait_for_completion)
83
+ _ATEXIT_REGISTERED = True
84
+ _JOBS.append({
85
+ "stranded": list(stranded),
86
+ "deadline": _now_wall() + grace,
87
+ "query": query, "cache_engine_key": cache_engine_key,
88
+ "max_results": max_results, "domain": domain,
89
+ "mode": mode, "depth": depth, "decision": decision or {},
90
+ "skip_cache": bool(skip_cache) or cache_instance is None,
91
+ "cache": cache_instance,
92
+ })
93
+ _start_worker_locked()
94
+
95
+
96
+ def wait_for_completion(timeout_s: float | None = None) -> None:
97
+ """等收割线程收尾(atexit 出口 / 测试用)。宽限耗尽即返回,不无限等。"""
98
+ t = _WORKER
99
+ if t is not None and t.is_alive():
100
+ budget = timeout_s if timeout_s is not None else grace_seconds() + 0.5
101
+ t.join(timeout=budget)
102
+
103
+
104
+ def stats() -> dict:
105
+ """收割计数(可观测口,供诊断;不改任何行为)。"""
106
+ with _LOCK:
107
+ return dict(_STATS)
108
+
109
+
110
+ def _now_wall() -> float:
111
+ import time
112
+ return time.time()
113
+
114
+
115
+ def _start_worker_locked() -> None:
116
+ global _WORKER
117
+ if _WORKER is not None and _WORKER.is_alive():
118
+ return
119
+ _WORKER = threading.Thread(target=_worker, daemon=True,
120
+ name="argo-tail-completion")
121
+ _WORKER.start()
122
+
123
+
124
+ def _worker() -> None:
125
+ """串行收割队列:宽限内完成的记学习器+合并缓存,超宽限标 expired。"""
126
+ import time
127
+ while True:
128
+ with _LOCK:
129
+ if not _JOBS:
130
+ return
131
+ job = _JOBS.pop(0)
132
+ for eng, holder, now_fn in job["stranded"]:
133
+ remaining = job["deadline"] - _now_wall()
134
+ while holder.get("r") is None and remaining > 0:
135
+ time.sleep(min(_POLL_INTERVAL_S, remaining))
136
+ remaining = job["deadline"] - _now_wall()
137
+ res = (holder.get("r") or (None,))[1] if holder.get("r") else None
138
+ if res is None:
139
+ with _LOCK:
140
+ _STATS["expired"] += 1
141
+ continue
142
+ goods = [r for r in (res or [])
143
+ if isinstance(r, dict) and "error" not in r]
144
+ with _LOCK:
145
+ _STATS["salvaged"] += 1
146
+ try:
147
+ _record_adaptive(eng, goods, holder, now_fn, job)
148
+ except Exception:
149
+ pass
150
+ if not job["skip_cache"]:
151
+ try:
152
+ _merge_into_cache(eng, goods, job)
153
+ except Exception:
154
+ pass
155
+
156
+
157
+ def _record_adaptive(eng, goods, holder, now_fn, job) -> None:
158
+ """学习器记真实成绩(口径与 search_pipeline.finalize 对齐)。"""
159
+ from adaptive import get_learner, scenario_of
160
+ from query_signals import result_relevance
161
+ success = bool(goods)
162
+ try:
163
+ lat_ms = float((now_fn() - holder.get("t0", now_fn())) * 1000.0)
164
+ except Exception:
165
+ lat_ms = 0.0
166
+ quality = None
167
+ if success:
168
+ try:
169
+ quality = result_relevance(job["query"], goods)
170
+ except Exception:
171
+ quality = None
172
+ with _LOCK:
173
+ _STATS["recorded"] += 1
174
+ get_learner().record(eng, success=success, latency_ms=max(0.0, lat_ms),
175
+ cost=0.001, empty=not success, quality=quality,
176
+ **scenario_of(job["decision"]))
177
+
178
+
179
+ def _merge_into_cache(eng, goods, job) -> None:
180
+ """交付合并写回 L2 combo 缓存:读现条目 → URL 去重追加 → 同 key 覆盖。
181
+
182
+ 只在已有缓存条目时合并(本机制补的是「快源已写的那份」);主缓存被守卫
183
+ 拒写时这里不硬造。score 缺省用词面相关度,保证追加条目有可排序的分数。
184
+ """
185
+ from query_signals import result_relevance
186
+ cache = job["cache"]
187
+ cur = cache.get(job["query"], job["cache_engine_key"],
188
+ job["max_results"], domain=job["domain"],
189
+ mode=job["mode"], depth=job["depth"])
190
+ if not cur or not isinstance(cur.get("results"), list) or not goods:
191
+ return
192
+ existing = {r.get("url") for r in cur["results"]
193
+ if isinstance(r, dict)}
194
+ new: list[dict] = []
195
+ for r in goods:
196
+ if not isinstance(r, dict) or r.get("url") in existing:
197
+ continue
198
+ item = dict(r)
199
+ if "score" not in item:
200
+ try:
201
+ item["score"] = round(float(result_relevance(job["query"], [r])), 4)
202
+ except Exception:
203
+ item["score"] = 0.3
204
+ new.append(item)
205
+ if not new:
206
+ return
207
+ payload = dict(cur)
208
+ payload["results"] = list(cur["results"]) + new
209
+ payload["engines_used"] = sorted({*(cur.get("engines_used") or []), eng})
210
+ cache.set(job["query"], job["cache_engine_key"], job["max_results"],
211
+ payload, domain=job["domain"], mode=job["mode"],
212
+ depth=job["depth"])
@@ -23,6 +23,8 @@ import ipaddress
23
23
  import os
24
24
  import re
25
25
  import socket
26
+ import time
27
+ import urllib.parse
26
28
  from urllib.parse import urlparse
27
29
 
28
30
 
@@ -184,9 +186,10 @@ def host_is_private(host: str) -> bool:
184
186
  if host_is_private_name(host):
185
187
  return True
186
188
  norm = _normalize_numeric_host(host)
187
- if norm is not None and norm != host:
188
- # 数字字面量变体:直接按规范化结果查表,不依赖各系统对
189
- # 八进制/十六进制/单段整数解析行为的一致性
189
+ if norm is not None:
190
+ # 数字字面量:直接按规范化结果查表,不依赖 getaddrinfo。
191
+ # 标准点分形式(norm == host)此前会落到 getaddrinfo——既多一次系统
192
+ # 调用,又让「被 mock/劫持的解析器」能影响 SSRF 判定。
190
193
  return is_private_ip(norm)
191
194
  try:
192
195
  infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
@@ -234,3 +237,123 @@ def is_safe_fetch_url(url: str) -> bool:
234
237
  """便捷布尔封装:仅 http/https 且目标非私有地址。"""
235
238
  ok, _ = check_url(url)
236
239
  return ok
240
+
241
+
242
+ def max_redirects() -> int:
243
+ """最大重定向跳数(默认 5,合法 1-20,越界回落默认)。"""
244
+ try:
245
+ from engine_env import get_env
246
+ val = int(str(get_env(["ARGO_MAX_REDIRECTS"])).strip())
247
+ except (TypeError, ValueError):
248
+ return 5
249
+ return val if 0 < val <= 20 else 5
250
+
251
+
252
+ def check_redirect_target(url: str, start: float) -> dict | None:
253
+ """校验重定向目标是否可安全请求;被拦截返回统一错误 dict,放行返回 None。
254
+
255
+ 与 resolve_redirect_chain 的逐跳校验对齐——此前 _do_get/_do_post 的
256
+ 内联重定向循环只校验首跳,重定向到内网即可绕过 SSRF。
257
+ """
258
+ ok, reason = check_url(url)
259
+ if ok:
260
+ return None
261
+ return {
262
+ "status": 0, "headers": {}, "text": "",
263
+ "url": url,
264
+ "elapsed_ms": int((time.time() - start) * 1000),
265
+ "error": f"重定向目标被 SSRF 防护拦截: {reason}",
266
+ }
267
+
268
+
269
+ def resolve_redirect_chain(url: str, headers: dict, timeout: float,
270
+ limit: int | None = None) -> str | None:
271
+ """逐跳安全跟随重定向:每跳目标都过 SSRF 校验(curl 通道用)。
272
+
273
+ 返回最终 URL;任一跳被拦截或超过跳数返回 None。
274
+ """
275
+ if limit is None:
276
+ limit = max_redirects()
277
+ current = url
278
+ for _ in range(limit):
279
+ ok, _reason = check_url(current)
280
+ if not ok:
281
+ return None
282
+ parsed = urlparse(current)
283
+ if parsed.scheme not in ("http", "https"):
284
+ return None
285
+ try:
286
+ from net_proxy import open_connection, request_selector, resolve_proxy
287
+ proxy_url = resolve_proxy(current)
288
+ conn, via_proxy = open_connection(parsed, timeout, proxy_url)
289
+ path = parsed.path or "/"
290
+ if parsed.query:
291
+ path += "?" + parsed.query
292
+ conn.request("HEAD", request_selector(parsed, path, via_proxy),
293
+ headers=headers)
294
+ resp = conn.getresponse()
295
+ status = resp.status
296
+ location = resp.getheader("Location")
297
+ conn.close()
298
+ except Exception:
299
+ return None
300
+ if status not in (301, 302, 303, 307, 308) or not location:
301
+ return current
302
+ current = urllib.parse.urljoin(current, location)
303
+ return None
304
+
305
+
306
+ # ── 解析即锁定(DNS rebinding / TOCTOU 防护)─────────────────────────────────
307
+ #
308
+ # check_url() 在请求前解析一次 DNS,http.client 在 connect() 时再解析一次——
309
+ # 两次之间域名可改指内网(DNS rebinding)。下面两个连接类在 connect() 里
310
+ # 解析一次、过滤私网、把结果钉给本次 TCP 连接,窗口关闭;SNI/Host 仍用原域名。
311
+
312
+
313
+ def _is_ip_literal(host: str) -> bool:
314
+ """主机名是否为 IP 字面量(v4 / v6,含 [v6] 与 IPv4-mapped)。"""
315
+ h = (host or "").strip().strip("[]")
316
+ if not h:
317
+ return False
318
+ if ":" in h: # IPv6(含 ::ffff:a.b.c.d)
319
+ return True
320
+ parts = h.split(".")
321
+ return len(parts) == 4 and all(p.isdigit() for p in parts)
322
+
323
+
324
+ def resolve_public_addresses(host: str) -> list[str]:
325
+ """解析主机名 → 公网地址名单(过滤私网/保留段;fake-ip 占位段放行)。
326
+
327
+ 返回空列表表示「无可连的公网地址」——调用方应拒绝连接(fail-closed)。
328
+ """
329
+ try:
330
+ infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
331
+ except OSError:
332
+ return []
333
+ out: list[str] = []
334
+ seen: set[str] = set()
335
+ for info in infos:
336
+ ip = info[4][0]
337
+ if ip in seen:
338
+ continue
339
+ seen.add(ip)
340
+ if is_fake_ip_address(ip): # 代理接管域名(占位段),不可直达内网
341
+ out.append(ip)
342
+ continue
343
+ if is_private_ip(ip):
344
+ continue
345
+ out.append(ip)
346
+ return out
347
+
348
+
349
+ def pin_host(host: str) -> str | None:
350
+ """返回要钉扎的公网 IP;IP 字面量或显式放行私网时返回 None。
351
+
352
+ 无可连的公网地址则抛错(fail-closed)。供 safe_conn 的连接类使用。
353
+ """
354
+ if allow_private() or _is_ip_literal(host):
355
+ return None
356
+ addrs = resolve_public_addresses(host)
357
+ if not addrs:
358
+ raise OSError(f"SSRF 防护:{host} 无可连的公网地址")
359
+ return addrs[0]
@@ -6,6 +6,7 @@
6
6
  - 失败静默:任何异常都吞掉返回 False,绝不拖累搜索主路径
7
7
  - 目录可注入:ARGO_USAGE_LOG_DIR 覆盖(测试隔离),默认 <状态目录>/usage_log/
8
8
  - 总开关:ARGO_USAGE_LOG=0 / false 完全关闭
9
+ - 夹具守卫:query 流 engines_used 出现注册表外引擎名即判测试夹具,整条拒写
9
10
  - 脱敏:query 等敏感字段由调用方截断,本模块不放大
10
11
 
11
12
  Schema(每条一行 JSON,UTF-8):
@@ -68,6 +69,40 @@ def _enabled() -> bool:
68
69
  return env_flag("ARGO_USAGE_LOG")
69
70
 
70
71
 
72
+ # 引擎改名前的历史别名(表外但真实):与 usage_log_clean._ALIAS_KEEP 同一账。
73
+ _RENAMED_ALIAS_KEEP = frozenset({"openlibrary", "bing"})
74
+
75
+
76
+ def _canonical_engines() -> frozenset[str]:
77
+ """注册表引擎名全集(含历史别名豁免)。注册表加载失败返回空集——
78
+ 守卫随之失效放行(fail-open):守卫是防脏数据的,不能反过来丢真日志。
79
+
80
+ 不做外层缓存:available_engines 走 get_registry 的 mtime 指纹自愈,每次
81
+ 现取才能看见「daemon 常驻期间新增的本地引擎」;外层冻结只会把新引擎的
82
+ 真日志误杀成夹具。每次构建 frozenset 是微秒级,query 流每搜索只写一条。
83
+ """
84
+ try:
85
+ from engines import available_engines
86
+ return frozenset(available_engines()) | _RENAMED_ALIAS_KEEP
87
+ except Exception:
88
+ return frozenset()
89
+
90
+
91
+ def _fixture_engines(record: dict[str, Any]) -> list[str]:
92
+ """query 记录里的注册表外引擎名(别名豁免后)。
93
+
94
+ engines_used 缺失/非列表/为空一律视为无夹具信号:真实搜索的该字段
95
+ 永远是引擎 id 列表,守卫只对「有列表且含表外名」的形态做判定。
96
+ """
97
+ used = record.get("engines_used")
98
+ if not isinstance(used, list) or not used:
99
+ return []
100
+ canon = _canonical_engines()
101
+ if not canon:
102
+ return []
103
+ return [e for e in used if not isinstance(e, str) or e not in canon]
104
+
105
+
71
106
  def emit(stream: str, record: dict[str, Any]) -> bool:
72
107
  """追加一条日志记录到 <usage_log_dir>/<stream>.jsonl。
73
108
 
@@ -75,6 +110,16 @@ def emit(stream: str, record: dict[str, Any]) -> bool:
75
110
  """
76
111
  if not _enabled():
77
112
  return False
113
+ if stream == "query" and _fixture_engines(record):
114
+ # 夹具守卫(2026-10-25):测试夹具引擎名(fast / slow / quick_primary
115
+ # 这类注册表外名)屡次写进生产 query.jsonl(conftest 的
116
+ # ARGO_USAGE_LOG=0 与 is_synthetic_run 的黑名单各漏过一批,最近一次
117
+ # 2026-10-24),stats 的均值与最慢/高频引擎榜随之失真。黑名单永远
118
+ # 追不全新夹具名,这里在唯一落盘口用注册表白名单反向判定:表外名
119
+ # 即夹具,整条拒写。存量清理走 usage_log_clean.py(隔离不删除)。
120
+ # 只拦 query 流:route/recovery 的字段形态与测试惯例不同(route 的
121
+ # 格式测试本身用 e0..e14 假名),且实测污染只发生在 query 流。
122
+ return False
78
123
  try:
79
124
  line = json.dumps(
80
125
  {
@@ -113,6 +113,40 @@ def normalize_tracks(raw: Any, maximum: int = 9) -> list[dict[str, Any]]:
113
113
  return out
114
114
 
115
115
 
116
+ def build_cross_lingual_tracks(question: str, maximum: int = 4) -> list[dict[str, Any]]:
117
+ """为溯源与多语言任务自动构建分阶段跨语言长程轨道。"""
118
+ q = (question or "").strip()
119
+ has_tracing = bool(re.search(
120
+ r"(?:溯源|起源|出处|最初|源头|提出者|发明|草案|历史沿革|origin|provenance|root cause|first proposed|specification|rfc|cve)",
121
+ q, re.I
122
+ ))
123
+ tracks = []
124
+ tracks.append({
125
+ "id": "track-primary-lang",
126
+ "title": "原语言背景与核心定义",
127
+ "question": f"{q} 背景 定义 现状",
128
+ "rationale": "Primary language overview and conceptual framing.",
129
+ "depends_on": [],
130
+ })
131
+ if has_tracing or len(tracks) < maximum:
132
+ tracks.append({
133
+ "id": "track-origin-trace",
134
+ "title": "源头出处与权威一手规范长程溯源",
135
+ "question": f"{q} original paper inception specification first proposed RFC official repository",
136
+ "rationale": "Cross-lingual long-range tracing for primary sources, papers, or official specifications.",
137
+ "depends_on": ["track-primary-lang"],
138
+ })
139
+ tracks.append({
140
+ "id": "track-cross-reconciliation",
141
+ "title": "多语言证据交叉对账与互补验证",
142
+ "question": f"{q} milestone benchmark evaluation comparison timeline",
143
+ "rationale": "Cross-lingual reconciliation of evidence, timeline differences, and complementary facts.",
144
+ "depends_on": ["track-origin-trace"],
145
+ })
146
+ return tracks[:maximum]
147
+
148
+
149
+
116
150
  def stage_tracks(tracks: list[dict[str, Any]]) -> tuple[list[list[dict[str, Any]]], list[str]]:
117
151
  """轨道分层。work package 的分层逻辑与轨道同构:id/question/depends_on。"""
118
152
  packages = [
@@ -255,6 +289,17 @@ def evaluate_gates(
255
289
  if len(caveats) + len(unanswered) >= max(3, (len(sources) + 1) // 2):
256
290
  warnings.append({"id": "high_uncertainty",
257
291
  "detail": f"{len(caveats)} caveats and {len(unanswered)} unanswered questions exceed the evidence threshold."})
292
+
293
+ # 内容农场与镜像站占优检查
294
+ farm_markers = ("ctolib.com", "coder.work", "voidcn.com", "it1352.com", "oommake.com", "py4u.net",
295
+ "wenku.zone", "chinadmd.com", "jb51.net", "pc6.com", "onlinedown.net")
296
+ farm_hits = [s for s in sources if any(m in str(s.get("url", "")).lower() for m in farm_markers)]
297
+ if farm_hits and len(farm_hits) >= (len(sources) + 1) // 2:
298
+ warnings.append({
299
+ "id": "content_farm_dominant",
300
+ "detail": f"{len(farm_hits)} sources originate from content farms or SEO scraper mirrors; evidence quality severely degraded.",
301
+ })
302
+
258
303
  cap = "low" if failures else ("medium" if warnings else "high")
259
304
  return {"passed": not failures, "conclusion_cap": cap, "failures": failures, "warnings": warnings}
260
305
 
@@ -0,0 +1,139 @@
1
+ #!/usr/bin/env python3
2
+ """zhihu_quota.py — 知乎开放平台额度查询(官方 `/api/v1/quota`)。
3
+
4
+ 为什么需要(2026-10-21):知乎各能力的每日免费额度是**账号级**的,同一账号的
5
+ 多个 Access Secret 共用;本地按引擎计数会与真实额度漂移。官方额度查询不消耗
6
+ 业务额度,是唯一权威来源。本模块只做读取与缓存,不参与路由决策(路由仍用
7
+ quota.py 的本地计数),避免每次搜索多一次外呼。
8
+
9
+ 用法:
10
+ argo quota # 人读表
11
+ argo quota --json # 机器读
12
+ argo quota --refresh # 跳过 10 分钟缓存
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import time
20
+ import urllib.request
21
+
22
+ from cli_io import dumps
23
+ from engine_env import get_env
24
+ from engines_base import http_open
25
+
26
+ API_URL = "https://developer.zhihu.com/api/v1/quota"
27
+ CACHE_TTL_S = 600
28
+
29
+ # APIID → argo 引擎名(与 config.yaml / quota_profiles.json 对齐)
30
+ API_TO_ENGINE = {
31
+ "global_search": "zhihu_global",
32
+ "zhihu_search": "zhihu",
33
+ "hot_list": "zhihu_hot",
34
+ "question_discovery": "zhihu_questions",
35
+ "zhida_openai": "zhida",
36
+ "tools": "zhihu_tools",
37
+ "knowledge": "zhihu_kb",
38
+ "user_data": "zhihu_user",
39
+ "creator": "zhihu_creator",
40
+ }
41
+
42
+
43
+ def _cache_path():
44
+ import argo_paths
45
+ return argo_paths.state_path("zhihu_quota.json")
46
+
47
+
48
+ def _secret() -> str:
49
+ return get_env(["ARGO_ZHIHU_ACCESS_SECRET", "ZHIHU_ACCESS_SECRET"])
50
+
51
+
52
+ def _load_cache() -> dict | None:
53
+ try:
54
+ raw = json.loads(_cache_path().read_text(encoding="utf-8"))
55
+ except (OSError, ValueError):
56
+ return None
57
+ if not isinstance(raw, dict):
58
+ return None
59
+ if time.time() - float(raw.get("ts") or 0) > CACHE_TTL_S:
60
+ return None
61
+ return raw
62
+
63
+
64
+ def _save_cache(items: list[dict]) -> None:
65
+ try:
66
+ import argo_paths
67
+ argo_paths.atomic_write_json(_cache_path(),
68
+ {"ts": time.time(), "items": items})
69
+ except Exception:
70
+ pass
71
+
72
+
73
+ def fetch_quota(*, refresh: bool = False, timeout: float = 8.0) -> dict:
74
+ """返回 {ok, items, error, cached};永不抛异常(观测层不阻断主路径)。"""
75
+ if not refresh:
76
+ cached = _load_cache()
77
+ if cached is not None:
78
+ return {"ok": True, "items": cached.get("items") or [], "cached": True}
79
+ secret = _secret()
80
+ if not secret:
81
+ return {"ok": False, "items": [], "cached": False,
82
+ "error": "缺少 ZHIHU_ACCESS_SECRET"}
83
+ req = urllib.request.Request(API_URL, headers={
84
+ "Authorization": f"Bearer {secret}",
85
+ "X-Request-Timestamp": str(int(time.time())),
86
+ "Content-Type": "application/json",
87
+ "User-Agent": "argo-search/2.6 (unified-search@local)",
88
+ })
89
+ try:
90
+ with http_open(req, timeout=timeout, engine="zhihu_quota") as resp:
91
+ data = json.loads(resp.read().decode("utf-8", "replace"))
92
+ except Exception as exc:
93
+ return {"ok": False, "items": [], "cached": False,
94
+ "error": f"{type(exc).__name__}: {exc}"}
95
+ if data.get("Code") not in (0, None):
96
+ return {"ok": False, "items": [], "cached": False,
97
+ "error": f"Code={data.get('Code')} {data.get('Message') or ''}"}
98
+ items = []
99
+ for row in data.get("Data") or []:
100
+ if not isinstance(row, dict):
101
+ continue
102
+ api_id = str(row.get("APIID") or "")
103
+ items.append({
104
+ "api_id": api_id,
105
+ "engine": API_TO_ENGINE.get(api_id, api_id),
106
+ "name": row.get("APIName") or api_id,
107
+ "total": row.get("TotalQuota"),
108
+ "used": row.get("TotalUsed"),
109
+ "remaining": row.get("RemainingQuota"),
110
+ })
111
+ _save_cache(items)
112
+ return {"ok": True, "items": items, "cached": False}
113
+
114
+
115
+ def main(argv: list[str] | None = None) -> int:
116
+ ap = argparse.ArgumentParser(prog="argo quota",
117
+ description="知乎开放平台当日额度(官方 API)")
118
+ ap.add_argument("--json", action="store_true")
119
+ ap.add_argument("--refresh", action="store_true")
120
+ args = ap.parse_args(argv)
121
+ out = fetch_quota(refresh=args.refresh)
122
+ if args.json:
123
+ print(dumps(out))
124
+ return 0 if out.get("ok") else 1
125
+ if not out.get("ok"):
126
+ print(f"额度查询失败:{out.get('error')}")
127
+ return 1
128
+ print("知乎开放平台当日额度(北京时间 00:00 重置)")
129
+ print(f"{'引擎':<18}{'额度项':<20}{'总额':>8}{'已用':>8}{'剩余':>8}")
130
+ for row in out["items"]:
131
+ print(f"{row['engine']:<18}{row['name']:<20}"
132
+ f"{str(row['total']):>8}{str(row['used']):>8}{str(row['remaining']):>8}")
133
+ if out.get("cached"):
134
+ print("(10 分钟内缓存;--refresh 强制刷新)")
135
+ return 0
136
+
137
+
138
+ if __name__ == "__main__":
139
+ raise SystemExit(main())
@@ -43,6 +43,7 @@ _CJK_RE = re.compile(r"[\u4e00-\u9fff]+")
43
43
 
44
44
  _K1 = 1.5
45
45
  _B = 0.75
46
+ _DELTA = 0.5 # BM25+ 下界 (Lv & Zhai 2011):长文档命中词保底,消除过惩罚
46
47
  # 混合权重:BM25 侧。位次先验占 1 - _W_BM25。
47
48
  _W_BM25 = 0.6
48
49
  # 旧位次衰减口径(max(0.7 - idx*0.05, 0.1))——先验的归一化基准。
@@ -79,9 +80,10 @@ class BM25:
79
80
  引擎的 5 条结果上稳定得多——「含查询词的文档有多罕见」是跨源问题。
80
81
  """
81
82
 
82
- def __init__(self, docs: list[list[str]], k1: float = _K1, b: float = _B):
83
+ def __init__(self, docs: list[list[str]], k1: float = _K1, b: float = _B, delta: float = _DELTA):
83
84
  self.k1 = k1
84
85
  self.b = b
86
+ self.delta = delta
85
87
  self.n_docs = len(docs)
86
88
  self.doc_len = [len(d) for d in docs]
87
89
  self.avgdl = (sum(self.doc_len) / self.n_docs) if self.n_docs else 0.0
@@ -113,10 +115,30 @@ class BM25:
113
115
  f = self.tf.get(t, {}).get(doc_idx, 0)
114
116
  if not f:
115
117
  continue
116
- total += self.idf(t) * (f * (self.k1 + 1.0)) / (f + denom_norm)
118
+ sat = (f * (self.k1 + 1.0)) / (f + denom_norm)
119
+ # BM25+ 饱和项下界增益(Lv & Zhai 2011)
120
+ total += self.idf(t) * (sat + self.delta)
117
121
  return total
118
122
 
119
123
 
124
+ def _phrase_proximity_bonus(query: str, title: str, snippet: str) -> float:
125
+ """词邻近度与短语连续匹配加成(归一化为 0.0~1.0)。
126
+
127
+ 当完整查询短语在标题或摘要逐字紧密出现时给予增益,提升复合词精准命中率。
128
+ """
129
+ q_low = (query or "").lower().strip()
130
+ if not q_low or len(q_low) < 2:
131
+ return 0.0
132
+ t_low = (title or "").lower()
133
+ s_low = (snippet or "").lower()
134
+ if q_low in t_low:
135
+ return 1.0 # 标题完整覆盖查询短语
136
+ if q_low in s_low:
137
+ return 0.5 # 摘要完整覆盖查询短语
138
+ return 0.0
139
+
140
+
141
+
120
142
  def _doc_text(r: dict[str, Any]) -> str:
121
143
  """参与相关度计算的文档文本:title 权重高于 snippet——做法是标题
122
144
  拼两遍(TF 加成),不引入第二套字段权重机制。"""
@@ -153,7 +175,9 @@ def rerank(by_engine: dict[str, list[dict[str, Any]]], query: str,
153
175
  for (eng, idx, r), bm in zip(flat, raw):
154
176
  bm25_norm = (bm / max_raw) if max_raw > 0 else 0.0
155
177
  prior_norm = _position_prior(idx) / _PRIOR_BASE
156
- r["score"] = round(w_bm25 * bm25_norm + (1.0 - w_bm25) * prior_norm, 3)
178
+ prox_val = _phrase_proximity_bonus(query, r.get("title", ""), r.get("snippet", ""))
179
+ rel_score = min(bm25_norm * 0.80 + prox_val * 0.20, 1.0) if bm25_norm > 0 else 0.0
180
+ r["score"] = round(w_bm25 * rel_score + (1.0 - w_bm25) * prior_norm, 3)
157
181
 
158
182
  for eng in by_engine:
159
183
  by_engine[eng].sort(