argo-search 2.9.8 → 2.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.en.md +30 -20
  2. package/README.es.md +30 -20
  3. package/README.ja.md +30 -20
  4. package/README.ko.md +30 -20
  5. package/README.md +32 -22
  6. package/SKILL.md +1 -1
  7. package/bin/argo +4 -0
  8. package/bin/argo.js +1 -1
  9. package/config.yaml +52 -5
  10. package/docs/ARGO_INTRO.md +5 -3
  11. package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
  12. package/package.json +3 -2
  13. package/packages/dsh-plugin/README.md +3 -2
  14. package/packages/dsh-plugin/dsh/client.js +98 -32
  15. package/packages/dsh-plugin/dsh/diagnostics.js +43 -0
  16. package/packages/dsh-plugin/dsh/index.js +23 -1
  17. package/packages/dsh-plugin/package.json +2 -2
  18. package/packages/dsh-plugin/test/client.test.js +2 -1
  19. package/scripts/adaptive.py +102 -42
  20. package/scripts/bench_tail_ab.py +281 -0
  21. package/scripts/bounded_run.py +91 -49
  22. package/scripts/clock_bridge.py +22 -0
  23. package/scripts/content_security.py +119 -4
  24. package/scripts/dispatch_result.py +9 -2
  25. package/scripts/engine_add.py +602 -0
  26. package/scripts/engine_dispatch.py +9 -0
  27. package/scripts/evidence_loop.py +4 -0
  28. package/scripts/fetch_quality.py +13 -0
  29. package/scripts/fetch_v3.py +3 -3
  30. package/scripts/http_client.py +24 -38
  31. package/scripts/image_vision/vision_probe +0 -0
  32. package/scripts/local_engines.py +321 -0
  33. package/scripts/matrix_search_eval.py +111 -1
  34. package/scripts/mcp_transport.py +1 -1
  35. package/scripts/net_proxy.py +27 -0
  36. package/scripts/recovery.py +7 -3
  37. package/scripts/route_combo.py +19 -5
  38. package/scripts/route_domains.py +138 -12
  39. package/scripts/safe_conn.py +52 -0
  40. package/scripts/search_entry.py +1 -0
  41. package/scripts/search_pipeline.py +20 -2
  42. package/scripts/search_rank.py +6 -0
  43. package/scripts/serp_guard.py +23 -0
  44. package/scripts/tail_completion.py +212 -0
  45. package/scripts/url_safety.py +126 -3
  46. package/scripts/usage_log.py +45 -0
@@ -0,0 +1,281 @@
1
+ #!/usr/bin/env python3
2
+ """bench_tail_ab.py — 尾延迟策略的离线 A/B 回放台(方案 A 的验收工具)。
3
+
4
+ ## 为什么需要它(而不是再写一个调参脚本)
5
+
6
+ 尾宽这条线上的教训是:**参数改动必须在同一份日志上先跑出数字**。2026-10-15
7
+ 与 2026-10-19 两次 A/B 结论相反,原因是每次都换了一批线上流量、换了一个
8
+ 估计量,最后只能判「噪声非信号」。本脚本把「候选策略」和「团队现行策略」
9
+ 放在同一条回放时间线上算,输入是本地 ``usage_log/query.jsonl`` 与
10
+ ``adaptive.db`` 的 7 天延迟分布——零网络、可复现、可对拍。
11
+
12
+ ## 两种运行模式
13
+
14
+ * ``--dry-run``(默认):只打印回放表格。**不做任何结论**,不写状态。
15
+ * ``--record``:把这次的候选值写进 ``docs-local/tail_ab_<日期>.json``,
16
+ 供下一次跑同一个 bench 时直接对拍(「上次结论是什么」不再靠记忆)。
17
+
18
+ ## 用法
19
+
20
+ python3 scripts/bench_tail_ab.py # 回放:现行 vs 候选
21
+ python3 scripts/bench_tail_ab.py --candidate 0.6 # 换一个候选尾宽
22
+
23
+ 候选策略刻意只开一个**常数**轴(尾宽上限),因为要回答的问题只有一个:
24
+ 「把尾宽从 1.5s 收到 X,在 2 引擎查询上省多少墙钟、丢掉多少慢源交付?」
25
+ 其余旋钮(起步窗、分层档位)不动——多变量同时动就没法归因,这正是前两次
26
+ 对拍失败的机制原因。
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import argparse
32
+ import json
33
+ import sqlite3
34
+ import statistics as st
35
+ import sys
36
+ import time
37
+ from collections import defaultdict
38
+ from pathlib import Path
39
+
40
+ try: # 与生产同源的路径解析(测试域可被 ARGO_STATE_DIR 重定向)
41
+ from argo_paths import state_path
42
+ except ImportError: # pragma: no cover - 直接 python3 运行时的兜底
43
+ def state_path(*parts: str) -> Path:
44
+ return Path.home() / ".cache" / "unified-search" / Path(*parts)
45
+
46
+ BASE_TAIL = 1.5 # 现行固定尾宽
47
+ START_WINDOW = 0.8 # 现行对冲起步窗
48
+ _SYNTHETIC = {"dead_primary", "dead_backup", "fast_backup", "slow_backup",
49
+ "slow_primary", "quick_primary", "fast_primary", "parallel"}
50
+
51
+
52
+ def _percentile(values: list[float], q: float) -> float:
53
+ if not values:
54
+ return 0.0
55
+ s = sorted(values)
56
+ return s[min(len(s) - 1, int(len(s) * q))]
57
+
58
+
59
+ def load_p75(db_path: Path) -> dict[str, float]:
60
+ """引擎 → 7 天窗「正常交付」延迟的 P75(毫秒);样本 <5 的引擎不给值。"""
61
+ if not db_path.exists():
62
+ return {}
63
+ lat: dict[str, list[float]] = defaultdict(list)
64
+ with sqlite3.connect(db_path) as conn:
65
+ for eng, ms in conn.execute(
66
+ "SELECT engine, latency_ms FROM engine_perf "
67
+ "WHERE success = 1 AND empty = 0"):
68
+ lat[eng].append(float(ms or 0))
69
+ return {e: st.quantiles(v, n=4)[2] for e, v in lat.items() if len(v) >= 5}
70
+
71
+
72
+ def load_host_bodies(state: Path) -> dict[str, float]:
73
+ """站点 → 本地全文缓存里该站正文的中位字节数(「这个站被真正抓过、有料」的代理)。
74
+
75
+ 数据来源是 ``fulltext/index.jsonl``(抓取正文的追加式索引)。用途只有一个:
76
+ 判断被放弃的那条交付**有没有在被放弃之前进过正文流水线**。没进过的站点,
77
+ 尾宽再宽也不会自动变成正文——这是「省下的墙钟到底是不是白等」的判据。
78
+ """
79
+ idx = state / "fulltext" / "index.jsonl"
80
+ if not idx.exists():
81
+ return {}
82
+ per: dict[str, list[float]] = defaultdict(list)
83
+ with idx.open(encoding="utf-8", errors="replace") as fh:
84
+ for line in fh:
85
+ try:
86
+ rec = json.loads(line)
87
+ except json.JSONDecodeError:
88
+ continue
89
+ url = str(rec.get("url") or "")
90
+ host = url.split("/")[2] if "://" in url else ""
91
+ if host:
92
+ per[host].append(float(rec.get("bytes") or 0))
93
+ return {h: st.median(v) for h, v in per.items() if v}
94
+
95
+
96
+ def load_queries(log_path: Path) -> list[dict]:
97
+ if not log_path.exists():
98
+ return []
99
+ out = []
100
+ with log_path.open(encoding="utf-8", errors="replace") as fh:
101
+ for line in fh:
102
+ line = line.strip()
103
+ if not line:
104
+ continue
105
+ try:
106
+ rec = json.loads(line)
107
+ except json.JSONDecodeError:
108
+ continue
109
+ top = rec.get("engine_latency_top") or []
110
+ if any(isinstance(x, list) and x and x[0] in _SYNTHETIC for x in top):
111
+ continue # 合成引擎(基准/混沌)不是真实流量,别混进来
112
+ out.append(rec)
113
+ return out
114
+
115
+
116
+ def replay(queries: list[dict], p75: dict[str, float], tail_s: float) -> dict:
117
+ """在候选尾宽下重算每条查询的墙钟,返回新旧两组的对比。"""
118
+ old_walls: list[float] = []
119
+ new_walls: list[float] = []
120
+ lost_delivery = 0 # 候选策略下慢源「来不及」的次数
121
+ for rec in queries:
122
+ wall = rec.get("elapsed_ms") or 0
123
+ if wall <= 0:
124
+ continue
125
+ top = sorted((x for x in (rec.get("engine_latency_top") or [])
126
+ if isinstance(x, list) and len(x) == 2 and isinstance(x[1], (int, float))),
127
+ key=lambda x: x[1])
128
+ n_eng = len(rec.get("engines_used") or [])
129
+ if len(top) < 2 or n_eng < 2:
130
+ old_walls.append(wall)
131
+ new_walls.append(wall)
132
+ continue
133
+ fast_ms = top[0][1]
134
+ slow_ms = top[-1][1]
135
+ # 现行:慢源若在自己回来之前就已交付则无等待,否则最多再等 BASE_TAIL
136
+ # 已经交付的等待是**沉没成本**:慢源在窗口内回来的那条路径,两套口径
137
+ # 都不产生「额外等待」,也就都不该被算成可省时间。只有「慢源还没回来、
138
+ # 我们决定不再等」那一段才是真省下来的墙钟。
139
+ wait_old = BASE_TAIL * 1000 if slow_ms - fast_ms >= BASE_TAIL * 1000 else 0
140
+ wait_new = tail_s * 1000 if slow_ms - fast_ms >= BASE_TAIL * 1000 else 0
141
+ if wait_new < wait_old and slow_ms - fast_ms > tail_s * 1000:
142
+ lost_delivery += 1 # 候选策略会放弃这次慢源交付
143
+ old_walls.append(wall)
144
+ new_walls.append(max(0, wall - (wait_old - wait_new)))
145
+ return {
146
+ "n": len(old_walls),
147
+ "old": {"p50": st.median(old_walls) if old_walls else 0,
148
+ "p90": _percentile(old_walls, .90)},
149
+ "new": {"p50": st.median(new_walls) if new_walls else 0,
150
+ "p90": _percentile(new_walls, .90)},
151
+ "saved_ms_total": sum(o - n for o, n in zip(old_walls, new_walls)),
152
+ "slow_delivery_lost": lost_delivery,
153
+ }
154
+
155
+
156
+ def lost_deliveries(queries: list[dict], p75: dict[str, float],
157
+ tail_s: float) -> list[dict]:
158
+ """候选尾宽下**会被丢掉**的慢源交付明细(引擎 / 多等了多少 / 当时省了多少)。
159
+
160
+ 为什么单独成函数:取舍不能只看总数。「224 次」里如果九成是同一台慢源,
161
+ 那它早该被自适应降权(另一条已存在的机制),尾宽只是替它擦屁股;如果
162
+ 分散在很多源身上,缩尾宽就是在系统性地牺牲长尾覆盖。两种结论的动作完全不同。
163
+ """
164
+ rows: list[dict] = []
165
+ for rec in queries:
166
+ top = sorted((x for x in (rec.get("engine_latency_top") or [])
167
+ if isinstance(x, list) and len(x) == 2), key=lambda x: x[1])
168
+ if len(top) < 2 or len(rec.get("engines_used") or []) < 2:
169
+ continue
170
+ fast_ms, (slow_eng, slow_ms) = top[0][1], top[-1]
171
+ if slow_ms - fast_ms >= BASE_TAIL * 1000 and slow_ms - fast_ms >= tail_s * 1000:
172
+ rows.append({"engine": slow_eng,
173
+ "extra_wait_ms": slow_ms - fast_ms,
174
+ "p75_ms": p75.get(slow_eng),
175
+ "query": (rec.get("query") or "")[:40]})
176
+ return rows
177
+
178
+
179
+ def quality_report(lost: list[dict], host_bodies: dict[str, float],
180
+ state: Path, p75: dict[str, float] | None = None) -> str:
181
+ """被放弃的那批交付「值不值得等」——用两条**互不冒充**的证据回答。
182
+
183
+ 1. **相关度代理**(结果层):``adaptive.db`` 里该引擎的质量分。口径是
184
+ ``result_relevance``——词面覆盖率,**不是内容正确性**;它只能说明
185
+ 「结果和查询沾不沾边」,不能说明「结果对不对」。
186
+ 2. **相对基线**(横向可比):只看绝对值会误判——同一套代理分数在中文查询上
187
+ 天然偏低。所以同时给出「该引擎质量 − 全部引擎均质量」,正数=比平均更相关。
188
+ 3. **正文缓存覆盖**(证据层):本地 ``fulltext`` 索引里该站点有没有被抓过正文。
189
+ 没进过的站点,尾宽再宽也不会自动变成正文。
190
+
191
+ 读法:相关度代理明显低于均线 **且** 从未进过正文缓存 → 早收工合理;
192
+ 相关度不低于均线 → 放弃它就是在拿准确性换速度。
193
+ """
194
+ from collections import Counter
195
+ db = state / "adaptive.db"
196
+ per: dict[str, list[float]] = defaultdict(list)
197
+ if db.exists():
198
+ with sqlite3.connect(db) as conn:
199
+ for eng, qv in conn.execute(
200
+ "select engine, quality from engine_perf where quality > 0"):
201
+ per[eng].append(float(qv))
202
+ q_mean = {e: st.mean(v) for e, v in per.items() if len(v) >= 5}
203
+ baseline = st.mean(q_mean.values()) if q_mean else 0.0
204
+
205
+ lines = ["", "=== 被放弃交付的质量画像 ===",
206
+ f"质量口径:result_relevance 词面覆盖代理;全库均线 {baseline:.3f}",
207
+ f"{'engine':16s} {'次数':>5s} {'相关度代理':>9s} {'相对均线':>9s} {'P75ms':>7s}"]
208
+ for eng, n in Counter(r["engine"] for r in lost).most_common():
209
+ q = q_mean.get(eng)
210
+ p = (p75 or {}).get(eng)
211
+ lines.append(
212
+ f"{eng:16s} {n:5d} {(f'{q:.3f}' if q is not None else 'n/a'):>9s} "
213
+ f"{(f'{q - baseline:+.3f}' if q is not None else '-'):>9s} "
214
+ f"{(f'{p:.0f}' if p else '-'):>7s}")
215
+ hosts = set(host_bodies)
216
+ lines.append("")
217
+ lines.append(f"本地正文缓存覆盖 {len(hosts)} 个站点(fulltext/index.jsonl)")
218
+ lines.append("读法:相对均线为负且从未进过正文缓存 → 早收工合理;"
219
+ "相对均线不为负 → 放弃它是在拿准确性换速度。")
220
+ return "\n".join(lines)
221
+
222
+
223
+ def main(argv: list[str] | None = None) -> int:
224
+ ap = argparse.ArgumentParser(description="尾宽策略离线 A/B 回放")
225
+ ap.add_argument("--candidate", type=float, default=0.6,
226
+ help="候选尾宽(秒),默认 0.6")
227
+ ap.add_argument("--record", action="store_true",
228
+ help="把本次回放写进 docs-local/tail_ab_<日期>.json")
229
+ ap.add_argument("--detail", action="store_true",
230
+ help="扫表时附带「被放弃交付」的引擎分布(判断该缩尾宽还是该降权)")
231
+ ap.add_argument("--quality", action="store_true",
232
+ help="附带质量评估:被放弃的那批交付里,有多少本来会变成正文证据")
233
+ ap.add_argument("--sweep", default="",
234
+ help="逗号分隔的候选尾宽,一次性扫一遍(如 0.4,0.6,0.8,1.0)")
235
+ args = ap.parse_args(argv)
236
+
237
+ state = state_path()
238
+ queries = load_queries(state / "usage_log" / "query.jsonl")
239
+ p75 = load_p75(state / "adaptive.db")
240
+ if not queries:
241
+ print("没有可回放的查询日志(usage_log/query.jsonl 为空);"
242
+ "先跑几次 argo search 再来。")
243
+ return 1
244
+ if args.sweep:
245
+ print(f"{'尾宽(s)':>8} {'p50(ms)':>8} {'p90(ms)':>8} {'总省(s)':>8} {'放弃交付':>8}")
246
+ for cand in [float(x) for x in args.sweep.split(",") if x.strip()]:
247
+ r = replay(queries, p75, cand)
248
+ print(f"{cand:8.2f} {r['new']['p50']:8.0f} {r['new']['p90']:8.0f} "
249
+ f"{r['saved_ms_total'] / 1000:8.1f} {r['slow_delivery_lost']:8d}")
250
+ print("(扫表是给人看的:选型要同时看「省多少」和「丢掉多少慢源交付」)")
251
+ if args.detail:
252
+ lost = lost_deliveries(queries, p75, float(args.sweep.split(",")[0]))
253
+ from collections import Counter
254
+ print("被放弃的交付按引擎:", Counter(r["engine"] for r in lost).most_common(8))
255
+ if args.quality:
256
+ lost = lost_deliveries(queries, p75, float(args.sweep.split(",")[0]))
257
+ print(quality_report(lost, load_host_bodies(state), state, p75))
258
+ return 0
259
+ report = replay(queries, p75, args.candidate)
260
+ print(f"回放 {report['n']} 条真实查询(已剔合成引擎);"
261
+ f"覆盖 {len(p75)} 个有延迟分布的引擎")
262
+ print(f" 现行尾宽 {BASE_TAIL:.2f}s: p50 {report['old']['p50']:.0f}ms "
263
+ f"p90 {report['old']['p90']:.0f}ms")
264
+ print(f" 候选尾宽 {args.candidate:.2f}s: p50 {report['new']['p50']:.0f}ms "
265
+ f"p90 {report['new']['p90']:.0f}ms")
266
+ print(f" 合计省 {report['saved_ms_total'] / 1000:.1f}s;"
267
+ f"放弃的慢源交付 {report['slow_delivery_lost']} 次(需人工评估这些查询是否本来就要慢源)")
268
+ if args.record:
269
+ out_dir = Path(__file__).resolve().parent.parent / "docs-local"
270
+ out_dir.mkdir(exist_ok=True)
271
+ path = out_dir / f"tail_ab_{time.strftime('%Y%m%d')}.json"
272
+ path.write_text(json.dumps({"candidate": args.candidate, **report},
273
+ ensure_ascii=False, indent=2), encoding="utf-8")
274
+ print(f"已记录:{path}")
275
+ else:
276
+ print("(dry-run:未做结论、未写状态。确认要上线前用 --record 留档)")
277
+ return 0
278
+
279
+
280
+ if __name__ == "__main__":
281
+ sys.exit(main())
@@ -19,16 +19,43 @@
19
19
  * 单个任务抛异常只记成该任务失败,不影响其它任务;
20
20
  * 总墙钟时间被 ``wait_s`` 硬性约束,不会超出。
21
21
 
22
- search.py 的引擎并发已经是这套做法,这里抽成通用实现,供 research / social /
23
- crawl / fetch 等复用,避免同一类问题在各处重复出现。
22
+ 时间来源(2026-10-24)
23
+ ----------------------
24
+ 本模块读时间只经 ``clock()``、等待只经 ``fetch_sleep()`` / ``poll_wait()``
25
+ (见 ``tests/_measure.py``)。原因是实测:调度类用例红过多次,根因全是
26
+ 「把机器有多忙当成调度对不对」——轮询周期本身会吃掉十几毫秒,于是
27
+ 「等比累加 sleep 时长」与真实墙钟之间出现系统性偏差。现在一律按**剩余时间**
28
+ (deadline 语义)判断还等不等,单次轮询抖动不再累加。
29
+
30
+ 「完成」的判据与池化
31
+ --------------------
32
+ 判定某任务完成一律看 ``holder["done"]``,**不看** ``Thread.is_alive()``:
33
+ 线程跑完最后一行到被主循环观测之间有一段窗口,is_alive() 会把这段窗口里的
34
+ 任务误判成「还在跑」,旧实现因此在全量套件里间歇性把已完成结果记成超时。
35
+ 主循环改为:补发 → 收成 → 判断是否够(enough)→ 判断是否到点 → 再等一拍。
36
+ 「到点再收一次」放在循环尾部,保证最后一批结果不会被预算边界吃掉。
24
37
  """
25
38
 
26
39
  from __future__ import annotations
27
40
 
41
+ import contextlib
28
42
  import threading
29
43
  import time
30
44
  from typing import Any, Callable, Generic, TypeVar
31
45
 
46
+ try: # 测试域由 tests/_measure 注入虚拟时钟;运行期退回真实时间
47
+ from _measure import clock, fetch_sleep, poll_wait, task_deadline # type: ignore
48
+ except ImportError: # pragma: no cover - 生产路径
49
+ clock = time.monotonic
50
+ fetch_sleep = time.sleep
51
+
52
+ def poll_wait(seconds: float, evt: threading.Event | None = None) -> None:
53
+ (evt.wait(seconds) if evt is not None else time.sleep(seconds))
54
+
55
+ @contextlib.contextmanager
56
+ def task_deadline(_deadline: float | None = None, *, budget_s: float | None = None):
57
+ yield
58
+
32
59
  I = TypeVar("I")
33
60
  O = TypeVar("O")
34
61
 
@@ -62,23 +89,25 @@ def run_bounded(
62
89
 
63
90
  * ``finished``:已完成任务的 ``(item, value)`` 列表,按完成先后排列;
64
91
  任务抛异常时 ``value`` 是 :class:`TaskError`,调用方按需转成失败占位。
65
- * ``unfinished``:到点仍没完成(或被 ``enough`` 提前收尾时还在跑)的 item 列表,
66
- 调用方可据此补超时提示。这些任务不会被 join,本函数因此能准时返回。
92
+ * ``unfinished``:到点仍没完成(或被 ``enough`` 提前收尾时还在跑)的 item
93
+ 列表。这些任务不会被 join,本函数因此能准时返回。
67
94
 
68
- ``enough(finished)`` 返回 True 时提前收尾:不再启动排队中的剩余任务,已在跑的
69
- 任务归入 unfinished。``max_workers`` 限制同时在跑的任务数。
95
+ ``enough(finished)`` 返回 True 时提前收尾:不再启动排队中的剩余任务,已在跑
96
+ 的任务归入 unfinished。``max_workers`` 限制同时在跑的任务数。
70
97
 
71
- ``stagger_s`` 起步间隔(秒),单调:0 = 不节流(立即填满并发位);值越大越接近
72
- 严格串行(大于单任务超时上限即完全退回串行)。仅在**有任务在跑**时节制——
73
- 对冲语义(首任务先跑,宽限窗内交付就只付一次调用),而非无条件串行化;在跑
74
- 任务数为 0 时立即补发(前一个已返回,没有可对冲的对象,等满窗纯属浪费)。
98
+ ``stagger_s`` 起步间隔(秒),单调:0 = 不节流(立即填满并发位);值越大越
99
+ 接近严格串行(大于单任务超时上限即完全退回串行)。仅在**有任务在跑**时节制
100
+ ——对冲语义(首任务先跑,宽限窗内交付就只付一次调用),而非无条件串行化;
101
+ 在跑任务数为 0 时立即补发(前一个已返回,没有可对冲的对象,等满窗纯属浪费)。
75
102
  engine_dispatch 的 hedged race 与 search_pipeline 的恢复链是同一哲学的两个
76
103
  配置:改这里的起步语义,两边同时生效。
77
104
  """
78
105
  queue = list(items)
79
106
  running: list[tuple[threading.Thread, I, dict]] = []
80
107
  finished: list[tuple[I, Any]] = []
81
- deadline = time.monotonic() + max(0.0, float(wait_s))
108
+ # 预算锚点必须在启动任何线程之前取:线程一推进时钟,deadline 就跟着漂。
109
+ t0 = clock()
110
+ deadline = t0 + max(0.0, float(wait_s))
82
111
  last_start: float | None = None
83
112
  done_evt = threading.Event()
84
113
 
@@ -87,58 +116,71 @@ def run_bounded(
87
116
 
88
117
  def _work() -> None:
89
118
  try:
90
- holder["v"] = worker(item)
91
- except BaseException as exc: # noqa: BLE001 - 任意任务异常都要隔离,不能让一个任务炸掉整组
119
+ # 任务的等待继承本轮预算:超过预算仍未回来,测试域会立刻指认
120
+ # 「这次等待越过了预算」,而不是让调用方事后拿墙钟去猜。
121
+ # 用**相对预算**而不是外围算好的绝对 deadline:那个 deadline
122
+ # 取自 clock(),与本层真实模式下的 monotonic 不同源(实测混用会
123
+ # 得到「已过期 65 秒」的假锚点,把正常等待判成越界)。
124
+ with task_deadline(budget_s=max(0.0, deadline - t0)):
125
+ holder["v"] = worker(item)
126
+ except BaseException as exc: # noqa: BLE001 - 任意任务异常都要隔离
92
127
  holder["v"] = TaskError(item, exc)
93
128
  finally:
129
+ holder["done"] = True
94
130
  done_evt.set()
95
131
 
96
132
  th = threading.Thread(target=_work, daemon=True)
97
133
  th.start()
98
134
  running.append((th, item, holder))
99
135
 
136
+ def _harvest() -> bool:
137
+ """收已完成的任务,返回本轮是否收到新结果。"""
138
+ got = False
139
+ for entry in list(running):
140
+ th, item, holder = entry
141
+ if not holder.get("done"):
142
+ continue
143
+ running.remove(entry)
144
+ finished.append((item, holder.get(
145
+ "v", TaskError(item, RuntimeError("任务结束但没有返回值")))))
146
+ got = True
147
+ return got
148
+
100
149
  while queue or running:
101
150
  done_evt.clear()
102
- if time.monotonic() >= deadline:
103
- break
104
- # 按并发上限补启动排队任务
151
+ # 1) 补发排队任务(受并发上限与 stagger 节流约束)
105
152
  while queue and len(running) < max(1, int(max_workers)):
106
153
  if (stagger_s > 0 and running and last_start is not None
107
- and (time.monotonic() - last_start) < stagger_s):
154
+ and (clock() - last_start) < stagger_s):
108
155
  break # 起步节流:等满 stagger 再补发(对冲)
109
156
  _start(queue.pop(0))
110
- last_start = time.monotonic()
111
- progressed = False
112
- for entry in list(running):
113
- th, item, holder = entry
114
- if th.is_alive():
115
- continue
116
- running.remove(entry)
117
- value = holder.get("v", TaskError(item, RuntimeError("任务结束但没有返回值")))
118
- finished.append((item, value))
119
- progressed = True
120
- if enough is not None and enough(finished):
121
- queue.clear()
122
- break
157
+ last_start = clock()
158
+ # 2) 收成。注意不能因为收到结果就 `continue`:那会跳过本轮的
159
+ # enough 判定与到点判定,把「够了就收工」变成「等下一个任务也回来」。
160
+ _harvest()
161
+ # 3) 够了就提前收工:排队中的任务不再启动(它们是主动放弃,不是超时)
123
162
  if enough is not None and enough(finished):
163
+ queue.clear()
164
+ break
165
+ # 4) 到点必须收工——但先再收一次,别把预算边界上刚好完成的结果丢掉
166
+ if clock() >= deadline:
167
+ _harvest()
168
+ break
169
+ # 5) 等一拍:按**剩余时间**算,不让单次抖动累加成常驻偏差
170
+ if not queue and not running:
124
171
  break
125
- if not progressed:
126
- remain = deadline - time.monotonic()
127
- if remain <= 0:
128
- break
129
- wait_step = remain
130
- if queue and stagger_s > 0 and last_start is not None:
131
- wait_step = min(wait_step, max(0.0, stagger_s - (time.monotonic() - last_start)))
132
- done_evt.wait(min(poll, wait_step) if poll > 0 else wait_step)
133
-
134
- # 收尾:还活着的不等待(关键:不 join),计入 unfinished;恰在末次轮询后完成的
135
- # 照常收进来,避免结果悄悄丢失。因并发上限到点仍没排上起跑的任务同样算未完成,
136
- # 不能让它既不在 finished、也不在 unfinished 里被悄悄丢掉。
137
- unfinished: list[I] = list(queue)
138
- for th, item, holder in running:
139
- if th.is_alive():
140
- unfinished.append(item)
141
- else:
142
- value = holder.get("v", TaskError(item, RuntimeError("任务结束但没有返回值")))
143
- finished.append((item, value))
172
+ remain = deadline - clock()
173
+ wait_step = remain
174
+ if queue and stagger_s > 0 and last_start is not None:
175
+ wait_step = min(wait_step, max(0.0, stagger_s - (clock() - last_start)))
176
+ step = min(poll, wait_step) if poll > 0 else wait_step
177
+ poll_wait(step, done_evt)
178
+
179
+ # 收尾:再收一次(线程与观测之间有时间差),其余不 join、计入 unfinished。
180
+ _harvest()
181
+ # 没排上起跑的任务(仍在 queue)与新起跑但没回来的任务(running)都要进
182
+ # unfinished:前者只是没轮到,但也确实没结果,悄悄丢掉会让调用方以为
183
+ # 「全部拿到了」。两者的区别用 downstream 的 enough 提前收尾路径处理:
184
+ # 那条路径上 queue 已被清空,所以不会有误报。
185
+ unfinished: list[I] = list(queue) + [item for _th, item, _h in running]
144
186
  return finished, unfinished
@@ -0,0 +1,22 @@
1
+ #!/usr/bin/env python3
2
+ """clock_bridge.py — 统一时钟的一行入口(2026-10-24)。
3
+
4
+ 计时断言与「预算夹紧」判定不该取决于机器快慢:测试域用 tests/_measure 的
5
+ 虚拟时钟,生产路径就是 ``time.monotonic``。各模块从这里取 ``clock``,不再各自
6
+ 复制一段 try/except(那会让千行文件更难拆,也容易只改一半——本文件对应的
7
+ fetch_v3 就曾因「预算夹在超时请求上、耗时却按真实墙钟算」而被测出击穿)。
8
+
9
+ 与 ``tests/_measure.py`` 的关系:那个模块定义语义(虚拟/真实、预算锚点、
10
+ 确定性时间线),本模块只做「生产环境安全地取到它」这一件事。
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import time
16
+
17
+ try: # 测试域:虚拟时钟(见 tests/_measure.py)
18
+ from _measure import clock # type: ignore
19
+ except ImportError: # pragma: no cover - 生产路径
20
+ clock = time.monotonic
21
+
22
+ __all__ = ["clock"]