codebee 0.1.56 → 0.1.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +4 -3
- package/app/.playwright-cli/console-2026-09-23T08-26-03-635Z.log +88 -0
- package/app/.playwright-cli/page-2026-09-23T08-26-03-845Z.yml +126 -0
- package/app/.playwright-cli/page-2026-09-23T08-29-44-203Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-35-42-046Z.yml +210 -0
- package/app/.playwright-cli/page-2026-09-23T08-36-52-858Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-37-33-938Z.png +0 -0
- package/app/.playwright-cli/run-detail-after.png +0 -0
- package/app/core/beekeeper.py +183 -0
- package/app/core/branching.py +31 -6
- package/app/core/jobs.py +3 -3
- package/app/core/pipeline.py +4143 -4112
- package/app/core/publish/browser.py +4 -0
- package/app/core/remote.py +2 -0
- package/app/core/runner.py +486 -474
- package/app/core/selfupdate.py +345 -336
- package/app/core/skill_scan.py +11 -2
- package/app/core/store.py +95 -64
- package/app/core/zentao.py +80 -0
- package/app/main.py +63 -5
- package/app/ui/app.js +13954 -13639
- package/app/ui/board.css +732 -650
- package/app/ui/i18n.js +10 -2
- package/app/ui/index.html +1 -2
- package/app/ui/style.css +6106 -6107
- package/package.json +1 -1
package/app/core/runner.py
CHANGED
|
@@ -22,15 +22,15 @@ import tempfile
|
|
|
22
22
|
import threading
|
|
23
23
|
import time
|
|
24
24
|
|
|
25
|
+
from . import beekeeper
|
|
25
26
|
from .env_scrub import scrub_env
|
|
26
27
|
from .error_codes import ErrorCode
|
|
27
28
|
|
|
28
29
|
# 非 Windows 必须置 0:POSIX 的 Popen 对非零 creationflags 直接抛 ValueError,
|
|
29
30
|
# 置 0 则两边通用(remote.py 同款守卫)。
|
|
30
31
|
CREATE_NO_WINDOW = 0x08000000 if os.name == "nt" else 0
|
|
31
|
-
DEFAULT_TIMEOUT = 1200 # 单步 20 分钟
|
|
32
|
-
|
|
33
|
-
DEFAULT_STREAM_ACTIVITY_TIMEOUT_S = 180 # 流式 CLI 的活动延长窗口
|
|
32
|
+
DEFAULT_TIMEOUT = 1200 # 单步 20 分钟
|
|
33
|
+
DEFAULT_STREAM_ACTIVITY_TIMEOUT_S = 180 # 流式 CLI 的活动延长窗口
|
|
34
34
|
|
|
35
35
|
_BASH_CANDIDATES = [
|
|
36
36
|
r"D:\Git\usr\bin\bash.exe",
|
|
@@ -219,6 +219,16 @@ def _drain_streams(proc, t_out, t_err, *, timeout=10):
|
|
|
219
219
|
remaining = max(0.0, deadline - time.time())
|
|
220
220
|
if remaining > 0:
|
|
221
221
|
t.join(timeout=remaining)
|
|
222
|
+
# Killing the tree does not necessarily reap the root process on Windows.
|
|
223
|
+
# Reap it with the remaining bounded budget before callers inspect
|
|
224
|
+
# ``returncode``; otherwise long-running timeout tests can leak a live
|
|
225
|
+
# Popen handle and emit ResourceWarning during interpreter shutdown.
|
|
226
|
+
remaining = max(0.0, deadline - time.time())
|
|
227
|
+
if remaining > 0 and proc.poll() is None:
|
|
228
|
+
try:
|
|
229
|
+
proc.wait(timeout=remaining)
|
|
230
|
+
except (subprocess.TimeoutExpired, OSError):
|
|
231
|
+
pass
|
|
222
232
|
|
|
223
233
|
|
|
224
234
|
def _pipe_reader(stream, chunks, log_fh, stamp=None):
|
|
@@ -238,13 +248,13 @@ def _pipe_reader(stream, chunks, log_fh, stamp=None):
|
|
|
238
248
|
b = stream.read1(65536)
|
|
239
249
|
if not b:
|
|
240
250
|
break
|
|
241
|
-
chunks.append(b)
|
|
242
|
-
if stamp is not None:
|
|
243
|
-
stamp[0] = time.time()
|
|
244
|
-
if len(stamp) > 1:
|
|
245
|
-
stamp[1] = True
|
|
246
|
-
if len(stamp) > 2:
|
|
247
|
-
stamp[2] = time.monotonic()
|
|
251
|
+
chunks.append(b)
|
|
252
|
+
if stamp is not None:
|
|
253
|
+
stamp[0] = time.time()
|
|
254
|
+
if len(stamp) > 1:
|
|
255
|
+
stamp[1] = True
|
|
256
|
+
if len(stamp) > 2:
|
|
257
|
+
stamp[2] = time.monotonic()
|
|
248
258
|
if log_fh:
|
|
249
259
|
try:
|
|
250
260
|
log_fh.write(b)
|
|
@@ -564,98 +574,98 @@ def pretty_cli_log(text, max_event_chars=4000):
|
|
|
564
574
|
return "\n".join(out)
|
|
565
575
|
|
|
566
576
|
|
|
567
|
-
def _normalize_abort_markers(value):
|
|
568
|
-
"""把重复错误配置规范成 ``[(marker, count), ...]``。
|
|
569
|
-
|
|
570
|
-
旧调用方传的是单个 ``(marker, count)`` 元组,必须继续支持;新的
|
|
571
|
-
调用方可以传字典或元组列表,以便同一 CLI 同时对多种断流文案止损。
|
|
572
|
-
"""
|
|
573
|
-
if not value:
|
|
574
|
-
return []
|
|
575
|
-
if isinstance(value, str):
|
|
576
|
-
return [(value, 1)]
|
|
577
|
-
if isinstance(value, dict):
|
|
578
|
-
raw = list(value.items())
|
|
579
|
-
elif (isinstance(value, (tuple, list)) and len(value) == 2
|
|
580
|
-
and isinstance(value[0], str)):
|
|
581
|
-
raw = [value]
|
|
582
|
-
else:
|
|
583
|
-
try:
|
|
584
|
-
raw = list(value)
|
|
585
|
-
except TypeError:
|
|
586
|
-
return []
|
|
587
|
-
out = []
|
|
588
|
-
for item in raw:
|
|
589
|
-
if isinstance(item, str):
|
|
590
|
-
marker, limit = item, 1
|
|
591
|
-
else:
|
|
592
|
-
try:
|
|
593
|
-
marker, limit = item
|
|
594
|
-
except (TypeError, ValueError):
|
|
595
|
-
continue
|
|
596
|
-
marker = str(marker or "")
|
|
597
|
-
try:
|
|
598
|
-
limit = max(1, int(limit))
|
|
599
|
-
except (TypeError, ValueError):
|
|
600
|
-
continue
|
|
601
|
-
if marker:
|
|
602
|
-
out.append((marker, limit))
|
|
603
|
-
return out
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
607
|
-
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None,
|
|
608
|
-
stall_timeout=0, repeat_abort=None, deadline=None,
|
|
609
|
-
abort_markers=None, activity_timeout=0):
|
|
577
|
+
def _normalize_abort_markers(value):
|
|
578
|
+
"""把重复错误配置规范成 ``[(marker, count), ...]``。
|
|
579
|
+
|
|
580
|
+
旧调用方传的是单个 ``(marker, count)`` 元组,必须继续支持;新的
|
|
581
|
+
调用方可以传字典或元组列表,以便同一 CLI 同时对多种断流文案止损。
|
|
582
|
+
"""
|
|
583
|
+
if not value:
|
|
584
|
+
return []
|
|
585
|
+
if isinstance(value, str):
|
|
586
|
+
return [(value, 1)]
|
|
587
|
+
if isinstance(value, dict):
|
|
588
|
+
raw = list(value.items())
|
|
589
|
+
elif (isinstance(value, (tuple, list)) and len(value) == 2
|
|
590
|
+
and isinstance(value[0], str)):
|
|
591
|
+
raw = [value]
|
|
592
|
+
else:
|
|
593
|
+
try:
|
|
594
|
+
raw = list(value)
|
|
595
|
+
except TypeError:
|
|
596
|
+
return []
|
|
597
|
+
out = []
|
|
598
|
+
for item in raw:
|
|
599
|
+
if isinstance(item, str):
|
|
600
|
+
marker, limit = item, 1
|
|
601
|
+
else:
|
|
602
|
+
try:
|
|
603
|
+
marker, limit = item
|
|
604
|
+
except (TypeError, ValueError):
|
|
605
|
+
continue
|
|
606
|
+
marker = str(marker or "")
|
|
607
|
+
try:
|
|
608
|
+
limit = max(1, int(limit))
|
|
609
|
+
except (TypeError, ValueError):
|
|
610
|
+
continue
|
|
611
|
+
if marker:
|
|
612
|
+
out.append((marker, limit))
|
|
613
|
+
return out
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
617
|
+
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None,
|
|
618
|
+
stall_timeout=0, repeat_abort=None, deadline=None,
|
|
619
|
+
abort_markers=None, activity_timeout=0):
|
|
610
620
|
"""通用子进程执行:并发读管道防死锁;超时/取消杀整棵进程树。
|
|
611
621
|
|
|
612
|
-
stall_timeout:停滞看门狗(秒,0=关闭)——超过该时长 stdout/stderr 无任何
|
|
613
|
-
新输出即判卡死,提前杀树返回(timed_out=True + stalled=True)。只对输出
|
|
614
|
-
持续流动的 CLI 开(codex JSONL 事件流);claude json 到结束才一次性输出,
|
|
615
|
-
开了会把正常长任务误杀。
|
|
616
|
-
|
|
617
|
-
activity_timeout:基础 timeout 到期后,最近仍有真实输出的流式 CLI 允许继续
|
|
618
|
-
运行的最长静默窗口(秒,0=关闭)。它独立于 stall_timeout,避免环境把
|
|
619
|
-
停滞看门狗设为 0 后,Claude/Codex 又在基础超时处被硬杀;总延长仍有上限。
|
|
620
|
-
|
|
621
|
-
repeat_abort:可选 (marker, count),也接受多个 ``(marker, count)``;同一
|
|
622
|
-
致命错误达到次数即提前杀树,处理 CLI 自身不断打印重连消息、因此永远
|
|
623
|
-
触发不了静默看门狗的假运行。abort_markers 是兼容扩展,用于不改变旧
|
|
624
|
-
repeat_abort 参数形状的情况下追加多种一次性断流标记。
|
|
625
|
-
|
|
626
|
-
deadline:可选的 ``time.monotonic()`` 绝对截止时刻。它比 timeout 优先,
|
|
627
|
-
到点会杀掉进程树并返回 deadline_exceeded=True,调用方可把整个任务收口
|
|
628
|
-
为 timeout,而不是把它误记成普通供应商失败。
|
|
629
|
-
|
|
630
|
-
返回 {ok, exit_code, stdout, stderr, duration, cancelled, timed_out, stalled,
|
|
631
|
-
deadline_exceeded, abort_marker}。
|
|
632
|
-
"""
|
|
622
|
+
stall_timeout:停滞看门狗(秒,0=关闭)——超过该时长 stdout/stderr 无任何
|
|
623
|
+
新输出即判卡死,提前杀树返回(timed_out=True + stalled=True)。只对输出
|
|
624
|
+
持续流动的 CLI 开(codex JSONL 事件流);claude json 到结束才一次性输出,
|
|
625
|
+
开了会把正常长任务误杀。
|
|
626
|
+
|
|
627
|
+
activity_timeout:基础 timeout 到期后,最近仍有真实输出的流式 CLI 允许继续
|
|
628
|
+
运行的最长静默窗口(秒,0=关闭)。它独立于 stall_timeout,避免环境把
|
|
629
|
+
停滞看门狗设为 0 后,Claude/Codex 又在基础超时处被硬杀;总延长仍有上限。
|
|
630
|
+
|
|
631
|
+
repeat_abort:可选 (marker, count),也接受多个 ``(marker, count)``;同一
|
|
632
|
+
致命错误达到次数即提前杀树,处理 CLI 自身不断打印重连消息、因此永远
|
|
633
|
+
触发不了静默看门狗的假运行。abort_markers 是兼容扩展,用于不改变旧
|
|
634
|
+
repeat_abort 参数形状的情况下追加多种一次性断流标记。
|
|
635
|
+
|
|
636
|
+
deadline:可选的 ``time.monotonic()`` 绝对截止时刻。它比 timeout 优先,
|
|
637
|
+
到点会杀掉进程树并返回 deadline_exceeded=True,调用方可把整个任务收口
|
|
638
|
+
为 timeout,而不是把它误记成普通供应商失败。
|
|
639
|
+
|
|
640
|
+
返回 {ok, exit_code, stdout, stderr, duration, cancelled, timed_out, stalled,
|
|
641
|
+
deadline_exceeded, abort_marker}。
|
|
642
|
+
"""
|
|
633
643
|
if shell_cmd:
|
|
634
644
|
# shell 串的解析器随平台:重定向/引号语法两边通用,只是解释器不同
|
|
635
645
|
argv = ["cmd", "/c", shell_cmd] if os.name == "nt" else ["/bin/sh", "-c", shell_cmd]
|
|
636
|
-
if argv is None:
|
|
637
|
-
return {"ok": False, "exit_code": None, "stdout": "", "stderr": "argv 为空",
|
|
638
|
-
"duration": 0.0, "cancelled": False, "timed_out": False,
|
|
639
|
-
"stalled": False, "deadline_exceeded": False, "abort_marker": None}
|
|
640
|
-
try:
|
|
641
|
-
deadline = float(deadline) if deadline is not None else None
|
|
642
|
-
except (TypeError, ValueError):
|
|
643
|
-
deadline = None
|
|
644
|
-
try:
|
|
645
|
-
timeout = max(0.0, float(timeout))
|
|
646
|
-
except (TypeError, ValueError):
|
|
647
|
-
timeout = float(DEFAULT_TIMEOUT)
|
|
648
|
-
try:
|
|
649
|
-
activity_timeout = max(0.0, float(activity_timeout))
|
|
650
|
-
except (TypeError, ValueError):
|
|
651
|
-
activity_timeout = 0.0
|
|
652
|
-
if deadline is not None and time.monotonic() >= deadline:
|
|
653
|
-
return {"ok": False, "exit_code": None, "stdout": "",
|
|
654
|
-
"stderr": "[任务总时限已到,未启动子进程]", "duration": 0.0,
|
|
655
|
-
"cancelled": False, "timed_out": True, "stalled": False,
|
|
656
|
-
"deadline_exceeded": True, "abort_marker": None}
|
|
657
|
-
aborts = _normalize_abort_markers(repeat_abort)
|
|
658
|
-
aborts.extend(_normalize_abort_markers(abort_markers))
|
|
646
|
+
if argv is None:
|
|
647
|
+
return {"ok": False, "exit_code": None, "stdout": "", "stderr": "argv 为空",
|
|
648
|
+
"duration": 0.0, "cancelled": False, "timed_out": False,
|
|
649
|
+
"stalled": False, "deadline_exceeded": False, "abort_marker": None}
|
|
650
|
+
try:
|
|
651
|
+
deadline = float(deadline) if deadline is not None else None
|
|
652
|
+
except (TypeError, ValueError):
|
|
653
|
+
deadline = None
|
|
654
|
+
try:
|
|
655
|
+
timeout = max(0.0, float(timeout))
|
|
656
|
+
except (TypeError, ValueError):
|
|
657
|
+
timeout = float(DEFAULT_TIMEOUT)
|
|
658
|
+
try:
|
|
659
|
+
activity_timeout = max(0.0, float(activity_timeout))
|
|
660
|
+
except (TypeError, ValueError):
|
|
661
|
+
activity_timeout = 0.0
|
|
662
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
663
|
+
return {"ok": False, "exit_code": None, "stdout": "",
|
|
664
|
+
"stderr": "[任务总时限已到,未启动子进程]", "duration": 0.0,
|
|
665
|
+
"cancelled": False, "timed_out": True, "stalled": False,
|
|
666
|
+
"deadline_exceeded": True, "abort_marker": None}
|
|
667
|
+
aborts = _normalize_abort_markers(repeat_abort)
|
|
668
|
+
aborts.extend(_normalize_abort_markers(abort_markers))
|
|
659
669
|
full_env = scrub_env(os.environ.copy(), mode="drop")
|
|
660
670
|
if env:
|
|
661
671
|
# 5A:env 关键字环境变量注入用户传入的 env(属于有意注入,例如模型 API key)
|
|
@@ -686,13 +696,16 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
686
696
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
687
697
|
creationflags=CREATE_NO_WINDOW,
|
|
688
698
|
start_new_session=(os.name != "nt")) # POSIX 需独立进程组供 killpg 杀树;Windows 忽略该参数
|
|
689
|
-
except Exception as e:
|
|
690
|
-
return {"ok": False, "exit_code": None, "stdout": "",
|
|
691
|
-
"stderr": "启动失败: %r" % e, "duration": 0.0,
|
|
692
|
-
"cancelled": False, "timed_out": False, "stalled": False,
|
|
693
|
-
"deadline_exceeded": False, "abort_marker": None}
|
|
699
|
+
except Exception as e:
|
|
700
|
+
return {"ok": False, "exit_code": None, "stdout": "",
|
|
701
|
+
"stderr": "启动失败: %r" % e, "duration": 0.0,
|
|
702
|
+
"cancelled": False, "timed_out": False, "stalled": False,
|
|
703
|
+
"deadline_exceeded": False, "abort_marker": None}
|
|
704
|
+
# 挂巢(beekeeper):服务无论以何种方式退出(含被硬杀),蜜蜂 CLI
|
|
705
|
+
# 一起带走。挂巢失败只降级,绝不影响本步起跑。
|
|
706
|
+
beekeeper.adopt(proc)
|
|
694
707
|
out_chunks, err_chunks = [], []
|
|
695
|
-
stamp = [time.time(), False, time.monotonic()] # [墙钟、是否有输出、单调时钟]
|
|
708
|
+
stamp = [time.time(), False, time.monotonic()] # [墙钟、是否有输出、单调时钟]
|
|
696
709
|
t_out = threading.Thread(target=_pipe_reader, args=(proc.stdout, out_chunks, log_fh, stamp), daemon=True)
|
|
697
710
|
t_err = threading.Thread(target=_pipe_reader, args=(proc.stderr, err_chunks, log_fh, stamp), daemon=True)
|
|
698
711
|
t_out.start()
|
|
@@ -709,41 +722,41 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
709
722
|
except Exception:
|
|
710
723
|
pass
|
|
711
724
|
threading.Thread(target=_feed, daemon=True).start()
|
|
712
|
-
start = time.time()
|
|
713
|
-
start_mono = time.monotonic()
|
|
714
|
-
cancelled = timed_out = stalled = repeat_aborted = False
|
|
715
|
-
deadline_exceeded = False
|
|
716
|
-
abort_marker = None
|
|
717
|
-
while True:
|
|
725
|
+
start = time.time()
|
|
726
|
+
start_mono = time.monotonic()
|
|
727
|
+
cancelled = timed_out = stalled = repeat_aborted = False
|
|
728
|
+
deadline_exceeded = False
|
|
729
|
+
abort_marker = None
|
|
730
|
+
while True:
|
|
718
731
|
try:
|
|
719
732
|
proc.wait(timeout=0.4)
|
|
720
733
|
break
|
|
721
734
|
except subprocess.TimeoutExpired:
|
|
722
735
|
pass
|
|
723
|
-
if cancel_event is not None and cancel_event.is_set():
|
|
724
|
-
cancelled = True
|
|
725
|
-
_kill_tree(proc.pid)
|
|
726
|
-
_drain_streams(proc, t_out, t_err, timeout=10)
|
|
727
|
-
break
|
|
728
|
-
if deadline is not None and time.monotonic() >= deadline:
|
|
729
|
-
deadline_exceeded = True
|
|
730
|
-
timed_out = True
|
|
731
|
-
_kill_tree(proc.pid)
|
|
732
|
-
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
733
|
-
break
|
|
734
|
-
elapsed = time.monotonic() - start_mono
|
|
735
|
-
if elapsed > timeout:
|
|
736
|
-
# Claude/Codex 的工具循环可能持续吐 stream-json 事件而迟迟
|
|
737
|
-
# 不给最终 result。活动延长与 stall 看门狗分离:无输出的挂死
|
|
738
|
-
# 仍按原单模型 timeout 快速换将,收到过输出则最多再给一个活动窗口。
|
|
739
|
-
active = (activity_timeout > 0 and stamp[1]
|
|
740
|
-
and time.monotonic() - stamp[2] < activity_timeout
|
|
741
|
-
and elapsed < timeout + activity_timeout)
|
|
742
|
-
if not active:
|
|
743
|
-
timed_out = True
|
|
744
|
-
_kill_tree(proc.pid)
|
|
745
|
-
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
746
|
-
break
|
|
736
|
+
if cancel_event is not None and cancel_event.is_set():
|
|
737
|
+
cancelled = True
|
|
738
|
+
_kill_tree(proc.pid)
|
|
739
|
+
_drain_streams(proc, t_out, t_err, timeout=10)
|
|
740
|
+
break
|
|
741
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
742
|
+
deadline_exceeded = True
|
|
743
|
+
timed_out = True
|
|
744
|
+
_kill_tree(proc.pid)
|
|
745
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
746
|
+
break
|
|
747
|
+
elapsed = time.monotonic() - start_mono
|
|
748
|
+
if elapsed > timeout:
|
|
749
|
+
# Claude/Codex 的工具循环可能持续吐 stream-json 事件而迟迟
|
|
750
|
+
# 不给最终 result。活动延长与 stall 看门狗分离:无输出的挂死
|
|
751
|
+
# 仍按原单模型 timeout 快速换将,收到过输出则最多再给一个活动窗口。
|
|
752
|
+
active = (activity_timeout > 0 and stamp[1]
|
|
753
|
+
and time.monotonic() - stamp[2] < activity_timeout
|
|
754
|
+
and elapsed < timeout + activity_timeout)
|
|
755
|
+
if not active:
|
|
756
|
+
timed_out = True
|
|
757
|
+
_kill_tree(proc.pid)
|
|
758
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
759
|
+
break
|
|
747
760
|
if stall_timeout and time.time() - stamp[0] > stall_timeout:
|
|
748
761
|
# 停滞看门狗:长静默多为卡死(网关挂起/CLI 假死),与其耗满总
|
|
749
762
|
# 超时不如提前杀——错误按超时归类,走既有的换模型/换将链路
|
|
@@ -752,22 +765,22 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
752
765
|
_kill_tree(proc.pid)
|
|
753
766
|
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
754
767
|
break
|
|
755
|
-
if aborts:
|
|
756
|
-
try:
|
|
757
|
-
recent = decode_output(b"".join(out_chunks)[-8000:] +
|
|
758
|
-
b"\n" + b"".join(err_chunks)[-8000:])
|
|
759
|
-
for marker, limit in aborts:
|
|
760
|
-
if recent.count(marker) >= limit:
|
|
761
|
-
repeat_aborted = True
|
|
762
|
-
abort_marker = marker
|
|
763
|
-
timed_out = True
|
|
764
|
-
_kill_tree(proc.pid)
|
|
765
|
-
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
766
|
-
break
|
|
767
|
-
if repeat_aborted:
|
|
768
|
-
break
|
|
769
|
-
except (TypeError, ValueError):
|
|
770
|
-
pass
|
|
768
|
+
if aborts:
|
|
769
|
+
try:
|
|
770
|
+
recent = decode_output(b"".join(out_chunks)[-8000:] +
|
|
771
|
+
b"\n" + b"".join(err_chunks)[-8000:])
|
|
772
|
+
for marker, limit in aborts:
|
|
773
|
+
if recent.count(marker) >= limit:
|
|
774
|
+
repeat_aborted = True
|
|
775
|
+
abort_marker = marker
|
|
776
|
+
timed_out = True
|
|
777
|
+
_kill_tree(proc.pid)
|
|
778
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
779
|
+
break
|
|
780
|
+
if repeat_aborted:
|
|
781
|
+
break
|
|
782
|
+
except (TypeError, ValueError):
|
|
783
|
+
pass
|
|
771
784
|
duration = round(time.time() - start, 1)
|
|
772
785
|
t_out.join(timeout=5)
|
|
773
786
|
t_err.join(timeout=5)
|
|
@@ -784,32 +797,32 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
784
797
|
pass
|
|
785
798
|
if cancelled:
|
|
786
799
|
stderr += "\n[已被用户取消]"
|
|
787
|
-
elif repeat_aborted:
|
|
788
|
-
limit = next((n for m, n in aborts if m == abort_marker), "?")
|
|
789
|
-
if len(aborts) == 1 and abort_markers is None:
|
|
790
|
-
# 保留旧版 UI/测试已经依赖的文案;多标记配置使用更具体的
|
|
791
|
-
# 标记名,方便定位究竟是哪一种断流触发了止损。
|
|
792
|
-
stderr += "\n[同一网络错误重复 %s 次,已提前终止进程树]" % limit
|
|
793
|
-
else:
|
|
794
|
-
stderr += "\n[错误标记 %s 重复 %s 次,已提前终止进程树]" % (
|
|
795
|
-
abort_marker, limit)
|
|
796
|
-
elif deadline_exceeded:
|
|
797
|
-
stderr += "\n[任务总时限已到,已终止进程树]"
|
|
800
|
+
elif repeat_aborted:
|
|
801
|
+
limit = next((n for m, n in aborts if m == abort_marker), "?")
|
|
802
|
+
if len(aborts) == 1 and abort_markers is None:
|
|
803
|
+
# 保留旧版 UI/测试已经依赖的文案;多标记配置使用更具体的
|
|
804
|
+
# 标记名,方便定位究竟是哪一种断流触发了止损。
|
|
805
|
+
stderr += "\n[同一网络错误重复 %s 次,已提前终止进程树]" % limit
|
|
806
|
+
else:
|
|
807
|
+
stderr += "\n[错误标记 %s 重复 %s 次,已提前终止进程树]" % (
|
|
808
|
+
abort_marker, limit)
|
|
809
|
+
elif deadline_exceeded:
|
|
810
|
+
stderr += "\n[任务总时限已到,已终止进程树]"
|
|
798
811
|
elif stalled:
|
|
799
812
|
stderr += "\n[输出停滞 %ss,已终止进程树]" % stall_timeout
|
|
800
813
|
elif timed_out:
|
|
801
814
|
stderr += "\n[超时 %ss,已终止进程树]" % timeout
|
|
802
|
-
return {
|
|
803
|
-
"ok": exit_code == 0 and not cancelled and not timed_out,
|
|
804
|
-
"exit_code": exit_code, "stdout": stdout, "stderr": stderr,
|
|
805
|
-
"duration": duration, "cancelled": cancelled, "timed_out": timed_out,
|
|
806
|
-
"stalled": stalled, "deadline_exceeded": deadline_exceeded,
|
|
807
|
-
"abort_marker": abort_marker,
|
|
808
|
-
"activity_extended": bool(activity_timeout and stamp[1]
|
|
809
|
-
and duration > timeout),
|
|
810
|
-
"activity_timeout": activity_timeout,
|
|
811
|
-
"last_output_age": max(0.0, time.time() - stamp[0]),
|
|
812
|
-
}
|
|
815
|
+
return {
|
|
816
|
+
"ok": exit_code == 0 and not cancelled and not timed_out,
|
|
817
|
+
"exit_code": exit_code, "stdout": stdout, "stderr": stderr,
|
|
818
|
+
"duration": duration, "cancelled": cancelled, "timed_out": timed_out,
|
|
819
|
+
"stalled": stalled, "deadline_exceeded": deadline_exceeded,
|
|
820
|
+
"abort_marker": abort_marker,
|
|
821
|
+
"activity_extended": bool(activity_timeout and stamp[1]
|
|
822
|
+
and duration > timeout),
|
|
823
|
+
"activity_timeout": activity_timeout,
|
|
824
|
+
"last_output_age": max(0.0, time.time() - stamp[0]),
|
|
825
|
+
}
|
|
813
826
|
finally:
|
|
814
827
|
if log_fh:
|
|
815
828
|
try:
|
|
@@ -886,8 +899,8 @@ def _codex_fail_msg(stdout):
|
|
|
886
899
|
2026-09-16 实测:配额/限流只出现在 error 与 turn.failed 事件里,旧解析器
|
|
887
900
|
两者都丢——进程退出码非 0 时错误串里只剩 "Reading prompt from stdin...",
|
|
888
901
|
_quota_error/_transient_error 判不出可降级,健康后继模型从未被尝试。
|
|
889
|
-
Reconnecting... 是 CLI 内部重试噪音(可能自愈),不取作终态错误;重复
|
|
890
|
-
重连由进程看门狗提前中断。turn.failed 优先于裸 error(后者取最后一条兜底)。
|
|
902
|
+
Reconnecting... 是 CLI 内部重试噪音(可能自愈),不取作终态错误;重复
|
|
903
|
+
重连由进程看门狗提前中断。turn.failed 优先于裸 error(后者取最后一条兜底)。
|
|
891
904
|
"""
|
|
892
905
|
terminal, last_err = "", ""
|
|
893
906
|
for line in (stdout or "").splitlines():
|
|
@@ -980,8 +993,8 @@ _TRANSIENT = ("503", "502", "529", "429", "no available channel", "temporarily",
|
|
|
980
993
|
# 2026-09-20 连载评审实测:codex 网关断流(stream disconnected)
|
|
981
994
|
# 与 opencode 服务端 500(Unexpected server error)都是「重试/换将
|
|
982
995
|
# 就可能活」的瞬态病,旧表判成终态导致整链早死
|
|
983
|
-
"stream disconnected", "stream closed before response.completed",
|
|
984
|
-
"unexpected server error",
|
|
996
|
+
"stream disconnected", "stream closed before response.completed",
|
|
997
|
+
"unexpected server error",
|
|
985
998
|
# 2026-09-22 mo-so 实案:claude 报 "Can't reach the API server —
|
|
986
999
|
# check your internet or DNS (ENOTFOUND)",一家端点解析不到不代表
|
|
987
1000
|
# 跨厂商链上别家也不通,旧表判成终态直接跳去换将别的 CLI
|
|
@@ -996,33 +1009,33 @@ def _transient_error(err):
|
|
|
996
1009
|
|
|
997
1010
|
# 欠费/配额/限流耗尽:换 KEY 与换厂商都该继续(同厂商另一账号往往还能用)。
|
|
998
1011
|
# 与 modelhub._QUOTA_HINTS 同源,这里独立一份避免 core 模块间循环依赖。
|
|
999
|
-
_QUOTA = ("insufficient", "quota", "balance", "credit", "billing", "arrears",
|
|
1000
|
-
"payment required", "402", "欠费", "余额", "额度", "exceeded",
|
|
1001
|
-
"too many requests", "rate limit", "rate_limit", "http 429",
|
|
1002
|
-
"并发", "超过限")
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
def _quota_error(err):
|
|
1006
|
-
err = (err or "").lower()
|
|
1007
|
-
return any(k in err for k in _QUOTA)
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
_AUTH = ("invalid api key", "invalid_api_key", "api key is invalid",
|
|
1011
|
-
"incorrect api key", "unauthorized", "unauthorised",
|
|
1012
|
-
"authentication failed", "authentication error", "authentication_error",
|
|
1013
|
-
"invalid token", "invalid_token", "http 401", "status code 401")
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
def _auth_error(err):
|
|
1017
|
-
"""只识别明确的凭据拒绝,不把一般模型权限错误(403)当成 KEY 错。"""
|
|
1018
|
-
err = (err or "").lower()
|
|
1019
|
-
return any(k in err for k in _AUTH)
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
def _attempt_credential(att):
|
|
1023
|
-
"""绑定链中凭据的非敏感身份;不把密钥值写入日志或尝试记录。"""
|
|
1024
|
-
return (str((att or {}).get("provider_id") or ""),
|
|
1025
|
-
str((att or {}).get("key_id") or ""))
|
|
1012
|
+
_QUOTA = ("insufficient", "quota", "balance", "credit", "billing", "arrears",
|
|
1013
|
+
"payment required", "402", "欠费", "余额", "额度", "exceeded",
|
|
1014
|
+
"too many requests", "rate limit", "rate_limit", "http 429",
|
|
1015
|
+
"并发", "超过限")
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def _quota_error(err):
|
|
1019
|
+
err = (err or "").lower()
|
|
1020
|
+
return any(k in err for k in _QUOTA)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
_AUTH = ("invalid api key", "invalid_api_key", "api key is invalid",
|
|
1024
|
+
"incorrect api key", "unauthorized", "unauthorised",
|
|
1025
|
+
"authentication failed", "authentication error", "authentication_error",
|
|
1026
|
+
"invalid token", "invalid_token", "http 401", "status code 401")
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def _auth_error(err):
|
|
1030
|
+
"""只识别明确的凭据拒绝,不把一般模型权限错误(403)当成 KEY 错。"""
|
|
1031
|
+
err = (err or "").lower()
|
|
1032
|
+
return any(k in err for k in _AUTH)
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def _attempt_credential(att):
|
|
1036
|
+
"""绑定链中凭据的非敏感身份;不把密钥值写入日志或尝试记录。"""
|
|
1037
|
+
return (str((att or {}).get("provider_id") or ""),
|
|
1038
|
+
str((att or {}).get("key_id") or ""))
|
|
1026
1039
|
|
|
1027
1040
|
|
|
1028
1041
|
# 限流(429)专项:与一般瞬态不同,它是「等一个窗口就常能自愈」的病——
|
|
@@ -1048,15 +1061,15 @@ def _refusal_error(err):
|
|
|
1048
1061
|
return any(k in err for k in _REFUSAL)
|
|
1049
1062
|
|
|
1050
1063
|
|
|
1051
|
-
def _attempt_upstream(att):
|
|
1052
|
-
"""返回尝试项的上游身份;同一 host 换模型/供应商别名不重复烧超时。"""
|
|
1053
|
-
provider = (att or {}).get("provider") or {}
|
|
1054
|
-
base_url = str(provider.get("base_url") or "").strip()
|
|
1055
|
-
match = re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://([^/?#]+)", base_url)
|
|
1056
|
-
if match:
|
|
1057
|
-
return match.group(1).lower()
|
|
1058
|
-
provider_id = str((att or {}).get("provider_id") or "").strip()
|
|
1059
|
-
return "provider:" + provider_id if provider_id else "unknown"
|
|
1064
|
+
def _attempt_upstream(att):
|
|
1065
|
+
"""返回尝试项的上游身份;同一 host 换模型/供应商别名不重复烧超时。"""
|
|
1066
|
+
provider = (att or {}).get("provider") or {}
|
|
1067
|
+
base_url = str(provider.get("base_url") or "").strip()
|
|
1068
|
+
match = re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://([^/?#]+)", base_url)
|
|
1069
|
+
if match:
|
|
1070
|
+
return match.group(1).lower()
|
|
1071
|
+
provider_id = str((att or {}).get("provider_id") or "").strip()
|
|
1072
|
+
return "provider:" + provider_id if provider_id else "unknown"
|
|
1060
1073
|
|
|
1061
1074
|
|
|
1062
1075
|
def _claude_drift_note(att_model, parsed):
|
|
@@ -1083,11 +1096,11 @@ def _rate_limited(err):
|
|
|
1083
1096
|
|
|
1084
1097
|
|
|
1085
1098
|
# 审计日志错误行探针:只认强信号,避免把回显提示词里的普通词当错误
|
|
1086
|
-
_LOG_ERR_HINT = re.compile(
|
|
1087
|
-
r"(429|402|rate[_ ]?limit|quota|余额|欠费|insufficient|billing|arrears"
|
|
1088
|
-
r"|401|unauthorized|unauthorised|invalid[_ ]api[_ ]key|authentication[_ ]error"
|
|
1089
|
-
r"|ENOTFOUND|ECONNREFUSED|failed to run prompt|api_error|overloaded)",
|
|
1090
|
-
re.I)
|
|
1099
|
+
_LOG_ERR_HINT = re.compile(
|
|
1100
|
+
r"(429|402|rate[_ ]?limit|quota|余额|欠费|insufficient|billing|arrears"
|
|
1101
|
+
r"|401|unauthorized|unauthorised|invalid[_ ]api[_ ]key|authentication[_ ]error"
|
|
1102
|
+
r"|ENOTFOUND|ECONNREFUSED|failed to run prompt|api_error|overloaded)",
|
|
1103
|
+
re.I)
|
|
1091
1104
|
|
|
1092
1105
|
|
|
1093
1106
|
def _augment_error_from_log(out, log_path, limit=300):
|
|
@@ -1110,7 +1123,7 @@ def _augment_error_from_log(out, log_path, limit=300):
|
|
|
1110
1123
|
out["error"] = (out["error"] + ";日志错误行: " + hits[-1][:limit])[-800:]
|
|
1111
1124
|
|
|
1112
1125
|
|
|
1113
|
-
def _grace_wait(cancel_event, seconds, log_path=None, why=""):
|
|
1126
|
+
def _grace_wait(cancel_event, seconds, log_path=None, why=""):
|
|
1114
1127
|
"""链尾限流宽限等待:小片睡眠随时响应取消。返回 False=已取消(别再重试)。"""
|
|
1115
1128
|
if log_path:
|
|
1116
1129
|
try:
|
|
@@ -1119,11 +1132,11 @@ def _grace_wait(cancel_event, seconds, log_path=None, why=""):
|
|
|
1119
1132
|
% (why or "上游限流(429)", seconds))
|
|
1120
1133
|
except Exception:
|
|
1121
1134
|
pass
|
|
1122
|
-
end = time.monotonic() + seconds
|
|
1123
|
-
while time.monotonic() < end:
|
|
1124
|
-
if cancel_event is not None and cancel_event.is_set():
|
|
1125
|
-
return False
|
|
1126
|
-
time.sleep(min(3, max(0.5, end - time.monotonic())))
|
|
1135
|
+
end = time.monotonic() + seconds
|
|
1136
|
+
while time.monotonic() < end:
|
|
1137
|
+
if cancel_event is not None and cancel_event.is_set():
|
|
1138
|
+
return False
|
|
1139
|
+
time.sleep(min(3, max(0.5, end - time.monotonic())))
|
|
1127
1140
|
return cancel_event is None or not cancel_event.is_set()
|
|
1128
1141
|
|
|
1129
1142
|
|
|
@@ -1240,11 +1253,11 @@ def _prompt_to_file(prompt, workdir):
|
|
|
1240
1253
|
return None
|
|
1241
1254
|
|
|
1242
1255
|
|
|
1243
|
-
# Windows 上不少 CLI 先经过 cmd.exe/.cmd 垫片,实际安全上限接近 8191,
|
|
1244
|
-
# 而不是 CreateProcess 的理论 32767。留出命令、路径和环境参数余量,超过
|
|
1245
|
-
# 该值统一落盘,避免 MiMo/Kimi 评审在启动前直接报「命令行太长」。
|
|
1246
|
-
_ARGV_PROMPT_SAFE = 20000
|
|
1247
|
-
_ARGV_PROMPT_SHIM_SAFE = 6000
|
|
1256
|
+
# Windows 上不少 CLI 先经过 cmd.exe/.cmd 垫片,实际安全上限接近 8191,
|
|
1257
|
+
# 而不是 CreateProcess 的理论 32767。留出命令、路径和环境参数余量,超过
|
|
1258
|
+
# 该值统一落盘,避免 MiMo/Kimi 评审在启动前直接报「命令行太长」。
|
|
1259
|
+
_ARGV_PROMPT_SAFE = 20000
|
|
1260
|
+
_ARGV_PROMPT_SHIM_SAFE = 6000
|
|
1248
1261
|
|
|
1249
1262
|
|
|
1250
1263
|
def _build_call(agent, kind, sid, readonly, model, prompt, images=None, workdir=None):
|
|
@@ -1382,27 +1395,27 @@ def _build_call(agent, kind, sid, readonly, model, prompt, images=None, workdir=
|
|
|
1382
1395
|
# 超长时落盘临时文件、参数位换成读文件指令——评审/作者 CLI 非交互模式均带
|
|
1383
1396
|
# 读文件工具(kimi 0.43 实测可读),读不到的调用输出不可解析,由评审全挂
|
|
1384
1397
|
# 防线兜底判失败,绝不静默降级。
|
|
1385
|
-
prompt_safe = _ARGV_PROMPT_SAFE
|
|
1386
|
-
if argv:
|
|
1387
|
-
launcher = str(argv[0]).lower()
|
|
1388
|
-
if (launcher.endswith((".cmd", ".bat")) or
|
|
1389
|
-
os.path.basename(launcher) in ("npm", "npx", "pnpm", "yarn", "bun")):
|
|
1390
|
-
prompt_safe = _ARGV_PROMPT_SHIM_SAFE
|
|
1391
|
-
if argv is not None and stdin_text is None and len(prompt) >= prompt_safe \
|
|
1392
|
-
and argv.count(prompt) == 1:
|
|
1398
|
+
prompt_safe = _ARGV_PROMPT_SAFE
|
|
1399
|
+
if argv:
|
|
1400
|
+
launcher = str(argv[0]).lower()
|
|
1401
|
+
if (launcher.endswith((".cmd", ".bat")) or
|
|
1402
|
+
os.path.basename(launcher) in ("npm", "npx", "pnpm", "yarn", "bun")):
|
|
1403
|
+
prompt_safe = _ARGV_PROMPT_SHIM_SAFE
|
|
1404
|
+
if argv is not None and stdin_text is None and len(prompt) >= prompt_safe \
|
|
1405
|
+
and argv.count(prompt) == 1:
|
|
1393
1406
|
pf = _prompt_to_file(prompt, workdir)
|
|
1394
|
-
if pf:
|
|
1395
|
-
argv[argv.index(prompt)] = (
|
|
1396
|
-
"[系统] 本次完整指令因命令行长度限制已写入文件:%s\n"
|
|
1397
|
-
"请先用读文件工具完整读取该文件,然后把文件内容当作你的任务指令执行。" % pf)
|
|
1398
|
-
# Keep the original payload on stdin as a compatibility path for
|
|
1399
|
-
# simple generic adapters and test shims that read stdin. CLI tools
|
|
1400
|
-
# that intentionally use argv still receive the bounded file hint.
|
|
1401
|
-
if kind == "generic" and stdin_text is None and len(tmpl) > 1:
|
|
1402
|
-
# 模板明确将提示词放进 argv 时,原文只保存在文件里;同时再
|
|
1403
|
-
# 写入 stdin 会让部分包装器重复读取,也无法解决 Windows 命令行上限。
|
|
1404
|
-
stdin_text = None
|
|
1405
|
-
tmp_files.append(pf)
|
|
1407
|
+
if pf:
|
|
1408
|
+
argv[argv.index(prompt)] = (
|
|
1409
|
+
"[系统] 本次完整指令因命令行长度限制已写入文件:%s\n"
|
|
1410
|
+
"请先用读文件工具完整读取该文件,然后把文件内容当作你的任务指令执行。" % pf)
|
|
1411
|
+
# Keep the original payload on stdin as a compatibility path for
|
|
1412
|
+
# simple generic adapters and test shims that read stdin. CLI tools
|
|
1413
|
+
# that intentionally use argv still receive the bounded file hint.
|
|
1414
|
+
if kind == "generic" and stdin_text is None and len(tmpl) > 1:
|
|
1415
|
+
# 模板明确将提示词放进 argv 时,原文只保存在文件里;同时再
|
|
1416
|
+
# 写入 stdin 会让部分包装器重复读取,也无法解决 Windows 命令行上限。
|
|
1417
|
+
stdin_text = None
|
|
1418
|
+
tmp_files.append(pf)
|
|
1406
1419
|
return argv, stdin_text, prompt, tmp_files
|
|
1407
1420
|
|
|
1408
1421
|
|
|
@@ -1426,50 +1439,50 @@ def _check_approval(agent):
|
|
|
1426
1439
|
return False, "sensitive step requires policy=ask UI flow (Phase 5) — currently blocking"
|
|
1427
1440
|
|
|
1428
1441
|
|
|
1429
|
-
def _stall_timeout(env_name, default):
|
|
1442
|
+
def _stall_timeout(env_name, default):
|
|
1430
1443
|
"""停滞看门狗秒数:stdout/stderr 持续无新输出超过该时长 → 判卡死提前杀,
|
|
1431
1444
|
错误按超时归类走既有换模型/换将链路。只对事件流持续流动的 CLI 启用
|
|
1432
1445
|
(codex JSONL / claude stream-json);qwen/opencode/generic 结束才一次性
|
|
1433
1446
|
输出,开了会误杀正常长任务,仍靠总超时兜底。env 可调,0=关闭。"""
|
|
1434
1447
|
try:
|
|
1435
1448
|
return max(0, int(os.environ.get(env_name, str(default))))
|
|
1436
|
-
except Exception:
|
|
1437
|
-
return default
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
def _activity_timeout(env_name, default=DEFAULT_STREAM_ACTIVITY_TIMEOUT_S):
|
|
1441
|
-
"""流式 CLI 的活动延长窗口,独立于停滞看门狗配置。"""
|
|
1442
|
-
try:
|
|
1443
|
-
return max(0, int(os.environ.get(env_name, str(default))))
|
|
1444
|
-
except Exception:
|
|
1445
|
-
return default
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
def _max_model_attempts(has_deadline=False):
|
|
1449
|
-
"""返回一次 runner 调用允许消耗的候选数。
|
|
1450
|
-
|
|
1451
|
-
生产任务会传入统一 deadline,默认收敛到 3 个候选;没有 deadline 的
|
|
1452
|
-
旧式直接调用保留最多 4 个候选,以兼容已有的「同一上游两模型后换
|
|
1453
|
-
供应商」策略。运维可用环境变量进一步收紧/放宽,最少 1 个。
|
|
1454
|
-
"""
|
|
1455
|
-
default = 3 if has_deadline else 4
|
|
1456
|
-
try:
|
|
1457
|
-
value = int(os.environ.get("TUTTI_MAX_MODEL_ATTEMPTS", str(default)))
|
|
1458
|
-
return max(1, min(12, value))
|
|
1459
|
-
except (TypeError, ValueError):
|
|
1460
|
-
return default
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
_INVALID_MODEL_NAMES = frozenset(("auto", "default", "none"))
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
def _invalid_model(model):
|
|
1467
|
-
return str(model or "").strip().lower() in _INVALID_MODEL_NAMES
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
1471
|
-
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None, resume=None,
|
|
1472
|
-
images=None, require_tools=False, deadline=None):
|
|
1449
|
+
except Exception:
|
|
1450
|
+
return default
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _activity_timeout(env_name, default=DEFAULT_STREAM_ACTIVITY_TIMEOUT_S):
|
|
1454
|
+
"""流式 CLI 的活动延长窗口,独立于停滞看门狗配置。"""
|
|
1455
|
+
try:
|
|
1456
|
+
return max(0, int(os.environ.get(env_name, str(default))))
|
|
1457
|
+
except Exception:
|
|
1458
|
+
return default
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def _max_model_attempts(has_deadline=False):
|
|
1462
|
+
"""返回一次 runner 调用允许消耗的候选数。
|
|
1463
|
+
|
|
1464
|
+
生产任务会传入统一 deadline,默认收敛到 3 个候选;没有 deadline 的
|
|
1465
|
+
旧式直接调用保留最多 4 个候选,以兼容已有的「同一上游两模型后换
|
|
1466
|
+
供应商」策略。运维可用环境变量进一步收紧/放宽,最少 1 个。
|
|
1467
|
+
"""
|
|
1468
|
+
default = 3 if has_deadline else 4
|
|
1469
|
+
try:
|
|
1470
|
+
value = int(os.environ.get("TUTTI_MAX_MODEL_ATTEMPTS", str(default)))
|
|
1471
|
+
return max(1, min(12, value))
|
|
1472
|
+
except (TypeError, ValueError):
|
|
1473
|
+
return default
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
_INVALID_MODEL_NAMES = frozenset(("auto", "default", "none"))
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def _invalid_model(model):
|
|
1480
|
+
return str(model or "").strip().lower() in _INVALID_MODEL_NAMES
|
|
1481
|
+
|
|
1482
|
+
|
|
1483
|
+
def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
1484
|
+
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None, resume=None,
|
|
1485
|
+
images=None, require_tools=False, deadline=None):
|
|
1473
1486
|
"""执行一次智能体调用,返回统一结构
|
|
1474
1487
|
{ok, text, json, cost_usd, tokens, error, error_code, raw}。
|
|
1475
1488
|
agent 来自 registry.effective_agents();resume 为已有会话 id,仅真实智能体生效
|
|
@@ -1483,66 +1496,65 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1483
1496
|
|
|
1484
1497
|
模型尝试顺序来自 _resolve_attempts:跨厂商链(每条独立 env)或
|
|
1485
1498
|
主模型 + 降级备选;瞬态/配额/超时/上游拒答换下一条,取消与解析
|
|
1486
|
-
失败不降级;同一上游超时两次即跳过该上游剩余模型。
|
|
1487
|
-
|
|
1488
|
-
5E:catalog `orch.timeout_ms`(毫秒)优先于 caller 传入的 timeout;
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
任务共享的 ``time.monotonic()`` 绝对截止时刻;传入后所有模型、空响应
|
|
1492
|
-
重试和限流宽限共用同一剩余预算。
|
|
1493
|
-
"""
|
|
1499
|
+
失败不降级;同一上游超时两次即跳过该上游剩余模型。
|
|
1500
|
+
|
|
1501
|
+
5E:catalog `orch.timeout_ms`(毫秒)优先于 caller 传入的 timeout;
|
|
1502
|
+
单模型时限由该配置或 caller timeout 决定,任务 deadline 可进一步收紧;
|
|
1503
|
+
持续 stream 活动交由 stall_timeout 看门狗收口。deadline 是
|
|
1504
|
+
任务共享的 ``time.monotonic()`` 绝对截止时刻;传入后所有模型、空响应
|
|
1505
|
+
重试和限流宽限共用同一剩余预算。
|
|
1506
|
+
"""
|
|
1494
1507
|
# 5E:catalog orch.timeout_ms 优先
|
|
1495
1508
|
orch_timeout_ms = (agent.get("orch") or {}).get("timeout_ms")
|
|
1496
|
-
if orch_timeout_ms:
|
|
1497
|
-
timeout = float(orch_timeout_ms) / 1000.0
|
|
1498
|
-
try:
|
|
1499
|
-
timeout = float(timeout)
|
|
1500
|
-
except (TypeError, ValueError):
|
|
1501
|
-
timeout = float(DEFAULT_TIMEOUT)
|
|
1502
|
-
timeout = max(0.0, timeout)
|
|
1503
|
-
try:
|
|
1504
|
-
deadline = float(deadline) if deadline is not None else None
|
|
1505
|
-
except (TypeError, ValueError):
|
|
1506
|
-
deadline = None
|
|
1507
|
-
if deadline is not None and time.monotonic() >= deadline:
|
|
1508
|
-
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1509
|
-
"tokens": 0, "usage": None, "error": "任务总时限已到,未启动模型",
|
|
1510
|
-
"error_code": ErrorCode.TIMEOUT,
|
|
1511
|
-
"raw": {"exit_code": None, "timed_out": True,
|
|
1512
|
-
"deadline_exceeded": True, "duration": 0.0},
|
|
1513
|
-
"kind": agent.get("kind", "generic"), "model": None,
|
|
1514
|
-
"attempts": []}
|
|
1515
|
-
timeout = min(timeout, MAX_ATTEMPT_TIMEOUT_S)
|
|
1509
|
+
if orch_timeout_ms:
|
|
1510
|
+
timeout = float(orch_timeout_ms) / 1000.0
|
|
1511
|
+
try:
|
|
1512
|
+
timeout = float(timeout)
|
|
1513
|
+
except (TypeError, ValueError):
|
|
1514
|
+
timeout = float(DEFAULT_TIMEOUT)
|
|
1515
|
+
timeout = max(0.0, timeout)
|
|
1516
|
+
try:
|
|
1517
|
+
deadline = float(deadline) if deadline is not None else None
|
|
1518
|
+
except (TypeError, ValueError):
|
|
1519
|
+
deadline = None
|
|
1520
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1521
|
+
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1522
|
+
"tokens": 0, "usage": None, "error": "任务总时限已到,未启动模型",
|
|
1523
|
+
"error_code": ErrorCode.TIMEOUT,
|
|
1524
|
+
"raw": {"exit_code": None, "timed_out": True,
|
|
1525
|
+
"deadline_exceeded": True, "duration": 0.0},
|
|
1526
|
+
"kind": agent.get("kind", "generic"), "model": None,
|
|
1527
|
+
"attempts": []}
|
|
1516
1528
|
kind = agent.get("kind", "generic")
|
|
1517
1529
|
if kind == "aider":
|
|
1518
1530
|
repo_issue = _git_repo_issue(workdir)
|
|
1519
1531
|
if repo_issue:
|
|
1520
1532
|
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1521
|
-
"tokens": 0, "usage": None, "error": repo_issue,
|
|
1522
|
-
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1523
|
-
"kind": kind, "model": agent.get("model"), "attempts": []}
|
|
1524
|
-
if kind == "codex":
|
|
1525
|
-
stall_t = _stall_timeout("TUTTI_CODEX_STALL_TIMEOUT", 600)
|
|
1526
|
-
activity_t = _activity_timeout("TUTTI_CODEX_ACTIVITY_TIMEOUT")
|
|
1527
|
-
elif kind == "claude":
|
|
1528
|
-
stall_t = _stall_timeout("TUTTI_CLAUDE_STALL_TIMEOUT", 600)
|
|
1529
|
-
activity_t = _activity_timeout("TUTTI_CLAUDE_ACTIVITY_TIMEOUT")
|
|
1530
|
-
else:
|
|
1533
|
+
"tokens": 0, "usage": None, "error": repo_issue,
|
|
1534
|
+
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1535
|
+
"kind": kind, "model": agent.get("model"), "attempts": []}
|
|
1536
|
+
if kind == "codex":
|
|
1537
|
+
stall_t = _stall_timeout("TUTTI_CODEX_STALL_TIMEOUT", 600)
|
|
1538
|
+
activity_t = _activity_timeout("TUTTI_CODEX_ACTIVITY_TIMEOUT")
|
|
1539
|
+
elif kind == "claude":
|
|
1540
|
+
stall_t = _stall_timeout("TUTTI_CLAUDE_STALL_TIMEOUT", 600)
|
|
1541
|
+
activity_t = _activity_timeout("TUTTI_CLAUDE_ACTIVITY_TIMEOUT")
|
|
1542
|
+
else:
|
|
1531
1543
|
# 数据驱动:catalog orch.stall_timeout_s——给 kimi/qwen 这类评审用
|
|
1532
1544
|
# CLI 配置后,静默挂死即杀(qwen 900s:2026-09-22 MCP 收尾死锁案);
|
|
1533
1545
|
# mimo 等结束才一次性输出的仍留 0(开了必误杀),靠总超时兜底
|
|
1534
1546
|
try:
|
|
1535
1547
|
stall_t = max(0, int((agent.get("orch") or {}).get("stall_timeout_s") or 0))
|
|
1536
|
-
except Exception:
|
|
1537
|
-
stall_t = 0
|
|
1538
|
-
activity_t = 0
|
|
1548
|
+
except Exception:
|
|
1549
|
+
stall_t = 0
|
|
1550
|
+
activity_t = 0
|
|
1539
1551
|
# 5G:approval NEVER 一线(无人值守不静默降级)
|
|
1540
1552
|
ok, reason = _check_approval(agent)
|
|
1541
1553
|
if not ok:
|
|
1542
1554
|
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0, "tokens": 0,
|
|
1543
|
-
"usage": None, "error": reason,
|
|
1544
|
-
"error_code": ErrorCode.ENV_BLOCK,
|
|
1545
|
-
"raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1555
|
+
"usage": None, "error": reason,
|
|
1556
|
+
"error_code": ErrorCode.ENV_BLOCK,
|
|
1557
|
+
"raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1546
1558
|
base_env = dict(agent.get("env") or {})
|
|
1547
1559
|
if kind == "claude":
|
|
1548
1560
|
bash = find_git_bash()
|
|
@@ -1555,64 +1567,64 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1555
1567
|
"usage": None,
|
|
1556
1568
|
"error": "该 CLI 未配置会话恢复(catalog orch.resume_argv_template),"
|
|
1557
1569
|
"无法在已有会话上继续",
|
|
1558
|
-
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1559
|
-
"sid": "", "raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1560
|
-
|
|
1561
|
-
attempts = _resolve_attempts(agent)
|
|
1562
|
-
# 启动前过滤明显的占位模型名。它们常由某个 CLI 的会话标题/默认配置
|
|
1563
|
-
# 泄漏进编排链;让子进程自己等待网关返回会把一个确定性配置错拖满
|
|
1564
|
-
# 多轮超时。非法项不消耗模型调用预算,后面的真实候选仍可接手。
|
|
1565
|
-
filtered_attempts = []
|
|
1566
|
-
preflight_failures = []
|
|
1567
|
-
for att in attempts:
|
|
1568
|
-
if _invalid_model(att.get("model")):
|
|
1569
|
-
preflight_failures.append({
|
|
1570
|
-
"model": att.get("model"),
|
|
1571
|
-
"provider_id": att.get("provider_id") or "",
|
|
1572
|
-
"provider": (att.get("provider") or {}).get("name", "")
|
|
1573
|
-
if isinstance(att.get("provider"), dict) else "",
|
|
1574
|
-
"ok": False, "skipped": True, "error":
|
|
1575
|
-
"无效模型名 %r,启动前跳过" % att.get("model"),
|
|
1576
|
-
})
|
|
1577
|
-
continue
|
|
1578
|
-
filtered_attempts.append(att)
|
|
1579
|
-
attempts = filtered_attempts[:_max_model_attempts(deadline is not None)]
|
|
1580
|
-
out = None
|
|
1581
|
-
attempt_history = list(preflight_failures)
|
|
1582
|
-
if not attempts:
|
|
1583
|
-
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1584
|
-
"tokens": 0, "usage": None,
|
|
1585
|
-
"error": (preflight_failures[-1].get("error")
|
|
1586
|
-
if preflight_failures else "没有可尝试的模型"),
|
|
1587
|
-
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1588
|
-
"kind": kind, "model": None, "attempts": attempt_history}
|
|
1589
|
-
grace_left = RATE_LIMIT_GRACE_N # 链尾限流宽限预算:整链撞 429 时原地等一个窗口再试一次
|
|
1590
|
-
timed_out_upstreams = {} # 同一上游两次超时后跳过剩余模型
|
|
1591
|
-
skipped_upstreams = set()
|
|
1592
|
-
auth_failed_credentials = set()
|
|
1593
|
-
for ai, att in enumerate(attempts):
|
|
1594
|
-
credential = _attempt_credential(att)
|
|
1595
|
-
if credential in auth_failed_credentials:
|
|
1596
|
-
attempt_history.append({
|
|
1597
|
-
"model": att.get("model") or "",
|
|
1598
|
-
"provider_id": att.get("provider_id") or "",
|
|
1599
|
-
"provider": ((att.get("provider") or {}).get("name", "")
|
|
1600
|
-
if isinstance(att.get("provider"), dict) else ""),
|
|
1601
|
-
"kind": kind, "ok": False, "skipped": True,
|
|
1602
|
-
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1603
|
-
"error": "同一凭据已被认证拒绝,跳过该凭据的其他模型",
|
|
1604
|
-
})
|
|
1605
|
-
if out is not None:
|
|
1606
|
-
out["attempts"] = list(attempt_history)
|
|
1607
|
-
continue
|
|
1608
|
-
upstream = _attempt_upstream(att)
|
|
1609
|
-
if upstream in skipped_upstreams:
|
|
1610
|
-
continue
|
|
1611
|
-
# 有任务 deadline
|
|
1612
|
-
#
|
|
1613
|
-
model_deadline = (att.get("_deadline") or deadline or
|
|
1614
|
-
(time.monotonic() + timeout))
|
|
1615
|
-
env = dict(base_env)
|
|
1570
|
+
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1571
|
+
"sid": "", "raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1572
|
+
|
|
1573
|
+
attempts = _resolve_attempts(agent)
|
|
1574
|
+
# 启动前过滤明显的占位模型名。它们常由某个 CLI 的会话标题/默认配置
|
|
1575
|
+
# 泄漏进编排链;让子进程自己等待网关返回会把一个确定性配置错拖满
|
|
1576
|
+
# 多轮超时。非法项不消耗模型调用预算,后面的真实候选仍可接手。
|
|
1577
|
+
filtered_attempts = []
|
|
1578
|
+
preflight_failures = []
|
|
1579
|
+
for att in attempts:
|
|
1580
|
+
if _invalid_model(att.get("model")):
|
|
1581
|
+
preflight_failures.append({
|
|
1582
|
+
"model": att.get("model"),
|
|
1583
|
+
"provider_id": att.get("provider_id") or "",
|
|
1584
|
+
"provider": (att.get("provider") or {}).get("name", "")
|
|
1585
|
+
if isinstance(att.get("provider"), dict) else "",
|
|
1586
|
+
"ok": False, "skipped": True, "error":
|
|
1587
|
+
"无效模型名 %r,启动前跳过" % att.get("model"),
|
|
1588
|
+
})
|
|
1589
|
+
continue
|
|
1590
|
+
filtered_attempts.append(att)
|
|
1591
|
+
attempts = filtered_attempts[:_max_model_attempts(deadline is not None)]
|
|
1592
|
+
out = None
|
|
1593
|
+
attempt_history = list(preflight_failures)
|
|
1594
|
+
if not attempts:
|
|
1595
|
+
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1596
|
+
"tokens": 0, "usage": None,
|
|
1597
|
+
"error": (preflight_failures[-1].get("error")
|
|
1598
|
+
if preflight_failures else "没有可尝试的模型"),
|
|
1599
|
+
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1600
|
+
"kind": kind, "model": None, "attempts": attempt_history}
|
|
1601
|
+
grace_left = RATE_LIMIT_GRACE_N # 链尾限流宽限预算:整链撞 429 时原地等一个窗口再试一次
|
|
1602
|
+
timed_out_upstreams = {} # 同一上游两次超时后跳过剩余模型
|
|
1603
|
+
skipped_upstreams = set()
|
|
1604
|
+
auth_failed_credentials = set()
|
|
1605
|
+
for ai, att in enumerate(attempts):
|
|
1606
|
+
credential = _attempt_credential(att)
|
|
1607
|
+
if credential in auth_failed_credentials:
|
|
1608
|
+
attempt_history.append({
|
|
1609
|
+
"model": att.get("model") or "",
|
|
1610
|
+
"provider_id": att.get("provider_id") or "",
|
|
1611
|
+
"provider": ((att.get("provider") or {}).get("name", "")
|
|
1612
|
+
if isinstance(att.get("provider"), dict) else ""),
|
|
1613
|
+
"kind": kind, "ok": False, "skipped": True,
|
|
1614
|
+
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1615
|
+
"error": "同一凭据已被认证拒绝,跳过该凭据的其他模型",
|
|
1616
|
+
})
|
|
1617
|
+
if out is not None:
|
|
1618
|
+
out["attempts"] = list(attempt_history)
|
|
1619
|
+
continue
|
|
1620
|
+
upstream = _attempt_upstream(att)
|
|
1621
|
+
if upstream in skipped_upstreams:
|
|
1622
|
+
continue
|
|
1623
|
+
# 有任务 deadline 时,所有候选共享同一个截止时刻;否则每个候选
|
|
1624
|
+
# 使用 catalog 配置或 caller 提供的单模型时限。
|
|
1625
|
+
model_deadline = (att.get("_deadline") or deadline or
|
|
1626
|
+
(time.monotonic() + timeout))
|
|
1627
|
+
env = dict(base_env)
|
|
1616
1628
|
env.update(att["env"])
|
|
1617
1629
|
if kind == "codex":
|
|
1618
1630
|
# 网关自定义模型名(如 [opencode]xxx)是合法 API 模型名不能改,但方括号
|
|
@@ -1626,40 +1638,40 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1626
1638
|
elif att["from_chain"] and "codex_provider" in eff_agent:
|
|
1627
1639
|
# 链内条目未注入供应商时不能沿用上一条(可能是另一家厂商)的 -c 覆盖
|
|
1628
1640
|
del eff_agent["codex_provider"]
|
|
1629
|
-
for attempt in range(2): # claude 偶发空响应(0 token)自动重试一次
|
|
1630
|
-
remaining = model_deadline - time.monotonic()
|
|
1631
|
-
if remaining <= 0:
|
|
1632
|
-
res = {"ok": False, "exit_code": None, "stdout": "", "stderr": "",
|
|
1633
|
-
"duration": timeout, "cancelled": False, "timed_out": True,
|
|
1634
|
-
"stalled": False,
|
|
1635
|
-
"deadline_exceeded": deadline is not None}
|
|
1636
|
-
else:
|
|
1637
|
-
argv, stdin_text, prompt_eff, tmp_files = _build_call(
|
|
1638
|
-
eff_agent, kind, sid, readonly,
|
|
1639
|
-
att["model"], prompt,
|
|
1640
|
-
images=images if kind == "codex" else None,
|
|
1641
|
-
workdir=workdir)
|
|
1642
|
-
try:
|
|
1643
|
-
repeat_guard = (("Reconnecting...", 2)
|
|
1644
|
-
if kind == "codex" else None)
|
|
1645
|
-
stream_abort_markers = (
|
|
1646
|
-
(("stream disconnected", 1),
|
|
1647
|
-
("stream closed before response.completed", 1))
|
|
1648
|
-
if kind == "codex" else None)
|
|
1649
|
-
res = run_process(argv=argv, stdin_text=stdin_text, cwd=workdir, env=env,
|
|
1650
|
-
timeout=min(timeout, remaining), deadline=deadline,
|
|
1651
|
-
cancel_event=cancel_event,
|
|
1652
|
-
log_path=log_path, stall_timeout=stall_t,
|
|
1653
|
-
activity_timeout=activity_t,
|
|
1654
|
-
repeat_abort=repeat_guard,
|
|
1655
|
-
abort_markers=stream_abort_markers)
|
|
1656
|
-
finally:
|
|
1657
|
-
# 超长指令临时文件:CLI 进程已结束(管道已收),即刻清场不污染工作目录
|
|
1658
|
-
for tf in tmp_files:
|
|
1659
|
-
try:
|
|
1660
|
-
os.remove(tf)
|
|
1661
|
-
except OSError:
|
|
1662
|
-
pass
|
|
1641
|
+
for attempt in range(2): # claude 偶发空响应(0 token)自动重试一次
|
|
1642
|
+
remaining = model_deadline - time.monotonic()
|
|
1643
|
+
if remaining <= 0:
|
|
1644
|
+
res = {"ok": False, "exit_code": None, "stdout": "", "stderr": "",
|
|
1645
|
+
"duration": timeout, "cancelled": False, "timed_out": True,
|
|
1646
|
+
"stalled": False,
|
|
1647
|
+
"deadline_exceeded": deadline is not None}
|
|
1648
|
+
else:
|
|
1649
|
+
argv, stdin_text, prompt_eff, tmp_files = _build_call(
|
|
1650
|
+
eff_agent, kind, sid, readonly,
|
|
1651
|
+
att["model"], prompt,
|
|
1652
|
+
images=images if kind == "codex" else None,
|
|
1653
|
+
workdir=workdir)
|
|
1654
|
+
try:
|
|
1655
|
+
repeat_guard = (("Reconnecting...", 2)
|
|
1656
|
+
if kind == "codex" else None)
|
|
1657
|
+
stream_abort_markers = (
|
|
1658
|
+
(("stream disconnected", 1),
|
|
1659
|
+
("stream closed before response.completed", 1))
|
|
1660
|
+
if kind == "codex" else None)
|
|
1661
|
+
res = run_process(argv=argv, stdin_text=stdin_text, cwd=workdir, env=env,
|
|
1662
|
+
timeout=min(timeout, remaining), deadline=deadline,
|
|
1663
|
+
cancel_event=cancel_event,
|
|
1664
|
+
log_path=log_path, stall_timeout=stall_t,
|
|
1665
|
+
activity_timeout=activity_t,
|
|
1666
|
+
repeat_abort=repeat_guard,
|
|
1667
|
+
abort_markers=stream_abort_markers)
|
|
1668
|
+
finally:
|
|
1669
|
+
# 超长指令临时文件:CLI 进程已结束(管道已收),即刻清场不污染工作目录
|
|
1670
|
+
for tf in tmp_files:
|
|
1671
|
+
try:
|
|
1672
|
+
os.remove(tf)
|
|
1673
|
+
except OSError:
|
|
1674
|
+
pass
|
|
1663
1675
|
out = {"ok": res["ok"], "text": "", "json": None, "cost_usd": 0.0,
|
|
1664
1676
|
"tokens": 0, "usage": None, "error": "", "error_code": "",
|
|
1665
1677
|
"sid": "", "raw": res, "kind": kind, "model": att["model"],
|
|
@@ -1677,9 +1689,9 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1677
1689
|
else "超时" if res["timed_out"]
|
|
1678
1690
|
else "取消" if res["cancelled"]
|
|
1679
1691
|
else "退出码 %s" % res["exit_code"])
|
|
1680
|
-
if res.get("deadline_exceeded"):
|
|
1681
|
-
head = "任务总时限已到"
|
|
1682
|
-
out["error"] = head + (";stderr/stdout: " + tail if tail else "")
|
|
1692
|
+
if res.get("deadline_exceeded"):
|
|
1693
|
+
head = "任务总时限已到"
|
|
1694
|
+
out["error"] = head + (";stderr/stdout: " + tail if tail else "")
|
|
1683
1695
|
if kind == "codex":
|
|
1684
1696
|
fm = _codex_fail_msg(res["stdout"])
|
|
1685
1697
|
if fm:
|
|
@@ -1789,44 +1801,44 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1789
1801
|
if not out["text"]:
|
|
1790
1802
|
out["error_code"] = _classify_failure(res, kind=kind, empty_output=True)
|
|
1791
1803
|
break
|
|
1792
|
-
_report_key(att, out)
|
|
1793
|
-
raw = out.get("raw") or {}
|
|
1794
|
-
attempt_history.append({
|
|
1795
|
-
"model": out.get("model") or att.get("model") or "",
|
|
1796
|
-
"provider_id": out.get("provider_id") or att.get("provider_id") or "",
|
|
1797
|
-
"provider": ((out.get("provider") or {}).get("name", "")
|
|
1798
|
-
if isinstance(out.get("provider"), dict) else ""),
|
|
1799
|
-
"kind": kind,
|
|
1800
|
-
"ok": bool(out.get("ok")),
|
|
1801
|
-
"duration": raw.get("duration", 0.0),
|
|
1802
|
-
"timed_out": bool(raw.get("timed_out")),
|
|
1803
|
-
"deadline_exceeded": bool(raw.get("deadline_exceeded")),
|
|
1804
|
-
"abort_marker": raw.get("abort_marker"),
|
|
1805
|
-
"error_code": str(out.get("error_code") or ""),
|
|
1806
|
-
"error": (out.get("error") or "")[:500],
|
|
1807
|
-
})
|
|
1808
|
-
out["attempts"] = list(attempt_history)
|
|
1809
|
-
if out["ok"] or ai == len(attempts) - 1:
|
|
1810
|
-
remaining = model_deadline - time.monotonic()
|
|
1811
|
-
if (not out["ok"] and grace_left > 0 and _rate_limited(out.get("error"))
|
|
1812
|
-
and remaining > 2
|
|
1813
|
-
and not (cancel_event is not None and cancel_event.is_set())):
|
|
1814
|
-
# 链尾限流宽限:enumerate 活列表——把链尾这条 append 回去,
|
|
1815
|
-
# 下一轮迭代就是「原地重试一次」;再撞限流时预算已耗尽照常判死。
|
|
1816
|
-
grace_left -= 1
|
|
1817
|
-
grace_s = min(RATE_LIMIT_GRACE_S, remaining - 1)
|
|
1818
|
-
if _grace_wait(cancel_event, grace_s, log_path=log_path):
|
|
1819
|
-
retry = dict(att, _deadline=model_deadline)
|
|
1820
|
-
attempts.append(retry)
|
|
1821
|
-
continue
|
|
1822
|
-
return out
|
|
1823
|
-
if _auth_error(out.get("error")):
|
|
1824
|
-
auth_failed_credentials.add(credential)
|
|
1825
|
-
if not any(_attempt_credential(candidate) not in auth_failed_credentials
|
|
1826
|
-
for candidate in attempts[ai + 1:]):
|
|
1827
|
-
return out
|
|
1828
|
-
continue
|
|
1829
|
-
# 换将闸门:什么失败值得烧链上下一个候选。
|
|
1804
|
+
_report_key(att, out)
|
|
1805
|
+
raw = out.get("raw") or {}
|
|
1806
|
+
attempt_history.append({
|
|
1807
|
+
"model": out.get("model") or att.get("model") or "",
|
|
1808
|
+
"provider_id": out.get("provider_id") or att.get("provider_id") or "",
|
|
1809
|
+
"provider": ((out.get("provider") or {}).get("name", "")
|
|
1810
|
+
if isinstance(out.get("provider"), dict) else ""),
|
|
1811
|
+
"kind": kind,
|
|
1812
|
+
"ok": bool(out.get("ok")),
|
|
1813
|
+
"duration": raw.get("duration", 0.0),
|
|
1814
|
+
"timed_out": bool(raw.get("timed_out")),
|
|
1815
|
+
"deadline_exceeded": bool(raw.get("deadline_exceeded")),
|
|
1816
|
+
"abort_marker": raw.get("abort_marker"),
|
|
1817
|
+
"error_code": str(out.get("error_code") or ""),
|
|
1818
|
+
"error": (out.get("error") or "")[:500],
|
|
1819
|
+
})
|
|
1820
|
+
out["attempts"] = list(attempt_history)
|
|
1821
|
+
if out["ok"] or ai == len(attempts) - 1:
|
|
1822
|
+
remaining = model_deadline - time.monotonic()
|
|
1823
|
+
if (not out["ok"] and grace_left > 0 and _rate_limited(out.get("error"))
|
|
1824
|
+
and remaining > 2
|
|
1825
|
+
and not (cancel_event is not None and cancel_event.is_set())):
|
|
1826
|
+
# 链尾限流宽限:enumerate 活列表——把链尾这条 append 回去,
|
|
1827
|
+
# 下一轮迭代就是「原地重试一次」;再撞限流时预算已耗尽照常判死。
|
|
1828
|
+
grace_left -= 1
|
|
1829
|
+
grace_s = min(RATE_LIMIT_GRACE_S, remaining - 1)
|
|
1830
|
+
if _grace_wait(cancel_event, grace_s, log_path=log_path):
|
|
1831
|
+
retry = dict(att, _deadline=model_deadline)
|
|
1832
|
+
attempts.append(retry)
|
|
1833
|
+
continue
|
|
1834
|
+
return out
|
|
1835
|
+
if _auth_error(out.get("error")):
|
|
1836
|
+
auth_failed_credentials.add(credential)
|
|
1837
|
+
if not any(_attempt_credential(candidate) not in auth_failed_credentials
|
|
1838
|
+
for candidate in attempts[ai + 1:]):
|
|
1839
|
+
return out
|
|
1840
|
+
continue
|
|
1841
|
+
# 换将闸门:什么失败值得烧链上下一个候选。
|
|
1830
1842
|
# · 瞬态/配额(文本特征,含 CLI 内部 api_retry 的 429/5xx 字样)照旧;
|
|
1831
1843
|
# · 超时:显式可换(看 res 标志位)——慢上游换快候选可能救回。
|
|
1832
1844
|
# 2026-09-23 夜班实案:旧逻辑里超时能否换将全赌输出尾部碰巧含有
|
|
@@ -1835,20 +1847,20 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1835
1847
|
# 换异上游候选常常就活;判死则整 run 白烧(同案:gemini 拒答后
|
|
1836
1848
|
# 链上云知声候选从未被尝试)。
|
|
1837
1849
|
raw = out.get("raw") or {}
|
|
1838
|
-
if (raw.get("cancelled") # 取消是用户意志,绝不降级(尾段可能带瞬态字样)
|
|
1839
|
-
or not (_transient_error(out.get("error")) or _quota_error(out.get("error"))
|
|
1840
|
-
or _refusal_error(out.get("error"))
|
|
1841
|
-
or raw.get("timed_out"))):
|
|
1850
|
+
if (raw.get("cancelled") # 取消是用户意志,绝不降级(尾段可能带瞬态字样)
|
|
1851
|
+
or not (_transient_error(out.get("error")) or _quota_error(out.get("error"))
|
|
1852
|
+
or _refusal_error(out.get("error"))
|
|
1853
|
+
or raw.get("timed_out"))):
|
|
1842
1854
|
return out # 取消/解析失败等真终态不降级
|
|
1843
|
-
# 同一上游的模型连续两次超时就收手——
|
|
1844
|
-
# 链上候选绑同一上游时换将=换壳不换命(2026-09-22 429 集群教训;
|
|
1845
|
-
# 2026-09-23 夜班实案同一慢上游连烧 3×20 分钟全超时零产出),最多
|
|
1846
|
-
# 两个模型预算后转向其他上游。其余瞬态/拒答仍按链长全部尝试:各家病根
|
|
1855
|
+
# 同一上游的模型连续两次超时就收手——
|
|
1856
|
+
# 链上候选绑同一上游时换将=换壳不换命(2026-09-22 429 集群教训;
|
|
1857
|
+
# 2026-09-23 夜班实案同一慢上游连烧 3×20 分钟全超时零产出),最多
|
|
1858
|
+
# 两个模型预算后转向其他上游。其余瞬态/拒答仍按链长全部尝试:各家病根
|
|
1847
1859
|
# 不同,每个候选都值得一次机会。
|
|
1848
|
-
if raw.get("timed_out"):
|
|
1849
|
-
timed_out_upstreams[upstream] = timed_out_upstreams.get(upstream, 0) + 1
|
|
1850
|
-
if timed_out_upstreams[upstream] >= 2:
|
|
1851
|
-
skipped_upstreams.add(upstream)
|
|
1860
|
+
if raw.get("timed_out"):
|
|
1861
|
+
timed_out_upstreams[upstream] = timed_out_upstreams.get(upstream, 0) + 1
|
|
1862
|
+
if timed_out_upstreams[upstream] >= 2:
|
|
1863
|
+
skipped_upstreams.add(upstream)
|
|
1852
1864
|
return out
|
|
1853
1865
|
|
|
1854
1866
|
|