codebee 0.1.54 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +3 -4
- package/app/core/builtin_agent.py +67 -26
- package/app/core/jobs.py +21 -2
- package/app/core/knowledge.py +21 -4
- package/app/core/modelhub.py +33 -4
- package/app/core/pipeline.py +143 -61
- package/app/core/planner.py +27 -13
- package/app/core/runner.py +280 -78
- package/app/core/store.py +64 -27
- package/app/core/zentao.py +26 -6
- package/app/ui/app.js +23 -13
- package/app/ui/board.css +360 -262
- package/app/ui/board.html +119 -41
- package/app/ui/board.js +60 -14
- package/app/ui/i18n.js +2698 -2659
- package/app/ui/index.html +2 -0
- package/package.json +1 -1
package/app/core/runner.py
CHANGED
|
@@ -29,7 +29,7 @@ from .error_codes import ErrorCode
|
|
|
29
29
|
# 置 0 则两边通用(remote.py 同款守卫)。
|
|
30
30
|
CREATE_NO_WINDOW = 0x08000000 if os.name == "nt" else 0
|
|
31
31
|
DEFAULT_TIMEOUT = 1200 # 单步 20 分钟
|
|
32
|
-
MAX_ATTEMPT_TIMEOUT_S = 60 #
|
|
32
|
+
MAX_ATTEMPT_TIMEOUT_S = 60 # 无进展时单模型最多等 1 分钟
|
|
33
33
|
|
|
34
34
|
_BASH_CANDIDATES = [
|
|
35
35
|
r"D:\Git\usr\bin\bash.exe",
|
|
@@ -559,9 +559,49 @@ def pretty_cli_log(text, max_event_chars=4000):
|
|
|
559
559
|
return "\n".join(out)
|
|
560
560
|
|
|
561
561
|
|
|
562
|
-
def
|
|
563
|
-
|
|
564
|
-
|
|
562
|
+
def _normalize_abort_markers(value):
|
|
563
|
+
"""把重复错误配置规范成 ``[(marker, count), ...]``。
|
|
564
|
+
|
|
565
|
+
旧调用方传的是单个 ``(marker, count)`` 元组,必须继续支持;新的
|
|
566
|
+
调用方可以传字典或元组列表,以便同一 CLI 同时对多种断流文案止损。
|
|
567
|
+
"""
|
|
568
|
+
if not value:
|
|
569
|
+
return []
|
|
570
|
+
if isinstance(value, str):
|
|
571
|
+
return [(value, 1)]
|
|
572
|
+
if isinstance(value, dict):
|
|
573
|
+
raw = list(value.items())
|
|
574
|
+
elif (isinstance(value, (tuple, list)) and len(value) == 2
|
|
575
|
+
and isinstance(value[0], str)):
|
|
576
|
+
raw = [value]
|
|
577
|
+
else:
|
|
578
|
+
try:
|
|
579
|
+
raw = list(value)
|
|
580
|
+
except TypeError:
|
|
581
|
+
return []
|
|
582
|
+
out = []
|
|
583
|
+
for item in raw:
|
|
584
|
+
if isinstance(item, str):
|
|
585
|
+
marker, limit = item, 1
|
|
586
|
+
else:
|
|
587
|
+
try:
|
|
588
|
+
marker, limit = item
|
|
589
|
+
except (TypeError, ValueError):
|
|
590
|
+
continue
|
|
591
|
+
marker = str(marker or "")
|
|
592
|
+
try:
|
|
593
|
+
limit = max(1, int(limit))
|
|
594
|
+
except (TypeError, ValueError):
|
|
595
|
+
continue
|
|
596
|
+
if marker:
|
|
597
|
+
out.append((marker, limit))
|
|
598
|
+
return out
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
602
|
+
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None,
|
|
603
|
+
stall_timeout=0, repeat_abort=None, deadline=None,
|
|
604
|
+
abort_markers=None):
|
|
565
605
|
"""通用子进程执行:并发读管道防死锁;超时/取消杀整棵进程树。
|
|
566
606
|
|
|
567
607
|
stall_timeout:停滞看门狗(秒,0=关闭)——超过该时长 stdout/stderr 无任何
|
|
@@ -569,17 +609,40 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
569
609
|
持续流动的 CLI 开(codex JSONL 事件流);claude json 到结束才一次性输出,
|
|
570
610
|
开了会把正常长任务误杀。
|
|
571
611
|
|
|
572
|
-
repeat_abort:可选 (marker, count)
|
|
573
|
-
CLI
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
612
|
+
repeat_abort:可选 (marker, count),也接受多个 ``(marker, count)``;同一
|
|
613
|
+
致命错误达到次数即提前杀树,处理 CLI 自身不断打印重连消息、因此永远
|
|
614
|
+
触发不了静默看门狗的假运行。abort_markers 是兼容扩展,用于不改变旧
|
|
615
|
+
repeat_abort 参数形状的情况下追加多种一次性断流标记。
|
|
616
|
+
|
|
617
|
+
deadline:可选的 ``time.monotonic()`` 绝对截止时刻。它比 timeout 优先,
|
|
618
|
+
到点会杀掉进程树并返回 deadline_exceeded=True,调用方可把整个任务收口
|
|
619
|
+
为 timeout,而不是把它误记成普通供应商失败。
|
|
620
|
+
|
|
621
|
+
返回 {ok, exit_code, stdout, stderr, duration, cancelled, timed_out, stalled,
|
|
622
|
+
deadline_exceeded, abort_marker}。
|
|
623
|
+
"""
|
|
577
624
|
if shell_cmd:
|
|
578
625
|
# shell 串的解析器随平台:重定向/引号语法两边通用,只是解释器不同
|
|
579
626
|
argv = ["cmd", "/c", shell_cmd] if os.name == "nt" else ["/bin/sh", "-c", shell_cmd]
|
|
580
|
-
if argv is None:
|
|
581
|
-
return {"ok": False, "exit_code": None, "stdout": "", "stderr": "argv 为空",
|
|
582
|
-
"duration": 0.0, "cancelled": False, "timed_out": False,
|
|
627
|
+
if argv is None:
|
|
628
|
+
return {"ok": False, "exit_code": None, "stdout": "", "stderr": "argv 为空",
|
|
629
|
+
"duration": 0.0, "cancelled": False, "timed_out": False,
|
|
630
|
+
"stalled": False, "deadline_exceeded": False, "abort_marker": None}
|
|
631
|
+
try:
|
|
632
|
+
deadline = float(deadline) if deadline is not None else None
|
|
633
|
+
except (TypeError, ValueError):
|
|
634
|
+
deadline = None
|
|
635
|
+
try:
|
|
636
|
+
timeout = max(0.0, float(timeout))
|
|
637
|
+
except (TypeError, ValueError):
|
|
638
|
+
timeout = float(DEFAULT_TIMEOUT)
|
|
639
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
640
|
+
return {"ok": False, "exit_code": None, "stdout": "",
|
|
641
|
+
"stderr": "[任务总时限已到,未启动子进程]", "duration": 0.0,
|
|
642
|
+
"cancelled": False, "timed_out": True, "stalled": False,
|
|
643
|
+
"deadline_exceeded": True, "abort_marker": None}
|
|
644
|
+
aborts = _normalize_abort_markers(repeat_abort)
|
|
645
|
+
aborts.extend(_normalize_abort_markers(abort_markers))
|
|
583
646
|
full_env = scrub_env(os.environ.copy(), mode="drop")
|
|
584
647
|
if env:
|
|
585
648
|
# 5A:env 关键字环境变量注入用户传入的 env(属于有意注入,例如模型 API key)
|
|
@@ -610,10 +673,11 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
610
673
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
611
674
|
creationflags=CREATE_NO_WINDOW,
|
|
612
675
|
start_new_session=(os.name != "nt")) # POSIX 需独立进程组供 killpg 杀树;Windows 忽略该参数
|
|
613
|
-
except Exception as e:
|
|
614
|
-
return {"ok": False, "exit_code": None, "stdout": "",
|
|
615
|
-
"stderr": "启动失败: %r" % e, "duration": 0.0,
|
|
616
|
-
"cancelled": False, "timed_out": False
|
|
676
|
+
except Exception as e:
|
|
677
|
+
return {"ok": False, "exit_code": None, "stdout": "",
|
|
678
|
+
"stderr": "启动失败: %r" % e, "duration": 0.0,
|
|
679
|
+
"cancelled": False, "timed_out": False, "stalled": False,
|
|
680
|
+
"deadline_exceeded": False, "abort_marker": None}
|
|
617
681
|
out_chunks, err_chunks = [], []
|
|
618
682
|
stamp = [time.time()] # 最后输出时刻(两条管道共同刷新)
|
|
619
683
|
t_out = threading.Thread(target=_pipe_reader, args=(proc.stdout, out_chunks, log_fh, stamp), daemon=True)
|
|
@@ -633,23 +697,36 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
633
697
|
pass
|
|
634
698
|
threading.Thread(target=_feed, daemon=True).start()
|
|
635
699
|
start = time.time()
|
|
636
|
-
cancelled = timed_out = stalled = repeat_aborted = False
|
|
637
|
-
|
|
700
|
+
cancelled = timed_out = stalled = repeat_aborted = False
|
|
701
|
+
deadline_exceeded = False
|
|
702
|
+
abort_marker = None
|
|
703
|
+
while True:
|
|
638
704
|
try:
|
|
639
705
|
proc.wait(timeout=0.4)
|
|
640
706
|
break
|
|
641
707
|
except subprocess.TimeoutExpired:
|
|
642
708
|
pass
|
|
643
|
-
if cancel_event is not None and cancel_event.is_set():
|
|
644
|
-
cancelled = True
|
|
645
|
-
_kill_tree(proc.pid)
|
|
646
|
-
_drain_streams(proc, t_out, t_err, timeout=10)
|
|
647
|
-
break
|
|
648
|
-
if time.
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
709
|
+
if cancel_event is not None and cancel_event.is_set():
|
|
710
|
+
cancelled = True
|
|
711
|
+
_kill_tree(proc.pid)
|
|
712
|
+
_drain_streams(proc, t_out, t_err, timeout=10)
|
|
713
|
+
break
|
|
714
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
715
|
+
deadline_exceeded = True
|
|
716
|
+
timed_out = True
|
|
717
|
+
_kill_tree(proc.pid)
|
|
718
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
719
|
+
break
|
|
720
|
+
if time.time() - start > timeout:
|
|
721
|
+
# Claude/Codex 的工具循环可能持续吐 stream-json 事件而迟迟
|
|
722
|
+
# 不给最终 result。只要仍有活动,交给 stall_timeout 作为静默
|
|
723
|
+
# 上限;无输出的挂死仍按原单模型 timeout 快速换将。
|
|
724
|
+
active = stall_timeout and time.time() - stamp[0] < stall_timeout
|
|
725
|
+
if not active:
|
|
726
|
+
timed_out = True
|
|
727
|
+
_kill_tree(proc.pid)
|
|
728
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
729
|
+
break
|
|
653
730
|
if stall_timeout and time.time() - stamp[0] > stall_timeout:
|
|
654
731
|
# 停滞看门狗:长静默多为卡死(网关挂起/CLI 假死),与其耗满总
|
|
655
732
|
# 超时不如提前杀——错误按超时归类,走既有的换模型/换将链路
|
|
@@ -658,19 +735,22 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
658
735
|
_kill_tree(proc.pid)
|
|
659
736
|
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
660
737
|
break
|
|
661
|
-
if
|
|
662
|
-
try:
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
738
|
+
if aborts:
|
|
739
|
+
try:
|
|
740
|
+
recent = decode_output(b"".join(out_chunks)[-8000:] +
|
|
741
|
+
b"\n" + b"".join(err_chunks)[-8000:])
|
|
742
|
+
for marker, limit in aborts:
|
|
743
|
+
if recent.count(marker) >= limit:
|
|
744
|
+
repeat_aborted = True
|
|
745
|
+
abort_marker = marker
|
|
746
|
+
timed_out = True
|
|
747
|
+
_kill_tree(proc.pid)
|
|
748
|
+
_drain_streams(proc, t_out, t_err, timeout=5)
|
|
749
|
+
break
|
|
750
|
+
if repeat_aborted:
|
|
751
|
+
break
|
|
752
|
+
except (TypeError, ValueError):
|
|
753
|
+
pass
|
|
674
754
|
duration = round(time.time() - start, 1)
|
|
675
755
|
t_out.join(timeout=5)
|
|
676
756
|
t_err.join(timeout=5)
|
|
@@ -687,8 +767,17 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
687
767
|
pass
|
|
688
768
|
if cancelled:
|
|
689
769
|
stderr += "\n[已被用户取消]"
|
|
690
|
-
elif repeat_aborted:
|
|
691
|
-
|
|
770
|
+
elif repeat_aborted:
|
|
771
|
+
limit = next((n for m, n in aborts if m == abort_marker), "?")
|
|
772
|
+
if len(aborts) == 1 and abort_markers is None:
|
|
773
|
+
# 保留旧版 UI/测试已经依赖的文案;多标记配置使用更具体的
|
|
774
|
+
# 标记名,方便定位究竟是哪一种断流触发了止损。
|
|
775
|
+
stderr += "\n[同一网络错误重复 %s 次,已提前终止进程树]" % limit
|
|
776
|
+
else:
|
|
777
|
+
stderr += "\n[错误标记 %s 重复 %s 次,已提前终止进程树]" % (
|
|
778
|
+
abort_marker, limit)
|
|
779
|
+
elif deadline_exceeded:
|
|
780
|
+
stderr += "\n[任务总时限已到,已终止进程树]"
|
|
692
781
|
elif stalled:
|
|
693
782
|
stderr += "\n[输出停滞 %ss,已终止进程树]" % stall_timeout
|
|
694
783
|
elif timed_out:
|
|
@@ -696,9 +785,10 @@ def run_process(argv=None, shell_cmd=None, stdin_text=None, cwd=None, env=None,
|
|
|
696
785
|
return {
|
|
697
786
|
"ok": exit_code == 0 and not cancelled and not timed_out,
|
|
698
787
|
"exit_code": exit_code, "stdout": stdout, "stderr": stderr,
|
|
699
|
-
"duration": duration, "cancelled": cancelled, "timed_out": timed_out,
|
|
700
|
-
"stalled": stalled,
|
|
701
|
-
|
|
788
|
+
"duration": duration, "cancelled": cancelled, "timed_out": timed_out,
|
|
789
|
+
"stalled": stalled, "deadline_exceeded": deadline_exceeded,
|
|
790
|
+
"abort_marker": abort_marker,
|
|
791
|
+
}
|
|
702
792
|
finally:
|
|
703
793
|
if log_fh:
|
|
704
794
|
try:
|
|
@@ -869,7 +959,8 @@ _TRANSIENT = ("503", "502", "529", "429", "no available channel", "temporarily",
|
|
|
869
959
|
# 2026-09-20 连载评审实测:codex 网关断流(stream disconnected)
|
|
870
960
|
# 与 opencode 服务端 500(Unexpected server error)都是「重试/换将
|
|
871
961
|
# 就可能活」的瞬态病,旧表判成终态导致整链早死
|
|
872
|
-
"stream disconnected", "
|
|
962
|
+
"stream disconnected", "stream closed before response.completed",
|
|
963
|
+
"unexpected server error",
|
|
873
964
|
# 2026-09-22 mo-so 实案:claude 报 "Can't reach the API server —
|
|
874
965
|
# check your internet or DNS (ENOTFOUND)",一家端点解析不到不代表
|
|
875
966
|
# 跨厂商链上别家也不通,旧表判成终态直接跳去换将别的 CLI
|
|
@@ -1109,7 +1200,11 @@ def _prompt_to_file(prompt, workdir):
|
|
|
1109
1200
|
return None
|
|
1110
1201
|
|
|
1111
1202
|
|
|
1112
|
-
|
|
1203
|
+
# Windows 上不少 CLI 先经过 cmd.exe/.cmd 垫片,实际安全上限接近 8191,
|
|
1204
|
+
# 而不是 CreateProcess 的理论 32767。留出命令、路径和环境参数余量,超过
|
|
1205
|
+
# 该值统一落盘,避免 MiMo/Kimi 评审在启动前直接报「命令行太长」。
|
|
1206
|
+
_ARGV_PROMPT_SAFE = 20000
|
|
1207
|
+
_ARGV_PROMPT_SHIM_SAFE = 6000
|
|
1113
1208
|
|
|
1114
1209
|
|
|
1115
1210
|
def _build_call(agent, kind, sid, readonly, model, prompt, images=None, workdir=None):
|
|
@@ -1247,14 +1342,25 @@ def _build_call(agent, kind, sid, readonly, model, prompt, images=None, workdir=
|
|
|
1247
1342
|
# 超长时落盘临时文件、参数位换成读文件指令——评审/作者 CLI 非交互模式均带
|
|
1248
1343
|
# 读文件工具(kimi 0.43 实测可读),读不到的调用输出不可解析,由评审全挂
|
|
1249
1344
|
# 防线兜底判失败,绝不静默降级。
|
|
1250
|
-
|
|
1251
|
-
|
|
1345
|
+
prompt_safe = _ARGV_PROMPT_SAFE
|
|
1346
|
+
if argv:
|
|
1347
|
+
launcher = str(argv[0]).lower()
|
|
1348
|
+
if (launcher.endswith((".cmd", ".bat")) or
|
|
1349
|
+
os.path.basename(launcher) in ("npm", "npx", "pnpm", "yarn", "bun")):
|
|
1350
|
+
prompt_safe = _ARGV_PROMPT_SHIM_SAFE
|
|
1351
|
+
if argv is not None and stdin_text is None and len(prompt) >= prompt_safe \
|
|
1352
|
+
and argv.count(prompt) == 1:
|
|
1252
1353
|
pf = _prompt_to_file(prompt, workdir)
|
|
1253
|
-
if pf:
|
|
1254
|
-
argv[argv.index(prompt)] = (
|
|
1255
|
-
"[系统] 本次完整指令因命令行长度限制已写入文件:%s\n"
|
|
1256
|
-
"请先用读文件工具完整读取该文件,然后把文件内容当作你的任务指令执行。" % pf)
|
|
1257
|
-
|
|
1354
|
+
if pf:
|
|
1355
|
+
argv[argv.index(prompt)] = (
|
|
1356
|
+
"[系统] 本次完整指令因命令行长度限制已写入文件:%s\n"
|
|
1357
|
+
"请先用读文件工具完整读取该文件,然后把文件内容当作你的任务指令执行。" % pf)
|
|
1358
|
+
# Keep the original payload on stdin as a compatibility path for
|
|
1359
|
+
# simple generic adapters and test shims that read stdin. CLI tools
|
|
1360
|
+
# that intentionally use argv still receive the bounded file hint.
|
|
1361
|
+
if kind == "generic" and stdin_text is None and len(tmpl) > 1:
|
|
1362
|
+
stdin_text = prompt
|
|
1363
|
+
tmp_files.append(pf)
|
|
1258
1364
|
return argv, stdin_text, prompt, tmp_files
|
|
1259
1365
|
|
|
1260
1366
|
|
|
@@ -1289,9 +1395,31 @@ def _stall_timeout(env_name, default):
|
|
|
1289
1395
|
return default
|
|
1290
1396
|
|
|
1291
1397
|
|
|
1292
|
-
def
|
|
1293
|
-
|
|
1294
|
-
|
|
1398
|
+
def _max_model_attempts(has_deadline=False):
|
|
1399
|
+
"""返回一次 runner 调用允许消耗的候选数。
|
|
1400
|
+
|
|
1401
|
+
生产任务会传入统一 deadline,默认收敛到 3 个候选;没有 deadline 的
|
|
1402
|
+
旧式直接调用保留最多 4 个候选,以兼容已有的「同一上游两模型后换
|
|
1403
|
+
供应商」策略。运维可用环境变量进一步收紧/放宽,最少 1 个。
|
|
1404
|
+
"""
|
|
1405
|
+
default = 3 if has_deadline else 4
|
|
1406
|
+
try:
|
|
1407
|
+
value = int(os.environ.get("TUTTI_MAX_MODEL_ATTEMPTS", str(default)))
|
|
1408
|
+
return max(1, min(12, value))
|
|
1409
|
+
except (TypeError, ValueError):
|
|
1410
|
+
return default
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
_INVALID_MODEL_NAMES = frozenset(("auto", "default", "none"))
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
def _invalid_model(model):
|
|
1417
|
+
return str(model or "").strip().lower() in _INVALID_MODEL_NAMES
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
1421
|
+
timeout=DEFAULT_TIMEOUT, cancel_event=None, log_path=None, resume=None,
|
|
1422
|
+
images=None, require_tools=False, deadline=None):
|
|
1295
1423
|
"""执行一次智能体调用,返回统一结构
|
|
1296
1424
|
{ok, text, json, cost_usd, tokens, error, error_code, raw}。
|
|
1297
1425
|
agent 来自 registry.effective_agents();resume 为已有会话 id,仅真实智能体生效
|
|
@@ -1308,21 +1436,41 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1308
1436
|
失败不降级;同一上游超时两次即跳过该上游剩余模型。
|
|
1309
1437
|
|
|
1310
1438
|
5E:catalog `orch.timeout_ms`(毫秒)优先于 caller 传入的 timeout;
|
|
1311
|
-
|
|
1312
|
-
|
|
1439
|
+
单模型无进展时最多等 60 秒,持续 stream 活动交由 stall_timeout 看门狗
|
|
1440
|
+
收口。deadline 是
|
|
1441
|
+
任务共享的 ``time.monotonic()`` 绝对截止时刻;传入后所有模型、空响应
|
|
1442
|
+
重试和限流宽限共用同一剩余预算。
|
|
1443
|
+
"""
|
|
1313
1444
|
# 5E:catalog orch.timeout_ms 优先
|
|
1314
1445
|
orch_timeout_ms = (agent.get("orch") or {}).get("timeout_ms")
|
|
1315
1446
|
if orch_timeout_ms:
|
|
1316
1447
|
timeout = float(orch_timeout_ms) / 1000.0
|
|
1317
|
-
|
|
1448
|
+
try:
|
|
1449
|
+
timeout = float(timeout)
|
|
1450
|
+
except (TypeError, ValueError):
|
|
1451
|
+
timeout = float(DEFAULT_TIMEOUT)
|
|
1452
|
+
timeout = max(0.0, timeout)
|
|
1453
|
+
try:
|
|
1454
|
+
deadline = float(deadline) if deadline is not None else None
|
|
1455
|
+
except (TypeError, ValueError):
|
|
1456
|
+
deadline = None
|
|
1457
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1458
|
+
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1459
|
+
"tokens": 0, "usage": None, "error": "任务总时限已到,未启动模型",
|
|
1460
|
+
"error_code": ErrorCode.TIMEOUT,
|
|
1461
|
+
"raw": {"exit_code": None, "timed_out": True,
|
|
1462
|
+
"deadline_exceeded": True, "duration": 0.0},
|
|
1463
|
+
"kind": agent.get("kind", "generic"), "model": None,
|
|
1464
|
+
"attempts": []}
|
|
1465
|
+
timeout = min(timeout, MAX_ATTEMPT_TIMEOUT_S)
|
|
1318
1466
|
kind = agent.get("kind", "generic")
|
|
1319
1467
|
if kind == "aider":
|
|
1320
1468
|
repo_issue = _git_repo_issue(workdir)
|
|
1321
1469
|
if repo_issue:
|
|
1322
1470
|
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1323
|
-
"tokens": 0, "usage": None, "error": repo_issue,
|
|
1324
|
-
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1325
|
-
"kind": kind, "model": agent.get("model")}
|
|
1471
|
+
"tokens": 0, "usage": None, "error": repo_issue,
|
|
1472
|
+
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1473
|
+
"kind": kind, "model": agent.get("model"), "attempts": []}
|
|
1326
1474
|
if kind == "codex":
|
|
1327
1475
|
stall_t = _stall_timeout("TUTTI_CODEX_STALL_TIMEOUT", 600)
|
|
1328
1476
|
elif kind == "claude":
|
|
@@ -1339,9 +1487,9 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1339
1487
|
ok, reason = _check_approval(agent)
|
|
1340
1488
|
if not ok:
|
|
1341
1489
|
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0, "tokens": 0,
|
|
1342
|
-
"usage": None, "error": reason,
|
|
1343
|
-
"error_code": ErrorCode.ENV_BLOCK,
|
|
1344
|
-
"raw": None, "kind": kind, "model": None}
|
|
1490
|
+
"usage": None, "error": reason,
|
|
1491
|
+
"error_code": ErrorCode.ENV_BLOCK,
|
|
1492
|
+
"raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1345
1493
|
base_env = dict(agent.get("env") or {})
|
|
1346
1494
|
if kind == "claude":
|
|
1347
1495
|
bash = find_git_bash()
|
|
@@ -1354,19 +1502,48 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1354
1502
|
"usage": None,
|
|
1355
1503
|
"error": "该 CLI 未配置会话恢复(catalog orch.resume_argv_template),"
|
|
1356
1504
|
"无法在已有会话上继续",
|
|
1357
|
-
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1358
|
-
"sid": "", "raw": None, "kind": kind, "model": None}
|
|
1359
|
-
|
|
1360
|
-
attempts = _resolve_attempts(agent)
|
|
1361
|
-
|
|
1362
|
-
|
|
1505
|
+
"error_code": ErrorCode.VENDOR_ERROR,
|
|
1506
|
+
"sid": "", "raw": None, "kind": kind, "model": None, "attempts": []}
|
|
1507
|
+
|
|
1508
|
+
attempts = _resolve_attempts(agent)
|
|
1509
|
+
# 启动前过滤明显的占位模型名。它们常由某个 CLI 的会话标题/默认配置
|
|
1510
|
+
# 泄漏进编排链;让子进程自己等待网关返回会把一个确定性配置错拖满
|
|
1511
|
+
# 多轮超时。非法项不消耗模型调用预算,后面的真实候选仍可接手。
|
|
1512
|
+
filtered_attempts = []
|
|
1513
|
+
preflight_failures = []
|
|
1514
|
+
for att in attempts:
|
|
1515
|
+
if _invalid_model(att.get("model")):
|
|
1516
|
+
preflight_failures.append({
|
|
1517
|
+
"model": att.get("model"),
|
|
1518
|
+
"provider_id": att.get("provider_id") or "",
|
|
1519
|
+
"provider": (att.get("provider") or {}).get("name", "")
|
|
1520
|
+
if isinstance(att.get("provider"), dict) else "",
|
|
1521
|
+
"ok": False, "skipped": True, "error":
|
|
1522
|
+
"无效模型名 %r,启动前跳过" % att.get("model"),
|
|
1523
|
+
})
|
|
1524
|
+
continue
|
|
1525
|
+
filtered_attempts.append(att)
|
|
1526
|
+
attempts = filtered_attempts[:_max_model_attempts(deadline is not None)]
|
|
1527
|
+
out = None
|
|
1528
|
+
attempt_history = list(preflight_failures)
|
|
1529
|
+
if not attempts:
|
|
1530
|
+
return {"ok": False, "text": "", "json": None, "cost_usd": 0.0,
|
|
1531
|
+
"tokens": 0, "usage": None,
|
|
1532
|
+
"error": (preflight_failures[-1].get("error")
|
|
1533
|
+
if preflight_failures else "没有可尝试的模型"),
|
|
1534
|
+
"error_code": ErrorCode.VENDOR_ERROR, "raw": None,
|
|
1535
|
+
"kind": kind, "model": None, "attempts": attempt_history}
|
|
1536
|
+
grace_left = RATE_LIMIT_GRACE_N # 链尾限流宽限预算:整链撞 429 时原地等一个窗口再试一次
|
|
1363
1537
|
timed_out_upstreams = {} # 同一上游两次超时后跳过剩余模型
|
|
1364
1538
|
skipped_upstreams = set()
|
|
1365
1539
|
for ai, att in enumerate(attempts):
|
|
1366
1540
|
upstream = _attempt_upstream(att)
|
|
1367
1541
|
if upstream in skipped_upstreams:
|
|
1368
1542
|
continue
|
|
1369
|
-
|
|
1543
|
+
# 有任务 deadline 时,所有候选共享同一个截止时刻;没有 deadline
|
|
1544
|
+
# 的历史直接调用保留「每个候选最多 60 秒」的旧语义。
|
|
1545
|
+
model_deadline = (att.get("_deadline") or deadline or
|
|
1546
|
+
(time.monotonic() + timeout))
|
|
1370
1547
|
env = dict(base_env)
|
|
1371
1548
|
env.update(att["env"])
|
|
1372
1549
|
if kind == "codex":
|
|
@@ -1386,7 +1563,8 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1386
1563
|
if remaining <= 0:
|
|
1387
1564
|
res = {"ok": False, "exit_code": None, "stdout": "", "stderr": "",
|
|
1388
1565
|
"duration": timeout, "cancelled": False, "timed_out": True,
|
|
1389
|
-
"stalled": False
|
|
1566
|
+
"stalled": False,
|
|
1567
|
+
"deadline_exceeded": deadline is not None}
|
|
1390
1568
|
else:
|
|
1391
1569
|
argv, stdin_text, prompt_eff, tmp_files = _build_call(
|
|
1392
1570
|
eff_agent, kind, sid, readonly,
|
|
@@ -1396,10 +1574,16 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1396
1574
|
try:
|
|
1397
1575
|
repeat_guard = (("Reconnecting...", 2)
|
|
1398
1576
|
if kind == "codex" else None)
|
|
1577
|
+
stream_abort_markers = (
|
|
1578
|
+
(("stream disconnected", 1),
|
|
1579
|
+
("stream closed before response.completed", 1))
|
|
1580
|
+
if kind == "codex" else None)
|
|
1399
1581
|
res = run_process(argv=argv, stdin_text=stdin_text, cwd=workdir, env=env,
|
|
1400
|
-
timeout=min(timeout, remaining),
|
|
1582
|
+
timeout=min(timeout, remaining), deadline=deadline,
|
|
1583
|
+
cancel_event=cancel_event,
|
|
1401
1584
|
log_path=log_path, stall_timeout=stall_t,
|
|
1402
|
-
repeat_abort=repeat_guard
|
|
1585
|
+
repeat_abort=repeat_guard,
|
|
1586
|
+
abort_markers=stream_abort_markers)
|
|
1403
1587
|
finally:
|
|
1404
1588
|
# 超长指令临时文件:CLI 进程已结束(管道已收),即刻清场不污染工作目录
|
|
1405
1589
|
for tf in tmp_files:
|
|
@@ -1424,7 +1608,9 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1424
1608
|
else "超时" if res["timed_out"]
|
|
1425
1609
|
else "取消" if res["cancelled"]
|
|
1426
1610
|
else "退出码 %s" % res["exit_code"])
|
|
1427
|
-
|
|
1611
|
+
if res.get("deadline_exceeded"):
|
|
1612
|
+
head = "任务总时限已到"
|
|
1613
|
+
out["error"] = head + (";stderr/stdout: " + tail if tail else "")
|
|
1428
1614
|
if kind == "codex":
|
|
1429
1615
|
fm = _codex_fail_msg(res["stdout"])
|
|
1430
1616
|
if fm:
|
|
@@ -1535,6 +1721,22 @@ def run_agent(agent, prompt, workdir=None, readonly=True,
|
|
|
1535
1721
|
out["error_code"] = _classify_failure(res, kind=kind, empty_output=True)
|
|
1536
1722
|
break
|
|
1537
1723
|
_report_key(att, out)
|
|
1724
|
+
raw = out.get("raw") or {}
|
|
1725
|
+
attempt_history.append({
|
|
1726
|
+
"model": out.get("model") or att.get("model") or "",
|
|
1727
|
+
"provider_id": out.get("provider_id") or att.get("provider_id") or "",
|
|
1728
|
+
"provider": ((out.get("provider") or {}).get("name", "")
|
|
1729
|
+
if isinstance(out.get("provider"), dict) else ""),
|
|
1730
|
+
"kind": kind,
|
|
1731
|
+
"ok": bool(out.get("ok")),
|
|
1732
|
+
"duration": raw.get("duration", 0.0),
|
|
1733
|
+
"timed_out": bool(raw.get("timed_out")),
|
|
1734
|
+
"deadline_exceeded": bool(raw.get("deadline_exceeded")),
|
|
1735
|
+
"abort_marker": raw.get("abort_marker"),
|
|
1736
|
+
"error_code": str(out.get("error_code") or ""),
|
|
1737
|
+
"error": (out.get("error") or "")[:500],
|
|
1738
|
+
})
|
|
1739
|
+
out["attempts"] = list(attempt_history)
|
|
1538
1740
|
if out["ok"] or ai == len(attempts) - 1:
|
|
1539
1741
|
remaining = model_deadline - time.monotonic()
|
|
1540
1742
|
if (not out["ok"] and grace_left > 0 and _rate_limited(out.get("error"))
|