codebee 0.1.65 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +520 -513
- package/LICENSE +21 -21
- package/README.md +494 -487
- package/app/core/aiflavor.py +197 -197
- package/app/core/artifacts.py +148 -148
- package/app/core/attachments.py +532 -532
- package/app/core/automation.py +758 -758
- package/app/core/backup.py +470 -470
- package/app/core/beekeeper.py +183 -183
- package/app/core/board.py +209 -209
- package/app/core/bookmeta.py +796 -759
- package/app/core/bookmeta_catalog.py +180 -180
- package/app/core/branching.py +137 -137
- package/app/core/builtin_agent.py +1234 -1191
- package/app/core/capability.py +144 -144
- package/app/core/catalog.py +23 -24
- package/app/core/chaptersafe.py +32 -32
- package/app/core/cleanup.py +303 -303
- package/app/core/compaction.py +201 -186
- package/app/core/covergen.py +356 -356
- package/app/core/diagnostics.py +114 -114
- package/app/core/dispatch.py +173 -173
- package/app/core/dispatch_log.py +113 -113
- package/app/core/env_scrub.py +85 -85
- package/app/core/error_codes.py +220 -65
- package/app/core/errorlog.py +137 -137
- package/app/core/evaluation.py +174 -0
- package/app/core/failure_notify.py +290 -0
- package/app/core/flows.py +355 -355
- package/app/core/goal_service.py +167 -167
- package/app/core/health.py +374 -342
- package/app/core/history.py +38 -38
- package/app/core/hooks.py +162 -156
- package/app/core/jobs.py +1034 -1022
- package/app/core/knowledge.py +6 -2
- package/app/core/manager.py +2115 -1996
- package/app/core/market.py +362 -362
- package/app/core/market_remote.py +66 -66
- package/app/core/mocks.py +64 -64
- package/app/core/modelhub.py +3517 -3332
- package/app/core/notify.py +106 -106
- package/app/core/paihang.py +109 -109
- package/app/core/paths.py +2 -2
- package/app/core/pipeline.py +4614 -4494
- package/app/core/planner.py +33 -31
- package/app/core/portguard.py +111 -111
- package/app/core/preview.py +293 -293
- package/app/core/publish/__init__.py +7 -7
- package/app/core/publish/auto.py +367 -367
- package/app/core/publish/browser.py +533 -533
- package/app/core/publish/fanqie.py +128 -128
- package/app/core/publish/flow.py +432 -406
- package/app/core/publish/flows-fanqie-calibrated.json +17 -9
- package/app/core/publish/flows-qimao-calibrated.json +6 -2
- package/app/core/publish/ledger.py +198 -198
- package/app/core/publish/manager.py +623 -590
- package/app/core/publish/qimao.py +141 -141
- package/app/core/publish/ws.py +139 -139
- package/app/core/redact.py +38 -38
- package/app/core/repeat_guard.py +129 -124
- package/app/core/router.py +119 -81
- package/app/core/runner.py +2050 -2042
- package/app/core/selfupdate.py +384 -384
- package/app/core/session_log.py +161 -161
- package/app/core/sessions.py +312 -312
- package/app/core/settings.py +149 -149
- package/app/core/settings_schema.py +262 -262
- package/app/core/share_page.py +114 -114
- package/app/core/skill_scan.py +114 -106
- package/app/core/skillpacks/fanqie-novel.md +80 -80
- package/app/core/skillpacks/market/character-bible.md +66 -66
- package/app/core/skillpacks/market/code-risk-checklist.md +58 -58
- package/app/core/skillpacks/market/git-workflow.md +57 -57
- package/app/core/skillpacks/market/release-notes.md +72 -72
- package/app/core/skillpacks/market/weekly-report.md +71 -71
- package/app/core/skillpacks/market/worldview-consistency.md +70 -70
- package/app/core/skills.py +221 -72
- package/app/core/step_runner.py +132 -132
- package/app/core/store.py +23 -0
- package/app/core/task_compile.py +199 -199
- package/app/core/telemetry.py +292 -292
- package/app/core/tlsctx.py +64 -64
- package/app/core/token_meter.py +147 -147
- package/app/core/upstream.py +41 -0
- package/app/core/usage.py +114 -3
- package/app/core/volumes.py +355 -355
- package/app/core/wxdigest.py +858 -858
- package/app/core/zentao.py +2447 -2447
- package/app/main.py +2838 -2832
- package/app/pick_dialog.py +110 -110
- package/app/ui/app.js +13994 -13977
- package/app/ui/board.css +20 -41
- package/app/ui/board.html +162 -162
- package/app/ui/board.js +305 -305
- package/app/ui/i18n.js +2716 -2711
- package/app/ui/icons/bee.svg +79 -79
- package/app/ui/index.html +1 -1
- package/app/ui/manifest.json +16 -16
- package/app/ui/qrcode.js +2297 -2297
- package/app/ui/style.css +6107 -6125
- package/bin/tutti.js +147 -147
- package/package.json +46 -41
- package/tools/wx_reader.py +261 -261
- package/app/.playwright-cli/console-2026-09-23T08-26-03-635Z.log +0 -88
- package/app/.playwright-cli/page-2026-09-23T08-26-03-845Z.yml +0 -126
- package/app/.playwright-cli/page-2026-09-23T08-29-44-203Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-35-42-046Z.yml +0 -210
- package/app/.playwright-cli/page-2026-09-23T08-36-52-858Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-37-33-938Z.png +0 -0
- package/app/.playwright-cli/run-detail-after.png +0 -0
package/app/core/tlsctx.py
CHANGED
|
@@ -1,64 +1,64 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""HTTPS 证书校验上下文:补齐 macOS 上 Python 缺失的 CA 源(不关校验)。
|
|
3
|
-
|
|
4
|
-
现象:用官方 pkg 装 Node 的 Mac 上往往再装 python.org 的 Python,它不读
|
|
5
|
-
系统钥匙串,默认验证路径下没有根证书,所有 HTTPS 请求报
|
|
6
|
-
CERTIFICATE_VERIFY_FAILED(unable to get local issuer certificate)。
|
|
7
|
-
|
|
8
|
-
修法是给默认上下文**追加**可用 CA 源,校验语义只增不减:
|
|
9
|
-
1. certifi(装了就用,跨平台最全);
|
|
10
|
-
2. /etc/ssl/cert.pem(macOS 系统自带 CA 束,Catalina 起就有)。
|
|
11
|
-
两个都拿不到时返回默认上下文——报错与旧行为一致,绝不静默关校验。
|
|
12
|
-
"""
|
|
13
|
-
import ssl
|
|
14
|
-
import sys
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def cafiles():
|
|
18
|
-
"""候选 CA 束路径;坏路径/不存在由 load_verify_locations 抛错后被吞。"""
|
|
19
|
-
out = []
|
|
20
|
-
try:
|
|
21
|
-
import certifi
|
|
22
|
-
out.append(certifi.where())
|
|
23
|
-
except Exception:
|
|
24
|
-
pass
|
|
25
|
-
if sys.platform == "darwin":
|
|
26
|
-
out.append("/etc/ssl/cert.pem")
|
|
27
|
-
return out
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def _build():
|
|
31
|
-
ctx = ssl.create_default_context()
|
|
32
|
-
for f in cafiles():
|
|
33
|
-
try:
|
|
34
|
-
ctx.load_verify_locations(cafile=f)
|
|
35
|
-
except Exception:
|
|
36
|
-
pass
|
|
37
|
-
return ctx
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
_CTX = None
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def context():
|
|
44
|
-
"""带兜底 CA 源的 ssl.SSLContext(进程内缓存;校验语义与默认一致)。"""
|
|
45
|
-
global _CTX
|
|
46
|
-
if _CTX is None:
|
|
47
|
-
_CTX = _build()
|
|
48
|
-
return _CTX
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def humanize(err_text):
|
|
52
|
-
"""证书验证失败 → 可行动的人话提示(非证书类错误原样返回)。
|
|
53
|
-
|
|
54
|
-
兜底 CA 仍验证失败只剩两类真实原因:本机代理对 HTTPS 做 TLS 拦截
|
|
55
|
-
(其根证书只在系统钥匙串/浏览器里,Python 的静态 CA 束不认),或极端
|
|
56
|
-
环境连 /etc/ssl/cert.pem 都没有。SSL_CERT_FILE 是 OpenSSL 标准逃生门,
|
|
57
|
-
create_default_context() 本就吃它,无需额外代码。"""
|
|
58
|
-
text = str(err_text or "")
|
|
59
|
-
if "CERTIFICATE_VERIFY_FAILED" not in text:
|
|
60
|
-
return text
|
|
61
|
-
return (text + " —— 证书验证仍失败:① 若开着代理/安全软件且开启了 HTTPS 拦截,"
|
|
62
|
-
"关掉拦截,或把其根证书导出后设环境变量 SSL_CERT_FILE 指向它再启动;"
|
|
63
|
-
"② 确认系统存在 /etc/ssl/cert.pem;③ 刚升级过 CodeBee 的话,"
|
|
64
|
-
"旧服务进程可能还在跑——关掉终端窗口重开")
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""HTTPS 证书校验上下文:补齐 macOS 上 Python 缺失的 CA 源(不关校验)。
|
|
3
|
+
|
|
4
|
+
现象:用官方 pkg 装 Node 的 Mac 上往往再装 python.org 的 Python,它不读
|
|
5
|
+
系统钥匙串,默认验证路径下没有根证书,所有 HTTPS 请求报
|
|
6
|
+
CERTIFICATE_VERIFY_FAILED(unable to get local issuer certificate)。
|
|
7
|
+
|
|
8
|
+
修法是给默认上下文**追加**可用 CA 源,校验语义只增不减:
|
|
9
|
+
1. certifi(装了就用,跨平台最全);
|
|
10
|
+
2. /etc/ssl/cert.pem(macOS 系统自带 CA 束,Catalina 起就有)。
|
|
11
|
+
两个都拿不到时返回默认上下文——报错与旧行为一致,绝不静默关校验。
|
|
12
|
+
"""
|
|
13
|
+
import ssl
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def cafiles():
|
|
18
|
+
"""候选 CA 束路径;坏路径/不存在由 load_verify_locations 抛错后被吞。"""
|
|
19
|
+
out = []
|
|
20
|
+
try:
|
|
21
|
+
import certifi
|
|
22
|
+
out.append(certifi.where())
|
|
23
|
+
except Exception:
|
|
24
|
+
pass
|
|
25
|
+
if sys.platform == "darwin":
|
|
26
|
+
out.append("/etc/ssl/cert.pem")
|
|
27
|
+
return out
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _build():
|
|
31
|
+
ctx = ssl.create_default_context()
|
|
32
|
+
for f in cafiles():
|
|
33
|
+
try:
|
|
34
|
+
ctx.load_verify_locations(cafile=f)
|
|
35
|
+
except Exception:
|
|
36
|
+
pass
|
|
37
|
+
return ctx
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_CTX = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def context():
|
|
44
|
+
"""带兜底 CA 源的 ssl.SSLContext(进程内缓存;校验语义与默认一致)。"""
|
|
45
|
+
global _CTX
|
|
46
|
+
if _CTX is None:
|
|
47
|
+
_CTX = _build()
|
|
48
|
+
return _CTX
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def humanize(err_text):
|
|
52
|
+
"""证书验证失败 → 可行动的人话提示(非证书类错误原样返回)。
|
|
53
|
+
|
|
54
|
+
兜底 CA 仍验证失败只剩两类真实原因:本机代理对 HTTPS 做 TLS 拦截
|
|
55
|
+
(其根证书只在系统钥匙串/浏览器里,Python 的静态 CA 束不认),或极端
|
|
56
|
+
环境连 /etc/ssl/cert.pem 都没有。SSL_CERT_FILE 是 OpenSSL 标准逃生门,
|
|
57
|
+
create_default_context() 本就吃它,无需额外代码。"""
|
|
58
|
+
text = str(err_text or "")
|
|
59
|
+
if "CERTIFICATE_VERIFY_FAILED" not in text:
|
|
60
|
+
return text
|
|
61
|
+
return (text + " —— 证书验证仍失败:① 若开着代理/安全软件且开启了 HTTPS 拦截,"
|
|
62
|
+
"关掉拦截,或把其根证书导出后设环境变量 SSL_CERT_FILE 指向它再启动;"
|
|
63
|
+
"② 确认系统存在 /etc/ssl/cert.pem;③ 刚升级过 CodeBee 的话,"
|
|
64
|
+
"旧服务进程可能还在跑——关掉终端窗口重开")
|
package/app/core/token_meter.py
CHANGED
|
@@ -1,148 +1,148 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""Token 压力估算:按 run 累计 usage,给出「已用/容量」压力比。
|
|
3
|
-
|
|
4
|
-
设计稿:docs/migration/02-context-compaction.md §1C。
|
|
5
|
-
参考 dsh packages/llm/token-meter/src/index.ts:滑动窗口 + usage 锚点 + 失败重定价。
|
|
6
|
-
|
|
7
|
-
Tutti 简化实现:每 run 一个累计窗口(deque 有上限),usage 来自
|
|
8
|
-
runner._parse_codex_jsonl / _parse_claude_json 已解析的字段;
|
|
9
|
-
容量查 DEFAULT_CAPACITY 表(可被 data/model_capacity.json 覆盖)。
|
|
10
|
-
"""
|
|
11
|
-
from __future__ import annotations
|
|
12
|
-
|
|
13
|
-
import json
|
|
14
|
-
import logging
|
|
15
|
-
import threading
|
|
16
|
-
import time
|
|
17
|
-
from collections import deque
|
|
18
|
-
from pathlib import Path
|
|
19
|
-
|
|
20
|
-
log = logging.getLogger(__name__)
|
|
21
|
-
|
|
22
|
-
# 默认模型容量(token)。未命中 key 时用 "" 兜底行。
|
|
23
|
-
DEFAULT_CAPACITY = {
|
|
24
|
-
"": 128_000,
|
|
25
|
-
"gpt-5": 400_000,
|
|
26
|
-
"gpt-5.5": 400_000,
|
|
27
|
-
"deepseek-chat": 128_000,
|
|
28
|
-
"deepseek-reasoner": 64_000,
|
|
29
|
-
"qwen3-max": 256_000,
|
|
30
|
-
"qwen3.8-flash": 256_000,
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
# 压缩触发阈值(设计稿 §1B 默认 0.8,可被 orchestration.json 覆盖)
|
|
34
|
-
DEFAULT_PRESSURE_THRESHOLD = 0.8
|
|
35
|
-
|
|
36
|
-
# 容量覆盖文件(热改立即生效)
|
|
37
|
-
_CAPACITY_FILE = Path(__file__).resolve().parents[2] / "data" / "model_capacity.json"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def _load_capacity():
|
|
41
|
-
"""默认表 + 可选覆盖文件合并。文件缺失/坏 JSON 静默用默认。"""
|
|
42
|
-
cap = dict(DEFAULT_CAPACITY)
|
|
43
|
-
try:
|
|
44
|
-
if _CAPACITY_FILE.is_file():
|
|
45
|
-
data = json.loads(_CAPACITY_FILE.read_text(encoding="utf-8"))
|
|
46
|
-
if isinstance(data, dict):
|
|
47
|
-
cap.update({str(k): int(v) for k, v in data.items()})
|
|
48
|
-
except Exception:
|
|
49
|
-
pass
|
|
50
|
-
return cap
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class TokenMeter:
|
|
54
|
-
"""每 run 独立累计窗口;线程安全。"""
|
|
55
|
-
|
|
56
|
-
def __init__(self, capacity=None, *, window_size=200):
|
|
57
|
-
self._capacity_override = capacity
|
|
58
|
-
self._window_size = window_size
|
|
59
|
-
self._lock = threading.Lock()
|
|
60
|
-
# run_id -> deque[(ts, input, output, reasoning, cached)]
|
|
61
|
-
self._windows: dict = {}
|
|
62
|
-
# run_id -> 最后一次见到的 model 名(pressure_ratio 未显式传 model 时用)
|
|
63
|
-
self._last_model: dict = {}
|
|
64
|
-
self._cap_cache = None
|
|
65
|
-
self._cap_mtime = 0.0
|
|
66
|
-
|
|
67
|
-
def _capacity(self):
|
|
68
|
-
"""容量表(带 mtime 缓存,热改 data/model_capacity.json 立即生效)。"""
|
|
69
|
-
try:
|
|
70
|
-
mtime = _CAPACITY_FILE.stat().st_mtime
|
|
71
|
-
except OSError:
|
|
72
|
-
mtime = 0.0
|
|
73
|
-
if self._cap_cache is None or mtime != self._cap_mtime:
|
|
74
|
-
if self._capacity_override is not None:
|
|
75
|
-
self._cap_cache = dict(self._capacity_override)
|
|
76
|
-
else:
|
|
77
|
-
self._cap_cache = _load_capacity()
|
|
78
|
-
self._cap_mtime = mtime
|
|
79
|
-
return self._cap_cache
|
|
80
|
-
|
|
81
|
-
def accumulate(self, run_id: str, usage, model: str = ""):
|
|
82
|
-
"""记录一次调用的 usage。usage 字段缺失按 0 处理;usage=None 跳过。
|
|
83
|
-
|
|
84
|
-
cached 单独记录但不计入 used()——缓存读不占新一轮上下文压力
|
|
85
|
-
(与 runner._parse_claude_json 的口径一致:cached 是省钱的部分)。
|
|
86
|
-
"""
|
|
87
|
-
if not usage:
|
|
88
|
-
return
|
|
89
|
-
with self._lock:
|
|
90
|
-
if model:
|
|
91
|
-
self._last_model[run_id] = model
|
|
92
|
-
window = self._windows.setdefault(run_id, deque(maxlen=self._window_size))
|
|
93
|
-
window.append((
|
|
94
|
-
time.time(),
|
|
95
|
-
int(usage.get("input") or 0),
|
|
96
|
-
int(usage.get("output") or 0),
|
|
97
|
-
int(usage.get("reasoning") or 0),
|
|
98
|
-
int(usage.get("cached") or 0),
|
|
99
|
-
))
|
|
100
|
-
|
|
101
|
-
def used(self, run_id: str) -> int:
|
|
102
|
-
"""窗口内累计(input + output + reasoning),不含 cached。"""
|
|
103
|
-
with self._lock:
|
|
104
|
-
window = self._windows.get(run_id)
|
|
105
|
-
if not window:
|
|
106
|
-
return 0
|
|
107
|
-
return sum(i + o + r for _, i, o, r, _ in window)
|
|
108
|
-
|
|
109
|
-
def cached(self, run_id: str) -> int:
|
|
110
|
-
with self._lock:
|
|
111
|
-
window = self._windows.get(run_id)
|
|
112
|
-
if not window:
|
|
113
|
-
return 0
|
|
114
|
-
return sum(c for _, _, _, _, c in window)
|
|
115
|
-
|
|
116
|
-
def pressure_ratio(self, run_id: str, model: str = "") -> float:
|
|
117
|
-
"""已用 / 容量。model 未传时用该 run 最后一次见到的模型名。"""
|
|
118
|
-
cap_table = self._capacity()
|
|
119
|
-
with self._lock:
|
|
120
|
-
m = model or self._last_model.get(run_id, "")
|
|
121
|
-
cap = cap_table.get(m, cap_table.get("", 128_000))
|
|
122
|
-
return self.used(run_id) / max(cap, 1)
|
|
123
|
-
|
|
124
|
-
def capacity(self, model: str = "") -> int:
|
|
125
|
-
"""模型上下文容量(未命中用默认行)。"""
|
|
126
|
-
cap_table = self._capacity()
|
|
127
|
-
return cap_table.get(model or "", cap_table.get("", 128_000))
|
|
128
|
-
|
|
129
|
-
def last_context(self, run_id: str) -> int:
|
|
130
|
-
"""最近一次调用的 input+output+reasoning——下个请求要重发的上下文锚点。
|
|
131
|
-
|
|
132
|
-
used() 是窗口累计(多次调用叠加,系统性偏大,适合做响应式预算闸);
|
|
133
|
-
事前预检判断「这个请求会不会被拒」要用最近一次的真实上下文体量,
|
|
134
|
-
两者口径不同不可混用(借鉴 freebuff base-chat 每步容量重估)。"""
|
|
135
|
-
with self._lock:
|
|
136
|
-
window = self._windows.get(run_id)
|
|
137
|
-
if not window:
|
|
138
|
-
return 0
|
|
139
|
-
_, i, o, r, _ = window[-1]
|
|
140
|
-
return i + o + r
|
|
141
|
-
|
|
142
|
-
def reset(self, run_id: str):
|
|
143
|
-
with self._lock:
|
|
144
|
-
self._windows.pop(run_id, None)
|
|
145
|
-
self._last_model.pop(run_id, None)
|
|
146
|
-
|
|
147
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Token 压力估算:按 run 累计 usage,给出「已用/容量」压力比。
|
|
3
|
+
|
|
4
|
+
设计稿:docs/migration/02-context-compaction.md §1C。
|
|
5
|
+
参考 dsh packages/llm/token-meter/src/index.ts:滑动窗口 + usage 锚点 + 失败重定价。
|
|
6
|
+
|
|
7
|
+
Tutti 简化实现:每 run 一个累计窗口(deque 有上限),usage 来自
|
|
8
|
+
runner._parse_codex_jsonl / _parse_claude_json 已解析的字段;
|
|
9
|
+
容量查 DEFAULT_CAPACITY 表(可被 data/model_capacity.json 覆盖)。
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import logging
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from collections import deque
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
# 默认模型容量(token)。未命中 key 时用 "" 兜底行。
|
|
23
|
+
DEFAULT_CAPACITY = {
|
|
24
|
+
"": 128_000,
|
|
25
|
+
"gpt-5": 400_000,
|
|
26
|
+
"gpt-5.5": 400_000,
|
|
27
|
+
"deepseek-chat": 128_000,
|
|
28
|
+
"deepseek-reasoner": 64_000,
|
|
29
|
+
"qwen3-max": 256_000,
|
|
30
|
+
"qwen3.8-flash": 256_000,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
# 压缩触发阈值(设计稿 §1B 默认 0.8,可被 orchestration.json 覆盖)
|
|
34
|
+
DEFAULT_PRESSURE_THRESHOLD = 0.8
|
|
35
|
+
|
|
36
|
+
# 容量覆盖文件(热改立即生效)
|
|
37
|
+
_CAPACITY_FILE = Path(__file__).resolve().parents[2] / "data" / "model_capacity.json"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _load_capacity():
|
|
41
|
+
"""默认表 + 可选覆盖文件合并。文件缺失/坏 JSON 静默用默认。"""
|
|
42
|
+
cap = dict(DEFAULT_CAPACITY)
|
|
43
|
+
try:
|
|
44
|
+
if _CAPACITY_FILE.is_file():
|
|
45
|
+
data = json.loads(_CAPACITY_FILE.read_text(encoding="utf-8"))
|
|
46
|
+
if isinstance(data, dict):
|
|
47
|
+
cap.update({str(k): int(v) for k, v in data.items()})
|
|
48
|
+
except Exception:
|
|
49
|
+
pass
|
|
50
|
+
return cap
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TokenMeter:
|
|
54
|
+
"""每 run 独立累计窗口;线程安全。"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, capacity=None, *, window_size=200):
|
|
57
|
+
self._capacity_override = capacity
|
|
58
|
+
self._window_size = window_size
|
|
59
|
+
self._lock = threading.Lock()
|
|
60
|
+
# run_id -> deque[(ts, input, output, reasoning, cached)]
|
|
61
|
+
self._windows: dict = {}
|
|
62
|
+
# run_id -> 最后一次见到的 model 名(pressure_ratio 未显式传 model 时用)
|
|
63
|
+
self._last_model: dict = {}
|
|
64
|
+
self._cap_cache = None
|
|
65
|
+
self._cap_mtime = 0.0
|
|
66
|
+
|
|
67
|
+
def _capacity(self):
|
|
68
|
+
"""容量表(带 mtime 缓存,热改 data/model_capacity.json 立即生效)。"""
|
|
69
|
+
try:
|
|
70
|
+
mtime = _CAPACITY_FILE.stat().st_mtime
|
|
71
|
+
except OSError:
|
|
72
|
+
mtime = 0.0
|
|
73
|
+
if self._cap_cache is None or mtime != self._cap_mtime:
|
|
74
|
+
if self._capacity_override is not None:
|
|
75
|
+
self._cap_cache = dict(self._capacity_override)
|
|
76
|
+
else:
|
|
77
|
+
self._cap_cache = _load_capacity()
|
|
78
|
+
self._cap_mtime = mtime
|
|
79
|
+
return self._cap_cache
|
|
80
|
+
|
|
81
|
+
def accumulate(self, run_id: str, usage, model: str = ""):
|
|
82
|
+
"""记录一次调用的 usage。usage 字段缺失按 0 处理;usage=None 跳过。
|
|
83
|
+
|
|
84
|
+
cached 单独记录但不计入 used()——缓存读不占新一轮上下文压力
|
|
85
|
+
(与 runner._parse_claude_json 的口径一致:cached 是省钱的部分)。
|
|
86
|
+
"""
|
|
87
|
+
if not usage:
|
|
88
|
+
return
|
|
89
|
+
with self._lock:
|
|
90
|
+
if model:
|
|
91
|
+
self._last_model[run_id] = model
|
|
92
|
+
window = self._windows.setdefault(run_id, deque(maxlen=self._window_size))
|
|
93
|
+
window.append((
|
|
94
|
+
time.time(),
|
|
95
|
+
int(usage.get("input") or 0),
|
|
96
|
+
int(usage.get("output") or 0),
|
|
97
|
+
int(usage.get("reasoning") or 0),
|
|
98
|
+
int(usage.get("cached") or 0),
|
|
99
|
+
))
|
|
100
|
+
|
|
101
|
+
def used(self, run_id: str) -> int:
|
|
102
|
+
"""窗口内累计(input + output + reasoning),不含 cached。"""
|
|
103
|
+
with self._lock:
|
|
104
|
+
window = self._windows.get(run_id)
|
|
105
|
+
if not window:
|
|
106
|
+
return 0
|
|
107
|
+
return sum(i + o + r for _, i, o, r, _ in window)
|
|
108
|
+
|
|
109
|
+
def cached(self, run_id: str) -> int:
|
|
110
|
+
with self._lock:
|
|
111
|
+
window = self._windows.get(run_id)
|
|
112
|
+
if not window:
|
|
113
|
+
return 0
|
|
114
|
+
return sum(c for _, _, _, _, c in window)
|
|
115
|
+
|
|
116
|
+
def pressure_ratio(self, run_id: str, model: str = "") -> float:
|
|
117
|
+
"""已用 / 容量。model 未传时用该 run 最后一次见到的模型名。"""
|
|
118
|
+
cap_table = self._capacity()
|
|
119
|
+
with self._lock:
|
|
120
|
+
m = model or self._last_model.get(run_id, "")
|
|
121
|
+
cap = cap_table.get(m, cap_table.get("", 128_000))
|
|
122
|
+
return self.used(run_id) / max(cap, 1)
|
|
123
|
+
|
|
124
|
+
def capacity(self, model: str = "") -> int:
|
|
125
|
+
"""模型上下文容量(未命中用默认行)。"""
|
|
126
|
+
cap_table = self._capacity()
|
|
127
|
+
return cap_table.get(model or "", cap_table.get("", 128_000))
|
|
128
|
+
|
|
129
|
+
def last_context(self, run_id: str) -> int:
|
|
130
|
+
"""最近一次调用的 input+output+reasoning——下个请求要重发的上下文锚点。
|
|
131
|
+
|
|
132
|
+
used() 是窗口累计(多次调用叠加,系统性偏大,适合做响应式预算闸);
|
|
133
|
+
事前预检判断「这个请求会不会被拒」要用最近一次的真实上下文体量,
|
|
134
|
+
两者口径不同不可混用(借鉴 freebuff base-chat 每步容量重估)。"""
|
|
135
|
+
with self._lock:
|
|
136
|
+
window = self._windows.get(run_id)
|
|
137
|
+
if not window:
|
|
138
|
+
return 0
|
|
139
|
+
_, i, o, r, _ = window[-1]
|
|
140
|
+
return i + o + r
|
|
141
|
+
|
|
142
|
+
def reset(self, run_id: str):
|
|
143
|
+
with self._lock:
|
|
144
|
+
self._windows.pop(run_id, None)
|
|
145
|
+
self._last_model.pop(run_id, None)
|
|
146
|
+
|
|
147
|
+
|
|
148
148
|
token_meter = TokenMeter() # 单例
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Shared upstream identity normalization for routing and retry guards."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ipaddress
|
|
6
|
+
from urllib.parse import urlsplit
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def normalize_upstream(value, default_scheme="https"):
|
|
10
|
+
"""Return a stable host[:port] identity, without URL path or credentials.
|
|
11
|
+
|
|
12
|
+
Host names are case-insensitive and a trailing dot is equivalent to the
|
|
13
|
+
non-dotted form. Default HTTP(S) ports are omitted so equivalent URLs share
|
|
14
|
+
one failure/permission bucket. Non-default ports remain distinct.
|
|
15
|
+
"""
|
|
16
|
+
raw = str(value or "").strip()
|
|
17
|
+
if not raw:
|
|
18
|
+
return ""
|
|
19
|
+
candidate = raw if "://" in raw else "%s://%s" % (default_scheme, raw)
|
|
20
|
+
try:
|
|
21
|
+
parsed = urlsplit(candidate)
|
|
22
|
+
host = parsed.hostname
|
|
23
|
+
if not host:
|
|
24
|
+
return ""
|
|
25
|
+
host = host.rstrip(".").lower()
|
|
26
|
+
ip_host = None
|
|
27
|
+
try:
|
|
28
|
+
ip_host = ipaddress.ip_address(host)
|
|
29
|
+
host = ip_host.compressed.lower()
|
|
30
|
+
except ValueError:
|
|
31
|
+
pass
|
|
32
|
+
port = parsed.port
|
|
33
|
+
scheme = (parsed.scheme or default_scheme).lower()
|
|
34
|
+
if port is not None and not ((scheme == "https" and port == 443)
|
|
35
|
+
or (scheme == "http" and port == 80)):
|
|
36
|
+
if getattr(ip_host, "version", None) == 6:
|
|
37
|
+
return "[%s]:%d" % (host, port)
|
|
38
|
+
return "%s:%d" % (host, port)
|
|
39
|
+
return host
|
|
40
|
+
except (TypeError, ValueError):
|
|
41
|
+
return ""
|
package/app/core/usage.py
CHANGED
|
@@ -30,7 +30,8 @@ DIMENSIONS = {
|
|
|
30
30
|
|
|
31
31
|
FIELDS = ("ts", "day", "run_id", "step", "task_id", "task_type", "role", "agent", "agent_label",
|
|
32
32
|
"tool", "model", "provider", "ok", "duration_s",
|
|
33
|
-
"input", "output", "cached", "reasoning", "total", "cost_usd", "source"
|
|
33
|
+
"input", "output", "cached", "reasoning", "total", "cost_usd", "source",
|
|
34
|
+
"first_token_ms", "tokens_per_sec")
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
def _month_file(day):
|
|
@@ -63,7 +64,8 @@ def _parse_bool(v):
|
|
|
63
64
|
|
|
64
65
|
def record(source="", run_id="", task_id="", task_type="", role="", step=0,
|
|
65
66
|
agent="", agent_label="", tool="", model="", provider="",
|
|
66
|
-
ok=True, duration_s=0.0, cost_usd=0.0, usage=None
|
|
67
|
+
ok=True, duration_s=0.0, cost_usd=0.0, usage=None,
|
|
68
|
+
first_token_ms=0.0, tokens_per_sec=0.0):
|
|
67
69
|
"""追加一条用量记录。usage 为细分 dict:{input, output, cached, reasoning, total};
|
|
68
70
|
缺省字段按 0 处理。任何异常都吞掉——统计永远不能拖垮业务调用方。
|
|
69
71
|
|
|
@@ -97,6 +99,19 @@ def record(source="", run_id="", task_id="", task_type="", role="", step=0,
|
|
|
97
99
|
total = _parse_int(u.get("total")) or (inp + out + cach)
|
|
98
100
|
rec.update({"input": inp, "output": out, "cached": cach,
|
|
99
101
|
"reasoning": reas, "total": total})
|
|
102
|
+
# 压缩省量注记(source=compaction 专用):上下文折叠净省的估算 token,
|
|
103
|
+
# 与消耗并列成账——省了多少不再是黑箱。零值不落字段防老记录膨胀。
|
|
104
|
+
saved = max(0, _parse_int(u.get("saved")))
|
|
105
|
+
if saved:
|
|
106
|
+
rec["saved"] = saved
|
|
107
|
+
# 体验指标(§07):只有内置直连的流式路径可测,CLI 事件流没有逐 token
|
|
108
|
+
# 时刻。零值不落字段——"缺字段"表示不可测,不能读成"首字 0 毫秒"。
|
|
109
|
+
ft = round(_parse_float(first_token_ms), 1)
|
|
110
|
+
if ft > 0:
|
|
111
|
+
rec["first_token_ms"] = ft
|
|
112
|
+
tps = round(_parse_float(tokens_per_sec), 2)
|
|
113
|
+
if tps > 0:
|
|
114
|
+
rec["tokens_per_sec"] = tps
|
|
100
115
|
line = json.dumps(rec, ensure_ascii=False)
|
|
101
116
|
with LOCK:
|
|
102
117
|
paths.USAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
@@ -638,6 +653,21 @@ def summary(days=30, recent_limit=30):
|
|
|
638
653
|
totals["cache_rate"] = round(totals["cached"] * 100.0 / denom, 1) if denom else 0.0
|
|
639
654
|
totals["peak_tokens"] = max((bd["tokens"] for bd in by_day), default=0)
|
|
640
655
|
totals["max_duration_s"] = round(max((float(r.get("duration_s") or 0.0) for r in records), default=0.0), 1)
|
|
656
|
+
# 压缩省量(source=compaction 记录的 saved 合计):与消耗并列的第七维,
|
|
657
|
+
# 台账从此能答「省了多少」;无压缩记录时为 0,老前端不读不受影响
|
|
658
|
+
totals["compaction_saved"] = sum(_num(r, "saved") for r in records)
|
|
659
|
+
# 体验指标:只统计可测记录(缺失与 0 都表示"这条没测到",不参与均值),
|
|
660
|
+
# 否则 CLI 路径会把首字延迟平均数拉到 0,读数即假数。
|
|
661
|
+
ft_list = sorted(_parse_float(r.get("first_token_ms")) for r in records
|
|
662
|
+
if _parse_float(r.get("first_token_ms")) > 0)
|
|
663
|
+
tps_list = [_parse_float(r.get("tokens_per_sec")) for r in records
|
|
664
|
+
if _parse_float(r.get("tokens_per_sec")) > 0]
|
|
665
|
+
totals["perf_samples"] = len(ft_list)
|
|
666
|
+
totals["avg_first_token_ms"] = (round(sum(ft_list) / len(ft_list), 1)
|
|
667
|
+
if ft_list else 0.0)
|
|
668
|
+
totals["p95_first_token_ms"] = _percentile(ft_list, 0.95) if ft_list else 0.0
|
|
669
|
+
totals["avg_tokens_per_sec"] = (round(sum(tps_list) / len(tps_list), 2)
|
|
670
|
+
if tps_list else 0.0)
|
|
641
671
|
cur, lng = _streaks(str(r.get("day") or "") for r in all_records)
|
|
642
672
|
totals["streak_current"] = cur
|
|
643
673
|
totals["streak_longest"] = lng
|
|
@@ -659,6 +689,8 @@ def summary(days=30, recent_limit=30):
|
|
|
659
689
|
}
|
|
660
690
|
|
|
661
691
|
|
|
692
|
+
THIN_SAMPLES = 10 # 低于此样本数即「薄样本」,口径见执行标准第 6 条
|
|
693
|
+
|
|
662
694
|
_DURATION_BASELINES = {
|
|
663
695
|
"direct": 90, "rank_scan": 240, "email": 180, "weekly_report": 240,
|
|
664
696
|
"translation": 240, "speech": 360, "video_script": 360, "article": 480,
|
|
@@ -672,7 +704,9 @@ def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=Non
|
|
|
672
704
|
|
|
673
705
|
同一 run 的多条步骤记录加总为一个样本,优先用成功 run,给中位数/平均/P90。
|
|
674
706
|
无历史样本时使用流程级保守基线;模式与思考程度只做透明倍率修正。"""
|
|
675
|
-
|
|
707
|
+
# days<=0(UI 的「全部」档)不能退化成「只扫今天」:基线会跟着窗口一起坍缩,
|
|
708
|
+
# 于是全历史被估成 480 秒常数。按保守年限扫全量。
|
|
709
|
+
span = 3650 if days is not None and int(days) <= 0 else max(1, min(3650, int(days or 90)))
|
|
676
710
|
with LOCK:
|
|
677
711
|
records = _iter_records(span)
|
|
678
712
|
runs = {}
|
|
@@ -703,11 +737,16 @@ def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=Non
|
|
|
703
737
|
except (TypeError, ValueError):
|
|
704
738
|
pass
|
|
705
739
|
if not basis:
|
|
740
|
+
# 全历史里一条同类样本都没有(article/doc 等低频类型)时才用基线,
|
|
741
|
+
# 并在 duration_source 上如实标 baseline,UI 必须点名它不是实测。
|
|
706
742
|
baseline = _DURATION_BASELINES.get(task_type, 480)
|
|
707
743
|
median_s = max(30, int(baseline * factor))
|
|
708
744
|
return {"task_type": task_type or "", "days": span, "samples": 0,
|
|
709
745
|
"estimated_duration_s": median_s,
|
|
710
746
|
"p90_duration_s": max(median_s + 30, int(median_s * 1.7)),
|
|
747
|
+
# 标准「成本与时延预算」第 6 条:样本 <10 即薄样本,UI 必须点名,
|
|
748
|
+
# 不能把"仅此一例"的历史当实测承诺。
|
|
749
|
+
"thin": True, "min_samples_for_estimate": THIN_SAMPLES,
|
|
711
750
|
"duration_source": "baseline"}
|
|
712
751
|
toks = sorted(g["tokens"] for g in basis)
|
|
713
752
|
costs = sorted(g["cost"] for g in basis)
|
|
@@ -742,6 +781,78 @@ def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=Non
|
|
|
742
781
|
"estimated_duration_s": max(30, int(dmed * factor)),
|
|
743
782
|
"p90_duration_s": max(60, int(dp90 * factor)),
|
|
744
783
|
"duration_samples": len(durations),
|
|
784
|
+
"thin": n < THIN_SAMPLES, "min_samples_for_estimate": THIN_SAMPLES,
|
|
745
785
|
"duration_source": duration_source,
|
|
746
786
|
"since": (datetime.date.today() - datetime.timedelta(days=span - 1)).isoformat(),
|
|
747
787
|
}
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _audit_file(day):
|
|
791
|
+
return paths.USAGE_DIR / ("estimate-audit-%s.jsonl" % day[:7].replace("-", ""))
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
_AUDITED = {} # run_id -> 落盘路径;幂等去重,上限后整体清空防无界增长
|
|
795
|
+
_AUDIT_LIMIT = 4000
|
|
796
|
+
TERMINAL_STATUSES = ("done", "failed", "cancelled", "timeout")
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def audit_run(run):
|
|
800
|
+
"""事后对账:开跑前的预估快照 vs 本次真实用量,一条 run 一行。
|
|
801
|
+
|
|
802
|
+
预估值只取 run 上的快照(store.create_run 落的 estimated_duration_s 等),
|
|
803
|
+
绝不回读当前 estimate()——那会把本次刚写入的消耗算进"预估",比值永远偏低。
|
|
804
|
+
快照缺失(回填前的老 run、非编排类)时如实标 basis=none,不猜一个数充数。
|
|
805
|
+
"""
|
|
806
|
+
try:
|
|
807
|
+
run_id = str((run or {}).get("id") or "")
|
|
808
|
+
if not run_id or run_id in _AUDITED:
|
|
809
|
+
return None
|
|
810
|
+
status = str(run.get("status") or "")
|
|
811
|
+
if status not in TERMINAL_STATUSES:
|
|
812
|
+
return None
|
|
813
|
+
task_type = str(run.get("task_type") or (run.get("op") or {}).get("type") or "")
|
|
814
|
+
est_s = _parse_float(run.get("estimated_duration_s"))
|
|
815
|
+
est_p90 = _parse_float(run.get("estimated_p90_s"))
|
|
816
|
+
basis = "stored"
|
|
817
|
+
if est_s <= 0:
|
|
818
|
+
# 快照缺失时退而用同类型历史口径,但必须落 basis=recomputed 标出来源
|
|
819
|
+
snap = estimate(task_type=task_type, days=(run.get("estimate_days") or 90))
|
|
820
|
+
est_s = _parse_float(snap.get("estimated_duration_s"))
|
|
821
|
+
est_p90 = _parse_float(snap.get("p90_duration_s"))
|
|
822
|
+
basis = "recomputed" if est_s > 0 else "none"
|
|
823
|
+
actual_s = 0.0
|
|
824
|
+
actual_tokens = 0
|
|
825
|
+
for r in _iter_records(366):
|
|
826
|
+
if str(r.get("run_id") or "") != run_id:
|
|
827
|
+
continue
|
|
828
|
+
actual_s += max(0.0, _parse_float(r.get("duration_s")))
|
|
829
|
+
actual_tokens += _parse_int(r.get("total"))
|
|
830
|
+
rec = {
|
|
831
|
+
"ts": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
832
|
+
"day": time.strftime("%Y-%m-%d"),
|
|
833
|
+
"run_id": run_id[:64],
|
|
834
|
+
"task_id": str(run.get("task_id") or "")[:64],
|
|
835
|
+
"task_type": task_type[:32] or "unknown",
|
|
836
|
+
"status": status[:16],
|
|
837
|
+
"est_duration_s": round(est_s, 1),
|
|
838
|
+
"est_p90_s": round(est_p90, 1),
|
|
839
|
+
"est_source": str(run.get("estimate_source") or "")[:16],
|
|
840
|
+
"est_samples": _parse_int(run.get("estimate_samples")),
|
|
841
|
+
"est_basis": basis[:16],
|
|
842
|
+
"actual_duration_s": round(actual_s, 1),
|
|
843
|
+
"actual_tokens": actual_tokens,
|
|
844
|
+
"ratio": round(actual_s / est_s, 3) if est_s > 0 else 0.0,
|
|
845
|
+
"over_p90": bool(est_p90 > 0 and actual_s > est_p90),
|
|
846
|
+
}
|
|
847
|
+
line = json.dumps(rec, ensure_ascii=False)
|
|
848
|
+
path = _audit_file(rec["day"])
|
|
849
|
+
with LOCK:
|
|
850
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
851
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
852
|
+
f.write(line + "\n")
|
|
853
|
+
if len(_AUDITED) >= _AUDIT_LIMIT:
|
|
854
|
+
_AUDITED.clear()
|
|
855
|
+
_AUDITED[run_id] = str(path)
|
|
856
|
+
return rec
|
|
857
|
+
except Exception:
|
|
858
|
+
return None
|