codebee 0.1.46 → 0.1.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +395 -388
- package/app/core/flows.py +12 -3
- package/app/core/hooks.py +156 -0
- package/app/core/pipeline.py +149 -4
- package/app/core/planner.py +82 -11
- package/app/core/store.py +68 -1
- package/app/core/usage.py +290 -286
- package/app/core/volumes.py +353 -0
- package/app/core/zentao.py +9 -4
- package/app/main.py +38 -0
- package/app/ui/app.js +405 -16
- package/app/ui/i18n.js +26 -0
- package/app/ui/index.html +72 -0
- package/app/ui/style.css +160 -0
- package/package.json +2 -1
- package/tools/wx_reader.py +261 -0
package/app/core/usage.py
CHANGED
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
"""
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
|
-
import json
|
|
14
|
-
import math
|
|
15
|
-
import threading
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import threading
|
|
16
16
|
import time
|
|
17
17
|
|
|
18
18
|
from . import paths
|
|
@@ -33,12 +33,12 @@ FIELDS = ("ts", "day", "run_id", "step", "task_id", "task_type", "role", "agent"
|
|
|
33
33
|
"input", "output", "cached", "reasoning", "total", "cost_usd", "source")
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def _month_file(day):
|
|
37
|
-
return paths.USAGE_DIR / ("usage-%s.jsonl" % day[:7].replace("-", ""))
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def _quality_file(day):
|
|
41
|
-
return paths.USAGE_DIR / ("routing-quality-%s.jsonl" % day[:7].replace("-", ""))
|
|
36
|
+
def _month_file(day):
|
|
37
|
+
return paths.USAGE_DIR / ("usage-%s.jsonl" % day[:7].replace("-", ""))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _quality_file(day):
|
|
41
|
+
return paths.USAGE_DIR / ("routing-quality-%s.jsonl" % day[:7].replace("-", ""))
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
def _parse_int(v):
|
|
@@ -98,13 +98,13 @@ def record(source="", run_id="", task_id="", task_type="", role="", step=0,
|
|
|
98
98
|
rec.update({"input": inp, "output": out, "cached": cach,
|
|
99
99
|
"reasoning": reas, "total": total})
|
|
100
100
|
line = json.dumps(rec, ensure_ascii=False)
|
|
101
|
-
with LOCK:
|
|
102
|
-
paths.USAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
103
|
-
with open(_month_file(rec["day"]), "a", encoding="utf-8") as f:
|
|
104
|
-
f.write(line + "\n")
|
|
105
|
-
_ROUTING_CACHE["data"].clear()
|
|
106
|
-
_ROUTING_RECORDS_CACHE.clear()
|
|
107
|
-
_HOURLY_CACHE["ts"] = 0.0
|
|
101
|
+
with LOCK:
|
|
102
|
+
paths.USAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
103
|
+
with open(_month_file(rec["day"]), "a", encoding="utf-8") as f:
|
|
104
|
+
f.write(line + "\n")
|
|
105
|
+
_ROUTING_CACHE["data"].clear()
|
|
106
|
+
_ROUTING_RECORDS_CACHE.clear()
|
|
107
|
+
_HOURLY_CACHE["ts"] = 0.0
|
|
108
108
|
except Exception:
|
|
109
109
|
pass
|
|
110
110
|
|
|
@@ -204,11 +204,11 @@ def _tool_of(agent_id):
|
|
|
204
204
|
return a or "unknown"
|
|
205
205
|
|
|
206
206
|
|
|
207
|
-
_HOURLY_CACHE = {"ts": 0.0, "val": {}}
|
|
208
|
-
_HOURLY_TTL = 60.0 # 秒:路由调用频繁但台账追加低频,60s 缓存足够新鲜
|
|
209
|
-
_ROUTING_CACHE = {"data": {}}
|
|
210
|
-
_ROUTING_RECORDS_CACHE = {}
|
|
211
|
-
_ROUTING_TTL = 15.0
|
|
207
|
+
_HOURLY_CACHE = {"ts": 0.0, "val": {}}
|
|
208
|
+
_HOURLY_TTL = 60.0 # 秒:路由调用频繁但台账追加低频,60s 缓存足够新鲜
|
|
209
|
+
_ROUTING_CACHE = {"data": {}}
|
|
210
|
+
_ROUTING_RECORDS_CACHE = {}
|
|
211
|
+
_ROUTING_TTL = 15.0
|
|
212
212
|
|
|
213
213
|
|
|
214
214
|
def agent_tokens_recent(agent, hours=1):
|
|
@@ -225,17 +225,17 @@ def agent_tokens_recent(agent, hours=1):
|
|
|
225
225
|
if str(r.get("ts") or "") < bound:
|
|
226
226
|
continue
|
|
227
227
|
a = str(r.get("agent") or "")
|
|
228
|
-
# record() 将规范化后的 token 总数落在顶层 ``total``;旧实现
|
|
229
|
-
# 误读不存在的嵌套 usage 字段,导致配额路由永远认为本小时
|
|
230
|
-
# 用量为 0,超过供应商额度也不会降权。
|
|
231
|
-
total[a] = total.get(a, 0) + max(0, _parse_int(r.get("total")))
|
|
228
|
+
# record() 将规范化后的 token 总数落在顶层 ``total``;旧实现
|
|
229
|
+
# 误读不存在的嵌套 usage 字段,导致配额路由永远认为本小时
|
|
230
|
+
# 用量为 0,超过供应商额度也不会降权。
|
|
231
|
+
total[a] = total.get(a, 0) + max(0, _parse_int(r.get("total")))
|
|
232
232
|
_HOURLY_CACHE.update(ts=now, val=total)
|
|
233
233
|
return int(_HOURLY_CACHE["val"].get(agent) or 0)
|
|
234
234
|
except Exception:
|
|
235
235
|
return 0
|
|
236
236
|
|
|
237
237
|
|
|
238
|
-
def _iter_records(days):
|
|
238
|
+
def _iter_records(days):
|
|
239
239
|
"""按时间范围读取台账(days=0 表示全部)。返回按写入顺序的记录列表。"""
|
|
240
240
|
out = []
|
|
241
241
|
try:
|
|
@@ -243,14 +243,14 @@ def _iter_records(days):
|
|
|
243
243
|
except Exception:
|
|
244
244
|
return out
|
|
245
245
|
since_day = ""
|
|
246
|
-
if days:
|
|
247
|
-
import datetime
|
|
248
|
-
d = (datetime.date.today() - datetime.timedelta(days=int(days) - 1)).isoformat()
|
|
249
|
-
since_day = d
|
|
250
|
-
since_month = since_day[:7].replace("-", "") if since_day else ""
|
|
251
|
-
for p in files:
|
|
252
|
-
if since_month and p.stem.rsplit("-", 1)[-1] < since_month:
|
|
253
|
-
continue
|
|
246
|
+
if days:
|
|
247
|
+
import datetime
|
|
248
|
+
d = (datetime.date.today() - datetime.timedelta(days=int(days) - 1)).isoformat()
|
|
249
|
+
since_day = d
|
|
250
|
+
since_month = since_day[:7].replace("-", "") if since_day else ""
|
|
251
|
+
for p in files:
|
|
252
|
+
if since_month and p.stem.rsplit("-", 1)[-1] < since_month:
|
|
253
|
+
continue
|
|
254
254
|
try:
|
|
255
255
|
for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
256
256
|
line = line.strip()
|
|
@@ -267,202 +267,202 @@ def _iter_records(days):
|
|
|
267
267
|
out.append(r)
|
|
268
268
|
except Exception:
|
|
269
269
|
continue
|
|
270
|
-
return out
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
def _iter_quality_records(days):
|
|
274
|
-
"""读取独立质量台账;不混入用量统计的调用数、token 和成本。"""
|
|
275
|
-
out = []
|
|
276
|
-
try:
|
|
277
|
-
files = sorted(paths.USAGE_DIR.glob("routing-quality-*.jsonl")) \
|
|
278
|
-
if paths.USAGE_DIR.is_dir() else []
|
|
279
|
-
except Exception:
|
|
280
|
-
return out
|
|
281
|
-
since_day = ""
|
|
282
|
-
if days:
|
|
283
|
-
import datetime
|
|
284
|
-
since_day = (datetime.date.today()
|
|
285
|
-
- datetime.timedelta(days=int(days) - 1)).isoformat()
|
|
286
|
-
since_month = since_day[:7].replace("-", "") if since_day else ""
|
|
287
|
-
for path in files:
|
|
288
|
-
if since_month and path.stem.rsplit("-", 1)[-1] < since_month:
|
|
289
|
-
continue
|
|
290
|
-
try:
|
|
291
|
-
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
292
|
-
try:
|
|
293
|
-
row = json.loads(line)
|
|
294
|
-
except Exception:
|
|
295
|
-
continue
|
|
296
|
-
if isinstance(row, dict) and (not since_day or row.get("day", "") >= since_day):
|
|
297
|
-
out.append(row)
|
|
298
|
-
except Exception:
|
|
299
|
-
continue
|
|
300
|
-
return out
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
def _routing_records(days):
|
|
304
|
-
"""同一轮候选评分共享一次台账解析,避免每个候选重复扫全盘。"""
|
|
305
|
-
key = (str(paths.USAGE_DIR), int(days))
|
|
306
|
-
now = time.time()
|
|
307
|
-
with LOCK:
|
|
308
|
-
cached = _ROUTING_RECORDS_CACHE.get(key)
|
|
309
|
-
if cached and now - cached[0] <= _ROUTING_TTL:
|
|
310
|
-
return list(cached[1]), list(cached[2])
|
|
311
|
-
calls, quality = _iter_records(days), _iter_quality_records(days)
|
|
312
|
-
with LOCK:
|
|
313
|
-
_ROUTING_RECORDS_CACHE[key] = (now, list(calls), list(quality))
|
|
314
|
-
return calls, quality
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
def record_quality_for_run(run_id, quality_ok, agent=""):
|
|
318
|
-
"""把验收结论归因到最终实际产出者,不重复计入用量。"""
|
|
319
|
-
try:
|
|
320
|
-
run_id = str(run_id or "")[:64]
|
|
321
|
-
if not run_id:
|
|
322
|
-
return 0
|
|
323
|
-
calls = [r for r in _iter_records(2)
|
|
324
|
-
if r.get("run_id") == run_id and r.get("ok") is True]
|
|
325
|
-
prefixes = ("implement", "fix", "draft", "revise", "polish", "direct", "author")
|
|
326
|
-
calls = [r for r in calls if str(r.get("role") or "").lower().startswith(prefixes)]
|
|
327
|
-
agent = str(agent or "")[:40]
|
|
328
|
-
unique = {}
|
|
329
|
-
for row in calls:
|
|
330
|
-
key = tuple(str(row.get(k) or "") for k in
|
|
331
|
-
("task_id", "task_type", "role", "agent", "model", "provider"))
|
|
332
|
-
unique[key] = row
|
|
333
|
-
if not unique:
|
|
334
|
-
return 0
|
|
335
|
-
existing = {(r.get("run_id"), r.get("role"), r.get("agent"),
|
|
336
|
-
r.get("model"), r.get("provider"))
|
|
337
|
-
for r in _iter_quality_records(2)}
|
|
338
|
-
day = time.strftime("%Y-%m-%d")
|
|
339
|
-
rows = []
|
|
340
|
-
for row in unique.values():
|
|
341
|
-
identity = (run_id, row.get("role"), row.get("agent"),
|
|
342
|
-
row.get("model"), row.get("provider"))
|
|
343
|
-
if identity in existing:
|
|
344
|
-
continue
|
|
345
|
-
# 成功调用不等于产出通过。换将前的实现者即使传输成功,也已被质量
|
|
346
|
-
# 门淘汰,必须留下负样本;最终实际产出者才继承本次验收结论。
|
|
347
|
-
row_quality_ok = bool(quality_ok)
|
|
348
|
-
if agent and str(row.get("agent") or "") != agent:
|
|
349
|
-
row_quality_ok = False
|
|
350
|
-
rows.append({"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "day": day,
|
|
351
|
-
"run_id": run_id, "task_id": row.get("task_id") or "",
|
|
352
|
-
"task_type": row.get("task_type") or "unknown",
|
|
353
|
-
"role": row.get("role") or "unknown",
|
|
354
|
-
"agent": row.get("agent") or "unknown",
|
|
355
|
-
"model": row.get("model") or "(默认)",
|
|
356
|
-
"provider": row.get("provider") or "",
|
|
357
|
-
"quality_ok": row_quality_ok})
|
|
358
|
-
if not rows:
|
|
359
|
-
return 0
|
|
360
|
-
with LOCK:
|
|
361
|
-
paths.USAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
362
|
-
with open(_quality_file(day), "a", encoding="utf-8") as handle:
|
|
363
|
-
for row in rows:
|
|
364
|
-
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
365
|
-
_ROUTING_CACHE["data"].clear()
|
|
366
|
-
_ROUTING_RECORDS_CACHE.clear()
|
|
367
|
-
return len(rows)
|
|
368
|
-
except Exception:
|
|
369
|
-
return 0
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
def _num(r, key):
|
|
373
|
-
return _parse_int(r.get(key))
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
def _percentile(values, percentile):
|
|
377
|
-
"""线性插值百分位,空样本返回 0。"""
|
|
378
|
-
if not values:
|
|
379
|
-
return 0.0
|
|
380
|
-
ordered = sorted(float(v) for v in values)
|
|
381
|
-
if len(ordered) == 1:
|
|
382
|
-
return round(ordered[0], 2)
|
|
383
|
-
pos = (len(ordered) - 1) * float(percentile)
|
|
384
|
-
lo = int(math.floor(pos))
|
|
385
|
-
hi = int(math.ceil(pos))
|
|
386
|
-
if lo == hi:
|
|
387
|
-
return round(ordered[lo], 2)
|
|
388
|
-
return round(ordered[lo] + (ordered[hi] - ordered[lo]) * (pos - lo), 2)
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
def routing_stats(task_type="", role="", agent="", model="", provider="",
|
|
392
|
-
days=90, min_samples=3):
|
|
393
|
-
"""返回路由用的近期真实运行指标。
|
|
394
|
-
|
|
395
|
-
只读取脱敏用量台账;精确维度样本不足时依次回退到任务+角色、任务、
|
|
396
|
-
全局,成功率使用 Beta(3, 1) 平滑,避免单次失败永久压低新候选。
|
|
397
|
-
"""
|
|
398
|
-
try:
|
|
399
|
-
task_type = str(task_type or "")[:32]
|
|
400
|
-
role = str(role or "")[:40]
|
|
401
|
-
agent = str(agent or "")[:40]
|
|
402
|
-
model = str(model or "")[:80]
|
|
403
|
-
provider = str(provider or "")[:60]
|
|
404
|
-
days = max(0, int(days))
|
|
405
|
-
min_samples = max(1, int(min_samples))
|
|
406
|
-
except (TypeError, ValueError):
|
|
407
|
-
days, min_samples = 90, 3
|
|
408
|
-
# DATA_DIR 可在测试、多实例或运行时配置切换;纳入缓存键,避免跨目录
|
|
409
|
-
# 复用另一套台账的线上指标。
|
|
410
|
-
key = (str(paths.DATA_DIR), task_type, role, agent, model, provider,
|
|
411
|
-
days, min_samples)
|
|
412
|
-
now = time.time()
|
|
413
|
-
with LOCK:
|
|
414
|
-
cached = _ROUTING_CACHE["data"].get(key)
|
|
415
|
-
if cached and now - cached[0] <= _ROUTING_TTL:
|
|
416
|
-
return dict(cached[1])
|
|
417
|
-
records, quality_records = _routing_records(days)
|
|
418
|
-
|
|
419
|
-
def matches(record, filters):
|
|
420
|
-
return all(str(record.get(field) or "") == value
|
|
421
|
-
for field, value in filters.items() if value)
|
|
422
|
-
|
|
423
|
-
# 先保留所有已提供的维度;后续逐层放宽,确保模型和 CLI 都能共享一套查询。
|
|
424
|
-
exact = {"task_type": task_type, "role": role, "agent": agent,
|
|
425
|
-
"model": model, "provider": provider}
|
|
426
|
-
fallbacks = [(exact, "exact")]
|
|
427
|
-
task_role = {"task_type": task_type, "role": role}
|
|
428
|
-
if task_role != exact:
|
|
429
|
-
fallbacks.append((task_role, "task-role"))
|
|
430
|
-
if task_type:
|
|
431
|
-
fallbacks.append(({"task_type": task_type}, "task"))
|
|
432
|
-
fallbacks.append(({}, "global"))
|
|
433
|
-
selected, label = [], "global"
|
|
434
|
-
for filters, candidate_label in fallbacks:
|
|
435
|
-
candidate = [r for r in records if matches(r, filters)]
|
|
436
|
-
if len(candidate) >= min_samples or (candidate_label == "global" and candidate):
|
|
437
|
-
selected, label = candidate, candidate_label
|
|
438
|
-
break
|
|
439
|
-
chosen_filters = next((filters for filters, candidate_label in fallbacks
|
|
440
|
-
if candidate_label == label), {})
|
|
441
|
-
quality_selected = [r for r in quality_records if matches(r, chosen_filters)]
|
|
442
|
-
success_rows = quality_selected or selected
|
|
443
|
-
success_key = "quality_ok" if quality_selected else "ok"
|
|
444
|
-
successes = sum(1 for r in success_rows if bool(r.get(success_key)))
|
|
445
|
-
durations = [max(0.0, _parse_float(r.get("duration_s"))) for r in selected]
|
|
446
|
-
costs = [max(0.0, _parse_float(r.get("cost_usd"))) for r in selected]
|
|
447
|
-
samples = len(selected)
|
|
448
|
-
success_samples = len(success_rows)
|
|
449
|
-
result = {
|
|
450
|
-
"samples": samples,
|
|
451
|
-
"quality_samples": len(quality_selected),
|
|
452
|
-
"success_samples": success_samples,
|
|
453
|
-
"successes": successes,
|
|
454
|
-
"success_rate": round((successes + 3.0) / (success_samples + 4.0), 4),
|
|
455
|
-
"p50_duration_s": _percentile(durations, 0.50),
|
|
456
|
-
"p95_duration_s": _percentile(durations, 0.95),
|
|
457
|
-
"avg_cost_usd": round(sum(costs) / samples, 6) if samples else 0.0,
|
|
458
|
-
"fallback": label,
|
|
459
|
-
}
|
|
460
|
-
with LOCK:
|
|
461
|
-
_ROUTING_CACHE["data"][key] = (now, dict(result))
|
|
462
|
-
return result
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
def _group(records, key):
|
|
270
|
+
return out
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _iter_quality_records(days):
|
|
274
|
+
"""读取独立质量台账;不混入用量统计的调用数、token 和成本。"""
|
|
275
|
+
out = []
|
|
276
|
+
try:
|
|
277
|
+
files = sorted(paths.USAGE_DIR.glob("routing-quality-*.jsonl")) \
|
|
278
|
+
if paths.USAGE_DIR.is_dir() else []
|
|
279
|
+
except Exception:
|
|
280
|
+
return out
|
|
281
|
+
since_day = ""
|
|
282
|
+
if days:
|
|
283
|
+
import datetime
|
|
284
|
+
since_day = (datetime.date.today()
|
|
285
|
+
- datetime.timedelta(days=int(days) - 1)).isoformat()
|
|
286
|
+
since_month = since_day[:7].replace("-", "") if since_day else ""
|
|
287
|
+
for path in files:
|
|
288
|
+
if since_month and path.stem.rsplit("-", 1)[-1] < since_month:
|
|
289
|
+
continue
|
|
290
|
+
try:
|
|
291
|
+
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
292
|
+
try:
|
|
293
|
+
row = json.loads(line)
|
|
294
|
+
except Exception:
|
|
295
|
+
continue
|
|
296
|
+
if isinstance(row, dict) and (not since_day or row.get("day", "") >= since_day):
|
|
297
|
+
out.append(row)
|
|
298
|
+
except Exception:
|
|
299
|
+
continue
|
|
300
|
+
return out
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _routing_records(days):
|
|
304
|
+
"""同一轮候选评分共享一次台账解析,避免每个候选重复扫全盘。"""
|
|
305
|
+
key = (str(paths.USAGE_DIR), int(days))
|
|
306
|
+
now = time.time()
|
|
307
|
+
with LOCK:
|
|
308
|
+
cached = _ROUTING_RECORDS_CACHE.get(key)
|
|
309
|
+
if cached and now - cached[0] <= _ROUTING_TTL:
|
|
310
|
+
return list(cached[1]), list(cached[2])
|
|
311
|
+
calls, quality = _iter_records(days), _iter_quality_records(days)
|
|
312
|
+
with LOCK:
|
|
313
|
+
_ROUTING_RECORDS_CACHE[key] = (now, list(calls), list(quality))
|
|
314
|
+
return calls, quality
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def record_quality_for_run(run_id, quality_ok, agent=""):
|
|
318
|
+
"""把验收结论归因到最终实际产出者,不重复计入用量。"""
|
|
319
|
+
try:
|
|
320
|
+
run_id = str(run_id or "")[:64]
|
|
321
|
+
if not run_id:
|
|
322
|
+
return 0
|
|
323
|
+
calls = [r for r in _iter_records(2)
|
|
324
|
+
if r.get("run_id") == run_id and r.get("ok") is True]
|
|
325
|
+
prefixes = ("implement", "fix", "draft", "revise", "polish", "direct", "author")
|
|
326
|
+
calls = [r for r in calls if str(r.get("role") or "").lower().startswith(prefixes)]
|
|
327
|
+
agent = str(agent or "")[:40]
|
|
328
|
+
unique = {}
|
|
329
|
+
for row in calls:
|
|
330
|
+
key = tuple(str(row.get(k) or "") for k in
|
|
331
|
+
("task_id", "task_type", "role", "agent", "model", "provider"))
|
|
332
|
+
unique[key] = row
|
|
333
|
+
if not unique:
|
|
334
|
+
return 0
|
|
335
|
+
existing = {(r.get("run_id"), r.get("role"), r.get("agent"),
|
|
336
|
+
r.get("model"), r.get("provider"))
|
|
337
|
+
for r in _iter_quality_records(2)}
|
|
338
|
+
day = time.strftime("%Y-%m-%d")
|
|
339
|
+
rows = []
|
|
340
|
+
for row in unique.values():
|
|
341
|
+
identity = (run_id, row.get("role"), row.get("agent"),
|
|
342
|
+
row.get("model"), row.get("provider"))
|
|
343
|
+
if identity in existing:
|
|
344
|
+
continue
|
|
345
|
+
# 成功调用不等于产出通过。换将前的实现者即使传输成功,也已被质量
|
|
346
|
+
# 门淘汰,必须留下负样本;最终实际产出者才继承本次验收结论。
|
|
347
|
+
row_quality_ok = bool(quality_ok)
|
|
348
|
+
if agent and str(row.get("agent") or "") != agent:
|
|
349
|
+
row_quality_ok = False
|
|
350
|
+
rows.append({"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "day": day,
|
|
351
|
+
"run_id": run_id, "task_id": row.get("task_id") or "",
|
|
352
|
+
"task_type": row.get("task_type") or "unknown",
|
|
353
|
+
"role": row.get("role") or "unknown",
|
|
354
|
+
"agent": row.get("agent") or "unknown",
|
|
355
|
+
"model": row.get("model") or "(默认)",
|
|
356
|
+
"provider": row.get("provider") or "",
|
|
357
|
+
"quality_ok": row_quality_ok})
|
|
358
|
+
if not rows:
|
|
359
|
+
return 0
|
|
360
|
+
with LOCK:
|
|
361
|
+
paths.USAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
362
|
+
with open(_quality_file(day), "a", encoding="utf-8") as handle:
|
|
363
|
+
for row in rows:
|
|
364
|
+
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
365
|
+
_ROUTING_CACHE["data"].clear()
|
|
366
|
+
_ROUTING_RECORDS_CACHE.clear()
|
|
367
|
+
return len(rows)
|
|
368
|
+
except Exception:
|
|
369
|
+
return 0
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _num(r, key):
|
|
373
|
+
return _parse_int(r.get(key))
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _percentile(values, percentile):
|
|
377
|
+
"""线性插值百分位,空样本返回 0。"""
|
|
378
|
+
if not values:
|
|
379
|
+
return 0.0
|
|
380
|
+
ordered = sorted(float(v) for v in values)
|
|
381
|
+
if len(ordered) == 1:
|
|
382
|
+
return round(ordered[0], 2)
|
|
383
|
+
pos = (len(ordered) - 1) * float(percentile)
|
|
384
|
+
lo = int(math.floor(pos))
|
|
385
|
+
hi = int(math.ceil(pos))
|
|
386
|
+
if lo == hi:
|
|
387
|
+
return round(ordered[lo], 2)
|
|
388
|
+
return round(ordered[lo] + (ordered[hi] - ordered[lo]) * (pos - lo), 2)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def routing_stats(task_type="", role="", agent="", model="", provider="",
|
|
392
|
+
days=90, min_samples=3):
|
|
393
|
+
"""返回路由用的近期真实运行指标。
|
|
394
|
+
|
|
395
|
+
只读取脱敏用量台账;精确维度样本不足时依次回退到任务+角色、任务、
|
|
396
|
+
全局,成功率使用 Beta(3, 1) 平滑,避免单次失败永久压低新候选。
|
|
397
|
+
"""
|
|
398
|
+
try:
|
|
399
|
+
task_type = str(task_type or "")[:32]
|
|
400
|
+
role = str(role or "")[:40]
|
|
401
|
+
agent = str(agent or "")[:40]
|
|
402
|
+
model = str(model or "")[:80]
|
|
403
|
+
provider = str(provider or "")[:60]
|
|
404
|
+
days = max(0, int(days))
|
|
405
|
+
min_samples = max(1, int(min_samples))
|
|
406
|
+
except (TypeError, ValueError):
|
|
407
|
+
days, min_samples = 90, 3
|
|
408
|
+
# DATA_DIR 可在测试、多实例或运行时配置切换;纳入缓存键,避免跨目录
|
|
409
|
+
# 复用另一套台账的线上指标。
|
|
410
|
+
key = (str(paths.DATA_DIR), task_type, role, agent, model, provider,
|
|
411
|
+
days, min_samples)
|
|
412
|
+
now = time.time()
|
|
413
|
+
with LOCK:
|
|
414
|
+
cached = _ROUTING_CACHE["data"].get(key)
|
|
415
|
+
if cached and now - cached[0] <= _ROUTING_TTL:
|
|
416
|
+
return dict(cached[1])
|
|
417
|
+
records, quality_records = _routing_records(days)
|
|
418
|
+
|
|
419
|
+
def matches(record, filters):
|
|
420
|
+
return all(str(record.get(field) or "") == value
|
|
421
|
+
for field, value in filters.items() if value)
|
|
422
|
+
|
|
423
|
+
# 先保留所有已提供的维度;后续逐层放宽,确保模型和 CLI 都能共享一套查询。
|
|
424
|
+
exact = {"task_type": task_type, "role": role, "agent": agent,
|
|
425
|
+
"model": model, "provider": provider}
|
|
426
|
+
fallbacks = [(exact, "exact")]
|
|
427
|
+
task_role = {"task_type": task_type, "role": role}
|
|
428
|
+
if task_role != exact:
|
|
429
|
+
fallbacks.append((task_role, "task-role"))
|
|
430
|
+
if task_type:
|
|
431
|
+
fallbacks.append(({"task_type": task_type}, "task"))
|
|
432
|
+
fallbacks.append(({}, "global"))
|
|
433
|
+
selected, label = [], "global"
|
|
434
|
+
for filters, candidate_label in fallbacks:
|
|
435
|
+
candidate = [r for r in records if matches(r, filters)]
|
|
436
|
+
if len(candidate) >= min_samples or (candidate_label == "global" and candidate):
|
|
437
|
+
selected, label = candidate, candidate_label
|
|
438
|
+
break
|
|
439
|
+
chosen_filters = next((filters for filters, candidate_label in fallbacks
|
|
440
|
+
if candidate_label == label), {})
|
|
441
|
+
quality_selected = [r for r in quality_records if matches(r, chosen_filters)]
|
|
442
|
+
success_rows = quality_selected or selected
|
|
443
|
+
success_key = "quality_ok" if quality_selected else "ok"
|
|
444
|
+
successes = sum(1 for r in success_rows if bool(r.get(success_key)))
|
|
445
|
+
durations = [max(0.0, _parse_float(r.get("duration_s"))) for r in selected]
|
|
446
|
+
costs = [max(0.0, _parse_float(r.get("cost_usd"))) for r in selected]
|
|
447
|
+
samples = len(selected)
|
|
448
|
+
success_samples = len(success_rows)
|
|
449
|
+
result = {
|
|
450
|
+
"samples": samples,
|
|
451
|
+
"quality_samples": len(quality_selected),
|
|
452
|
+
"success_samples": success_samples,
|
|
453
|
+
"successes": successes,
|
|
454
|
+
"success_rate": round((successes + 3.0) / (success_samples + 4.0), 4),
|
|
455
|
+
"p50_duration_s": _percentile(durations, 0.50),
|
|
456
|
+
"p95_duration_s": _percentile(durations, 0.95),
|
|
457
|
+
"avg_cost_usd": round(sum(costs) / samples, 6) if samples else 0.0,
|
|
458
|
+
"fallback": label,
|
|
459
|
+
}
|
|
460
|
+
with LOCK:
|
|
461
|
+
_ROUTING_CACHE["data"][key] = (now, dict(result))
|
|
462
|
+
return result
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _group(records, key):
|
|
466
466
|
"""按 key 聚合:{name: {calls, ok, tokens, input, output, cached, cache_rate, cost_usd, duration_s}}。"""
|
|
467
467
|
groups = {}
|
|
468
468
|
for r in records:
|
|
@@ -585,18 +585,22 @@ def summary(days=30, recent_limit=30):
|
|
|
585
585
|
d = (now - datetime.timedelta(days=span - 1 - i)).isoformat()
|
|
586
586
|
g = day_groups.get(d, {})
|
|
587
587
|
by_day.append({"day": d, "calls": g.get("calls", 0),
|
|
588
|
+
"ok": g.get("ok", 0),
|
|
588
589
|
"tokens": g.get("tokens", 0),
|
|
589
590
|
"input": g.get("input", 0), "output": g.get("output", 0),
|
|
590
591
|
"cached": g.get("cached", 0),
|
|
591
592
|
"cache_rate": g.get("cache_rate", 0.0),
|
|
593
|
+
"duration_s": g.get("duration_s", 0.0),
|
|
592
594
|
"cost_usd": round(g.get("cost_usd", 0.0), 4)})
|
|
593
595
|
else:
|
|
594
596
|
for d in sorted(day_groups):
|
|
595
597
|
g = day_groups[d]
|
|
596
|
-
by_day.append({"day": d, "calls": g["calls"], "
|
|
598
|
+
by_day.append({"day": d, "calls": g["calls"], "ok": g["ok"],
|
|
599
|
+
"tokens": g["tokens"],
|
|
597
600
|
"input": g["input"], "output": g["output"],
|
|
598
601
|
"cached": g.get("cached", 0),
|
|
599
602
|
"cache_rate": g.get("cache_rate", 0.0),
|
|
603
|
+
"duration_s": g.get("duration_s", 0.0),
|
|
600
604
|
"cost_usd": g["cost_usd"]})
|
|
601
605
|
|
|
602
606
|
# 按日按模型 tokens(范围内;与 by_day 同一天序列,无数据天给空表)
|
|
@@ -655,19 +659,19 @@ def summary(days=30, recent_limit=30):
|
|
|
655
659
|
}
|
|
656
660
|
|
|
657
661
|
|
|
658
|
-
_DURATION_BASELINES = {
|
|
659
|
-
"direct": 90, "rank_scan": 240, "email": 180, "weekly_report": 240,
|
|
660
|
-
"translation": 240, "speech": 360, "video_script": 360, "article": 480,
|
|
661
|
-
"doc": 480, "resume": 420, "code": 600, "tech_proposal": 720,
|
|
662
|
-
"research": 900, "novel": 720, "serial_novel": 2400,
|
|
663
|
-
}
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=None):
|
|
667
|
-
"""同类任务开跑前成本与耗时预估(数据源为本地用量台账)。
|
|
668
|
-
|
|
669
|
-
同一 run 的多条步骤记录加总为一个样本,优先用成功 run,给中位数/平均/P90。
|
|
670
|
-
无历史样本时使用流程级保守基线;模式与思考程度只做透明倍率修正。"""
|
|
662
|
+
_DURATION_BASELINES = {
|
|
663
|
+
"direct": 90, "rank_scan": 240, "email": 180, "weekly_report": 240,
|
|
664
|
+
"translation": 240, "speech": 360, "video_script": 360, "article": 480,
|
|
665
|
+
"doc": 480, "resume": 420, "code": 600, "tech_proposal": 720,
|
|
666
|
+
"research": 900, "novel": 720, "serial_novel": 2400,
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=None):
|
|
671
|
+
"""同类任务开跑前成本与耗时预估(数据源为本地用量台账)。
|
|
672
|
+
|
|
673
|
+
同一 run 的多条步骤记录加总为一个样本,优先用成功 run,给中位数/平均/P90。
|
|
674
|
+
无历史样本时使用流程级保守基线;模式与思考程度只做透明倍率修正。"""
|
|
671
675
|
span = max(1, min(3650, int(days or 90)))
|
|
672
676
|
with LOCK:
|
|
673
677
|
records = _iter_records(span)
|
|
@@ -679,53 +683,53 @@ def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=Non
|
|
|
679
683
|
rid = str(r.get("run_id") or "")
|
|
680
684
|
if not rid:
|
|
681
685
|
continue
|
|
682
|
-
g = runs.setdefault(rid, {"tokens": 0, "cost": 0.0,
|
|
683
|
-
"duration": 0.0, "ok": False})
|
|
684
|
-
g["tokens"] += _num(r, "total")
|
|
685
|
-
g["cost"] += _parse_float(r.get("cost_usd"))
|
|
686
|
-
g["duration"] += max(0.0, _parse_float(r.get("duration_s")))
|
|
686
|
+
g = runs.setdefault(rid, {"tokens": 0, "cost": 0.0,
|
|
687
|
+
"duration": 0.0, "ok": False})
|
|
688
|
+
g["tokens"] += _num(r, "total")
|
|
689
|
+
g["cost"] += _parse_float(r.get("cost_usd"))
|
|
690
|
+
g["duration"] += max(0.0, _parse_float(r.get("duration_s")))
|
|
687
691
|
if r.get("ok"):
|
|
688
692
|
g["ok"] = True
|
|
689
693
|
ok_runs = [g for g in runs.values() if g["ok"]]
|
|
690
694
|
basis = ok_runs or list(runs.values())
|
|
691
|
-
mode_factor = {"fast": 0.60, "auto": 1.0, "expert": 1.65,
|
|
692
|
-
"manual": 1.0}.get(str(mode or "auto"), 1.0)
|
|
693
|
-
thinking_factor = {"low": 0.78, "standard": 1.0, "high": 1.45,
|
|
694
|
-
"auto": 1.0}.get(str(thinking or "standard"), 1.0)
|
|
695
|
-
factor = mode_factor * thinking_factor
|
|
696
|
-
try:
|
|
697
|
-
if rounds is not None and int(rounds) > 2:
|
|
698
|
-
factor *= 1.0 + min(3, int(rounds) - 2) * 0.18
|
|
699
|
-
except (TypeError, ValueError):
|
|
700
|
-
pass
|
|
701
|
-
if not basis:
|
|
702
|
-
baseline = _DURATION_BASELINES.get(task_type, 480)
|
|
703
|
-
median_s = max(30, int(baseline * factor))
|
|
704
|
-
return {"task_type": task_type or "", "days": span, "samples": 0,
|
|
705
|
-
"estimated_duration_s": median_s,
|
|
706
|
-
"p90_duration_s": max(median_s + 30, int(median_s * 1.7)),
|
|
707
|
-
"duration_source": "baseline"}
|
|
695
|
+
mode_factor = {"fast": 0.60, "auto": 1.0, "expert": 1.65,
|
|
696
|
+
"manual": 1.0}.get(str(mode or "auto"), 1.0)
|
|
697
|
+
thinking_factor = {"low": 0.78, "standard": 1.0, "high": 1.45,
|
|
698
|
+
"auto": 1.0}.get(str(thinking or "standard"), 1.0)
|
|
699
|
+
factor = mode_factor * thinking_factor
|
|
700
|
+
try:
|
|
701
|
+
if rounds is not None and int(rounds) > 2:
|
|
702
|
+
factor *= 1.0 + min(3, int(rounds) - 2) * 0.18
|
|
703
|
+
except (TypeError, ValueError):
|
|
704
|
+
pass
|
|
705
|
+
if not basis:
|
|
706
|
+
baseline = _DURATION_BASELINES.get(task_type, 480)
|
|
707
|
+
median_s = max(30, int(baseline * factor))
|
|
708
|
+
return {"task_type": task_type or "", "days": span, "samples": 0,
|
|
709
|
+
"estimated_duration_s": median_s,
|
|
710
|
+
"p90_duration_s": max(median_s + 30, int(median_s * 1.7)),
|
|
711
|
+
"duration_source": "baseline"}
|
|
708
712
|
toks = sorted(g["tokens"] for g in basis)
|
|
709
|
-
costs = sorted(g["cost"] for g in basis)
|
|
710
|
-
durations = sorted(g["duration"] for g in basis if g["duration"] > 0)
|
|
713
|
+
costs = sorted(g["cost"] for g in basis)
|
|
714
|
+
durations = sorted(g["duration"] for g in basis if g["duration"] > 0)
|
|
711
715
|
n = len(toks)
|
|
712
716
|
mid = n // 2
|
|
713
717
|
med = toks[mid] if n % 2 else (toks[mid - 1] + toks[mid]) / 2.0
|
|
714
718
|
med_c = costs[mid] if n % 2 else (costs[mid - 1] + costs[mid]) / 2.0
|
|
715
719
|
import datetime
|
|
716
|
-
import math
|
|
717
|
-
if durations:
|
|
718
|
-
dn = len(durations)
|
|
719
|
-
dmid = dn // 2
|
|
720
|
-
dmed = (durations[dmid] if dn % 2
|
|
721
|
-
else (durations[dmid - 1] + durations[dmid]) / 2.0)
|
|
722
|
-
dp90 = durations[max(0, min(dn - 1, math.ceil(0.9 * dn) - 1))]
|
|
723
|
-
duration_source = "history"
|
|
724
|
-
else:
|
|
725
|
-
dmed = _DURATION_BASELINES.get(task_type, 480)
|
|
726
|
-
dp90 = dmed * 1.7
|
|
727
|
-
duration_source = "baseline"
|
|
728
|
-
return {
|
|
720
|
+
import math
|
|
721
|
+
if durations:
|
|
722
|
+
dn = len(durations)
|
|
723
|
+
dmid = dn // 2
|
|
724
|
+
dmed = (durations[dmid] if dn % 2
|
|
725
|
+
else (durations[dmid - 1] + durations[dmid]) / 2.0)
|
|
726
|
+
dp90 = durations[max(0, min(dn - 1, math.ceil(0.9 * dn) - 1))]
|
|
727
|
+
duration_source = "history"
|
|
728
|
+
else:
|
|
729
|
+
dmed = _DURATION_BASELINES.get(task_type, 480)
|
|
730
|
+
dp90 = dmed * 1.7
|
|
731
|
+
duration_source = "baseline"
|
|
732
|
+
return {
|
|
729
733
|
"task_type": task_type or "",
|
|
730
734
|
"days": span,
|
|
731
735
|
"samples": n,
|
|
@@ -734,10 +738,10 @@ def estimate(task_type="", days=90, mode="auto", thinking="standard", rounds=Non
|
|
|
734
738
|
"median_tokens": int(med),
|
|
735
739
|
"p90_tokens": toks[max(0, min(n - 1, math.ceil(0.9 * n) - 1))],
|
|
736
740
|
"avg_cost_usd": round(sum(costs) / n, 4),
|
|
737
|
-
"median_cost_usd": round(med_c, 4),
|
|
738
|
-
"estimated_duration_s": max(30, int(dmed * factor)),
|
|
739
|
-
"p90_duration_s": max(60, int(dp90 * factor)),
|
|
740
|
-
"duration_samples": len(durations),
|
|
741
|
-
"duration_source": duration_source,
|
|
741
|
+
"median_cost_usd": round(med_c, 4),
|
|
742
|
+
"estimated_duration_s": max(30, int(dmed * factor)),
|
|
743
|
+
"p90_duration_s": max(60, int(dp90 * factor)),
|
|
744
|
+
"duration_samples": len(durations),
|
|
745
|
+
"duration_source": duration_source,
|
|
742
746
|
"since": (datetime.date.today() - datetime.timedelta(days=span - 1)).isoformat(),
|
|
743
747
|
}
|