codebee 0.1.69 → 0.1.70

package/app/core/usage.py

@@ -19,6 +19,26 @@ from . import paths
 
 LOCK = threading.RLock()
 
+# ---------------------------------------------------------------- 用量版本（SSE 轻量事件）
+# 台账每落一条记录就 bump 一次版本号。SSE 循环看到变化只推一个几十字节的
+# usage 事件让前端刷新侧栏用量条，绝不为此构建/推送全量状态——运行中记账
+# 是秒级频率，全量 payload 会被放大成推送风暴。版本号自带合并：两次读取
+# 之间落多少条记录都只算一次变化。
+_VER_CV = threading.Condition()
+_USAGE_VER = 1
+
+
+def bump_usage():
+    global _USAGE_VER
+    with _VER_CV:
+        _USAGE_VER += 1
+        _VER_CV.notify_all()
+
+
+def usage_version():
+    return _USAGE_VER
+
+
 # 维度 → 显示名（聚合接口与前端共用）
 DIMENSIONS = {
     "tool": "工具（CLI）",
@@ -99,6 +119,18 @@ def record(source="", run_id="", task_id="", task_type="", role="", step=0,
         total = _parse_int(u.get("total")) or (inp + out + cach)
         rec.update({"input": inp, "output": out, "cached": cach,
                     "reasoning": reas, "total": total})
+        # 金额闭环：调用方没给费用（CLI/直连/评测普遍如此）→ 按标定单价折算
+        # （¥/百万 tokens；手标优先，回落导入价表；未标价 = 不记钱，宁缺毋滥）。
+        # 列名 cost_usd 沿用历史，语义 = 费用（¥）。
+        if _parse_float(rec.get("cost_usd")) <= 0 and (inp or out) \
+                and rec.get("model") not in ("", "(默认)"):
+            try:
+                from . import modelhub
+                pr = modelhub.model_price(str(rec["model"]), provider=str(rec["provider"]))
+                if pr:
+                    rec["cost_usd"] = round((pr["in"] * inp + pr["out"] * out) / 1e6, 4)
+            except Exception:
+                pass
         # 压缩省量注记（source=compaction 专用）：上下文折叠净省的估算 token，
         # 与消耗并列成账——省了多少不再是黑箱。零值不落字段防老记录膨胀。
         saved = max(0, _parse_int(u.get("saved")))
@@ -120,13 +152,107 @@ def record(source="", run_id="", task_id="", task_type="", role="", step=0,
             _ROUTING_CACHE["data"].clear()
             _ROUTING_RECORDS_CACHE.clear()
             _HOURLY_CACHE["ts"] = 0.0
+        bump_usage()  # 台账变了：SSE 推 usage 事件，侧栏用量条实时跟进
+        if _budget_caps_on():
+            # 花费预警（硬顶的事前半环）：只在配了上限时起线程，绝不拖慢记账
+            threading.Thread(target=_budget_alert_check, daemon=True,
+                             args=(rec["day"],)).start()
+    except Exception:
+        pass
+
+
+def _budget_caps_on():
+    try:
+        from .settings_schema import get as ss_get, register_default_namespaces
+        register_default_namespaces()
+        return (float(ss_get("budget", "daily_cost_yuan") or 0) > 0
+                or float(ss_get("budget", "monthly_cost_yuan") or 0) > 0)
+    except Exception:
+        return False
+
+
+_ALERT_PUSH = None        # main.py 启动注入 notify.push_text——usage(L0) 不反向
+                           # 依赖 notify(L1)，静态图不成环（同 set_run_estimator 模式）
+
+
+def set_alert_push(fn):
+    """注入花费预警的推送函数；未注入则预警静默跳过。"""
+    global _ALERT_PUSH
+    _ALERT_PUSH = fn if callable(fn) else None
+
+
+def _budget_alert_check(day):
+    """花费预警推送（50%/80%，日/月各自去重一次）：预警是硬顶的事前提醒，
+    触发阈值状态落 data/budget_alerts.json（按天重置）。失败静默。"""
+    try:
+        from .settings_schema import get as ss_get, register_default_namespaces
+        register_default_namespaces()
+        dcap = float(ss_get("budget", "daily_cost_yuan") or 0)
+        mcap = float(ss_get("budget", "monthly_cost_yuan") or 0)
+        today, month = cost_snapshot()
+        marks = []
+        for scope, spent, cap in (("d", today, dcap), ("m", month, mcap)):
+            if cap <= 0 or spent <= 0:
+                continue
+            pct = spent / cap
+            for th in (0.8, 0.5):
+                if pct >= th:
+                    marks.append((scope, int(th * 100), spent, cap))
+                    break
+        if not marks:
+            return
+        sp = paths.DATA_DIR / "budget_alerts.json"
+        try:
+            st = json.loads(sp.read_text(encoding="utf-8"))
+        except Exception:
+            st = {}
+        if st.get("day") != day or not isinstance(st.get("done"), list):
+            st = {"day": day, "done": []}
+        done = st["done"]
+        new = [m for m in marks if ("%s%d" % (m[0], m[1])) not in done]
+        if not new:
+            return
+        st["done"] = done + ["%s%d" % (m[0], m[1]) for m in new]
+        sp.parent.mkdir(parents=True, exist_ok=True)
+        tmp = sp.with_suffix(".tmp")
+        tmp.write_text(json.dumps(st, ensure_ascii=False), encoding="utf-8")
+        tmp.replace(sp)
+        if not _ALERT_PUSH:
+            return
+        lines = ["💸 CodeBee 花费预警"]
+        for scope, pct, spent, cap in new:
+            lines.append("%s花费 ¥%.2f，已达%s上限 ¥%.2f 的 %d%%" % (
+                "今日" if scope == "d" else "本月", spent,
+                "每日" if scope == "d" else "每月", cap, pct))
+        lines.append("（达 100% 后新步骤将停止，可在设置→编排设置→预算调整）")
+        _ALERT_PUSH("\n".join(lines))
     except Exception:
         pass
 
 
+EXPORT_COLUMNS = ("ts", "day", "source", "task_type", "role", "agent", "model",
+                  "provider", "ok", "input", "output", "cached", "reasoning",
+                  "total", "cost_usd")
+
+
+def export_csv(days=30, max_rows=10000):
+    """台账明细导出 CSV（UTF-8 BOM，Excel 直开；行数封顶防炸）。返回 bytes。"""
+    import csv as _csv
+    import io as _io
+    buf = _io.StringIO()
+    writer = _csv.writer(buf)
+    writer.writerow(EXPORT_COLUMNS)
+    n = 0
+    for r in _iter_records(max(0, int(days or 0))):
+        writer.writerow([r.get(c, "") for c in EXPORT_COLUMNS])
+        n += 1
+        if n >= max_rows:
+            break
+    return b"\xef\xbb\xbf" + buf.getvalue().encode("utf-8")
+
+
 def backfill_from_runs():
     """把历史 run.json 里已记录的 token 用量回填进台账（启动时调用，幂等）。
-
     埋点是后加的：此前的运行只在 run.json 的步骤里存了 tokens/cost_usd 总量，
     没有输入/输出/缓存细分，所以回填记录 input/output/cached 记 0、只有 total，
     并以 source="backfill" 标记便于区分。已回填的 (run_id, 步骤号) 会跳过，
@@ -250,6 +376,15 @@ def agent_tokens_recent(agent, hours=1):
         return 0
 
 
+# 文件级解析缓存：台账 append-only（写入只增不改前缀），按 (mtime_ns, size)
+# 命中就复用已解析记录。用量条实时化后 summary 会被按秒级反复调用，缓存让
+# 每次只重解析当天新落的小文件，历史月份不再逐次全量 json.loads。缓存的是
+# 未过滤的全量记录，days 过滤在读出后做，同一份缓存可服务不同取数窗口。
+# 读+stat 全程持 LOCK（RLock 可重入，summary 外层已持也不死锁），杜绝
+# 「读到旧内容、却按新 mtime 入缓存」的竞态把新记录吞到下次追加才可见。
+_FILE_CACHE = {}
+
+
 def _iter_records(days):
     """按时间范围读取台账（days=0 表示全部）。返回按写入顺序的记录列表。"""
     out = []
@@ -267,24 +402,62 @@ def _iter_records(days):
         if since_month and p.stem.rsplit("-", 1)[-1] < since_month:
             continue
         try:
-            for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
-                line = line.strip()
-                if not line.startswith("{"):
-                    continue
-                try:
-                    r = json.loads(line)
-                except Exception:
-                    continue
-                if not isinstance(r, dict):
-                    continue
-                if since_day and str(r.get("day", "")) < since_day:
-                    continue
-                out.append(r)
+            with LOCK:
+                st = p.stat()
+                key = str(p)
+                hit = _FILE_CACHE.get(key)
+                if hit and hit[0] == (st.st_mtime_ns, st.st_size):
+                    recs = hit[1]
+                else:
+                    recs = []
+                    for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
+                        line = line.strip()
+                        if not line.startswith("{"):
+                            continue
+                        try:
+                            r = json.loads(line)
+                        except Exception:
+                            continue
+                        if not isinstance(r, dict):
+                            continue
+                        recs.append(r)
+                    _FILE_CACHE[key] = ((st.st_mtime_ns, st.st_size), recs)
         except Exception:
             continue
+        for r in recs:
+            if since_day and str(r.get("day", "")) < since_day:
+                continue
+            out.append(r)
     return out
 
 
+def cost_snapshot():
+    """(今日实付, 当月实付) 美元快照——花费硬顶闸（pipeline 预算检查）用。
+
+    复用 _iter_records 的按文件缓存；窗口取 31 天恒覆盖整个当月，
+    单月文件量级下每次调用开销可忽略。任何异常返回 (0.0, 0.0)（闸放行，
+    统计层故障不能反过来把业务锁死）。
+    """
+    try:
+        import datetime
+        today = datetime.date.today().isoformat()
+        month = today[:7]
+        t = m = 0.0
+        for r in _iter_records(31):
+            c = _parse_float(r.get("cost_usd"))
+            if c <= 0:
+                continue
+            d = str(r.get("day") or "")
+            if d == today:
+                t += c
+                m += c
+            elif d.startswith(month):
+                m += c
+        return round(t, 4), round(m, 4)
+    except Exception:
+        return 0.0, 0.0
+
+
 def _iter_quality_records(days):
     """读取独立质量台账；不混入用量统计的调用数、token 和成本。"""
     out = []

package/app/core/zentao.py

@@ -69,6 +69,11 @@ from pathlib import Path
 
 from . import jobs, operations, paths, settings, store, tlsctx
 
+try:
+    import chinese_calendar as _hcal   # 法定节假日+调休数据（chinesecalendar）
+except ImportError:                    # 未装时跳过节假日功能不可用，开启即保存报错
+    _hcal = None
+
 log = logging.getLogger(__name__)
 
 _LOCK = threading.RLock()
@@ -115,6 +120,7 @@ _CFG_DEFAULTS = {
     "triage_ai": True,         # 模块路由未命中时用 AI 兜底排查
     "poll_enabled": False,
     "interval_minutes": INTERVAL_DEFAULT,
+    "skip_holidays": False,    # 法定休假日顺延定时扫描（对账不受影响）
 }
 
 _REPO_DEFAULTS = {"workdir": "", "git_rev": "", "verify_command": ""}
@@ -132,7 +138,7 @@ _PROFILE_DEFAULTS = {
 
 _UPDATABLE = ("base_url", "account", "password", "product_profiles",
               "auto_resolve", "auto_merge", "triage_ai",
-              "poll_enabled", "interval_minutes")
+              "poll_enabled", "interval_minutes", "skip_holidays")
 
 
 # ---------------------------------------------------------------- 出网边界（SSRF）
@@ -206,6 +212,10 @@ def _normalize_claim(d):
         c["triage"] = {"side": "", "reason": "", "by": "", "account": ""}
     c.setdefault("note", "")
     c.setdefault("attempts", 0)
+    c.setdefault("archived", False)       # 归档三件套：老数据无字段照常跑
+    c.setdefault("archived_by", "")       # manual=用户手点 / auto=扫描对账自动
+    c.setdefault("archived_at", "")
+    c.setdefault("archive_reason", "")
     return c
 
 
@@ -1810,6 +1820,93 @@ def _set_claim(bid, **patch):
         _save_locked()
 
 
+def archive_claim(bid, archived, reason=""):
+    """手动归档/取消归档一条修复记录。返回更新后的 claim；不存在抛 ZenError。"""
+    bid = str(bid)
+    _ensure_loaded()
+    with _LOCK:
+        c = _STATE["claims"].get(bid)
+        if c is None:
+            raise ZenError("修复记录 #%s 不存在" % bid)
+        if archived:
+            c.update({"archived": True, "archived_by": "manual",
+                      "archived_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+                      "archive_reason": str(reason or "").strip() or "手动归档"})
+        else:
+            c.update({"archived": False, "archived_by": "",
+                      "archived_at": "", "archive_reason": ""})
+        c["at"] = time.strftime("%Y-%m-%d %H:%M:%S")
+        _save_locked()
+        return dict(c)
+
+
+def _bug_status(cfg, bid):
+    """单查 bug 当前状态。失败抛异常（对账方决定本轮跳过，绝不盲归档）。"""
+    d = _call("GET", "/bugs/%s" % bid, cfg=cfg)
+    return str(d.get("status") or "")
+
+
+def _archive_sweep(cfg, profile, by_id):
+    """归档对账：本产品激活列表拉取成功后跑一遍（list_bugs 抛错到不了这里，
+    网络抖动不会 mass-archive）。
+
+    - 记录不在激活列表 → 单查确认：resolved/closed 才自动归档；仍激活
+      （多半已改派给别人）只备注一次，不归档；单查失败本轮跳过。
+    - 已自动归档的记录 bug 重新激活且仍可认领 → 取消归档重走排查
+      （手动归档是用户的明确决定，永不自动取消）。
+    """
+    pid = str(profile.get("product") or "")
+    with _LOCK:
+        mine = [dict(c) for c in _STATE["claims"].values()
+                if str(c.get("product") or "") == pid]
+    for c in mine:
+        bid = str(c.get("bug_id") or "")
+        if not bid:
+            continue
+        if bid in by_id:
+            if c.get("archived") and c.get("archived_by") == "auto" \
+                    and _claimable(by_id[bid], profile):
+                _set_claim(bid, archived=False, archived_by="",
+                           archived_at="", archive_reason="",
+                           note="禅道侧重新激活，重新排查")
+                try:
+                    _route_one(by_id[bid], profile, cfg)
+                except Exception as e:
+                    log.warning("zentao: bug %s 重新激活重排查失败：%s", bid, e)
+                    _set_claim(bid, state="need_manual",
+                               note="重新激活重排查失败：%s" % e)
+            continue
+        if c.get("archived"):
+            continue
+        try:
+            status = _bug_status(cfg, bid)
+        except Exception:
+            log.warning("zentao: 归档对账单查 bug %s 失败，本轮跳过", bid)
+            continue
+        if status in ("resolved", "closed"):
+            with _LOCK:
+                cc = _STATE["claims"].get(bid)
+                if cc is not None and not cc.get("archived") \
+                        and cc.get("state") != "fixing":
+                    cc.update({"archived": True, "archived_by": "auto",
+                               "archived_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+                               "archive_reason": "禅道侧%s，自动归档"
+                                                 % ("已解决" if status == "resolved" else "已关闭"),
+                               "at": time.strftime("%Y-%m-%d %H:%M:%S")})
+                    _save_locked()
+        else:
+            # 仍激活但不在我的列表（改派/过滤条件变化）：备注一次不归档；
+            # fixing 中的只标注不归档——流程收口交给对账，这里不抢状态
+            note = "bug 仍激活但不在本档案扫描列表（可能已改派），不归档"
+            with _LOCK:
+                cc = _STATE["claims"].get(bid)
+                if cc is not None and note not in str(cc.get("note") or ""):
+                    cc["note"] = (str(cc.get("note") or "") + "；" + note) \
+                        if cc.get("note") else note
+                    cc["at"] = time.strftime("%Y-%m-%d %H:%M:%S")
+                    _save_locked()
+
+
 def _route_account(profile, tri, side):
     """转派/升级目标：模块路由 account > 端负责人。"""
     if tri and tri.get("account"):
@@ -2047,6 +2144,8 @@ def _route_one(bug, profile, cfg, notify=True):
         "title": str(bug.get("title") or "")[:120],
         "triage": tri, "tasks": [], "state": "fixing", "note": "",
         "attempts": 0, "claimed_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+        # 归档字段就地归零：reopen 重走排查覆盖老 claim 时不能残留 True
+        "archived": False, "archived_by": "", "archived_at": "", "archive_reason": "",
     }
     opened = _acct(bug.get("openedBy"))
 
@@ -2160,7 +2259,8 @@ def _route_one(bug, profile, cfg, notify=True):
 
 
 def _scan(cfg):
-    """按产品档案逐个拉 bug：认领新 bug + 重试 need_manual 的存量。返回认领数。"""
+    """按产品档案逐个拉 bug：认领新 bug + 重试 need_manual 的存量 + 归档对账。
+    返回认领数。"""
     claimed = 0
     profiles = _profiles(cfg)
     if not profiles:
@@ -2171,7 +2271,8 @@ def _scan(cfg):
         with _LOCK:
             seen = set(_STATE["claims"].keys())
             retry_ids = [k for k, c in _STATE["claims"].items()
-                         if c.get("state") == "need_manual" and k in by_id
+                         if c.get("state") == "need_manual" and not c.get("archived")
+                         and k in by_id
                          and str(by_id[k].get("status") or "") == "active"]
         for bug in bugs:
             bid = str(bug.get("id") or "")
@@ -2185,9 +2286,42 @@ def _scan(cfg):
                 if _STATE["claims"].get(bid, {}).get("state") != "need_manual":
                     continue
             _route_one(by_id[bid], profile, cfg, notify=False)
+        # 归档对账：走到这里说明本产品列表拉取成功，消失判定才可信
+        try:
+            _archive_sweep(cfg, profile, by_id)
+        except Exception:
+            log.exception("zentao: 产品 %s 归档对账异常，跳过", profile.get("product"))
     return claimed
 
 
+def _now():
+    """当前时刻（独立成函数供测试注入时钟）。"""
+    return datetime.now()
+
+
+def _holiday_gap(now):
+    """今天是法定休假日 → 返回下一个工作日同时刻（作顺延后的 next_scan）；
+    今天是工作日（含调休补班的周末）或判定不了（库缺失/年份超出已知范围）
+    → None 照常扫描。只拦「扫描认领」，对账不受影响。"""
+    if _hcal is None:
+        return None
+    d = now.date()
+    try:
+        if _hcal.is_workday(d):
+            return None
+    except (NotImplementedError, ValueError, KeyError):
+        return None
+    d += timedelta(days=1)
+    for _ in range(366):
+        try:
+            if _hcal.is_workday(d):
+                return datetime.combine(d, now.time())
+        except (NotImplementedError, ValueError, KeyError):
+            break
+        d += timedelta(days=1)
+    return None   # 范围不足：按次日顺延兜底，到时再判一次
+
+
 def _poll(force=False):
     """执行一次扫描；同一进程内只允许一个扫描实例。"""
     if not _SCAN_LOCK.acquire(blocking=False):
@@ -2222,6 +2356,17 @@ def _poll_unlocked(force=False):
                             "error": "", "skipped": "not_due"}
             except ValueError:
                 pass
+        # 到点但今天是法定休假日：不认领新 Bug，next_scan 顺延到下一个工作日
+        # 同时刻（调休补班的周末照常扫；手动「立即扫描」不受此闸约束）
+        if cfg.get("skip_holidays"):
+            gap = _holiday_gap(_now())
+            if gap is not None:
+                with _LOCK:
+                    _STATE["next_scan"] = gap.strftime("%Y-%m-%d %H:%M:%S")
+                    _save_locked()
+                log.info("zentao: 法定休假日，扫描顺延至 %s", _STATE["next_scan"])
+                return {"ok": True, "claimed": 0, "reconciled": True,
+                        "error": "", "skipped": "holiday"}
     err = ""
     try:
         out["claimed"] = _scan(cfg)
@@ -2400,8 +2545,12 @@ def save_config(patch):
                 except (TypeError, ValueError):
                     raise ValueError("interval_minutes 必须是 %d-%d 的整数"
                                      % (INTERVAL_MIN, INTERVAL_MAX))
-            elif k in ("auto_resolve", "auto_merge", "triage_ai", "poll_enabled"):
+            elif k in ("auto_resolve", "auto_merge", "triage_ai", "poll_enabled",
+                       "skip_holidays"):
                 v = bool(v)
+                if k == "skip_holidays" and v and _hcal is None:
+                    raise ValueError("跳过法定节假日需要先安装 chinesecalendar 库"
+                                     "（pip install chinesecalendar）")
             cfg[k] = v
         _STATE["config"] = cfg
         if cfg.get("poll_enabled"):