codebee 0.1.65 → 0.1.66

package/app/core/error_codes.py

@@ -1,65 +1,220 @@
-# -*- coding: utf-8 -*-
-"""错误码体系：把 runner 失败归一成可枚举的 ErrorCode。
-
-设计稿：docs/migration/01-defense-patterns.md §5D + §2D。
-参考 dsh docs/subsystems/workflow.zh.md:108-120 WorkflowErrorCode。
-"""
-from __future__ import annotations
-
-from enum import Enum
-
-
-class ErrorCode(str, Enum):
-    """所有 runner 失败的可枚举分类。
-
-    字段值是 JSON 友好的字符串，方便持久化到 store / 跨进程传输。
-    """
-
-    # === fatal：脚本/参数错，必须人工修 ===
-    VENDOR_ERROR = "VENDOR_ERROR"          # vendor 内部崩溃（exit != 0）
-    VENDOR_REFUSAL = "VENDOR_REFUSAL"      # 明确拒绝（claude is_error / qwen refusal）
-    PARSE_FAIL = "PARSE_FAIL"              # 输出无法解析为预期 schema
-    ENV_BLOCK = "ENV_BLOCK"                # approval 拒 / 策略阻断
-    SIGNAL_IGNORED = "SIGNAL_IGNORED"      # 发了 SIGTERM/任务杀不掉
-
-    # === item：可重试 ===
-    TIMEOUT = "TIMEOUT"
-    CANCELLED = "CANCELLED"
-    MAX_TOKENS = "MAX_TOKENS"              # 模型撑爆
-    EMPTY = "EMPTY"                        # 空输出
-    NETWORK = "NETWORK"                    # 连接失败
-    CONTEXT_OVERFLOW = "CONTEXT_OVERFLOW"  # 触发压缩（与 MAX_TOKENS 不同）
-
-    # === ok ===
-    # "" 表示成功，不设枚举值
-
-
-# 哪些码视为 fatal（job 终止，不进入重试队列）
-FATAL_CODES = frozenset({
-    ErrorCode.VENDOR_ERROR,
-    ErrorCode.VENDOR_REFUSAL,
-    ErrorCode.PARSE_FAIL,
-    ErrorCode.ENV_BLOCK,
-    ErrorCode.SIGNAL_IGNORED,
-})
-
-
-def is_fatal(code) -> bool:
-    """code 可以是 ErrorCode 或字符串（如来自 dict.get('error_code')）。"""
-    if not code:
-        return False
-    try:
-        return ErrorCode(code) in FATAL_CODES
-    except ValueError:
-        return False
-
-
-def is_retryable(code) -> bool:
-    """item 类（可重试）。"""
-    if not code:
-        return False
-    try:
-        ec = ErrorCode(code)
-        return ec not in FATAL_CODES and ec != ErrorCode.CANCELLED
-    except ValueError:
-        return False
+# -*- coding: utf-8 -*-
+"""错误码体系：把 runner 失败归一成可枚举的 ErrorCode。
+
+设计稿：docs/migration/01-defense-patterns.md §5D + §2D。
+参考 dsh docs/subsystems/workflow.zh.md:108-120 WorkflowErrorCode。
+"""
+from __future__ import annotations
+
+from enum import Enum
+import re
+
+
+class ErrorCode(str, Enum):
+    """所有 runner 失败的可枚举分类。
+
+    字段值是 JSON 友好的字符串，方便持久化到 store / 跨进程传输。
+    """
+
+    # === fatal：脚本/参数错，必须人工修 ===
+    VENDOR_ERROR = "VENDOR_ERROR"          # vendor 内部崩溃（exit != 0）
+    VENDOR_REFUSAL = "VENDOR_REFUSAL"      # 明确拒绝（claude is_error / qwen refusal）
+    PARSE_FAIL = "PARSE_FAIL"              # 输出无法解析为预期 schema
+    ENV_BLOCK = "ENV_BLOCK"                # approval 拒 / 策略阻断
+    SIGNAL_IGNORED = "SIGNAL_IGNORED"      # 发了 SIGTERM/任务杀不掉
+
+    # === item：可重试 ===
+    TIMEOUT = "TIMEOUT"
+    CANCELLED = "CANCELLED"
+    MAX_TOKENS = "MAX_TOKENS"              # 模型撑爆
+    EMPTY = "EMPTY"                        # 空输出
+    NETWORK = "NETWORK"                    # 连接失败
+    UPSTREAM_SERVER = "UPSTREAM_SERVER"    # 上游 5xx / 服务端异常
+    MISSING_CREDENTIAL = "MISSING_CREDENTIAL"  # 当前路由没有可用 API key
+    AUTH = "AUTH"                          # 凭据认证失败（401）
+    FORBIDDEN = "FORBIDDEN"                # 上游拒绝当前账号/模型/请求（403）
+    QUOTA = "QUOTA"                        # 欠费、余额或调用额度耗尽
+    RATE_LIMIT = "RATE_LIMIT"              # 短时速率/并发限制（429）
+    LOCAL_STATE = "LOCAL_STATE"            # 本地 CLI 状态/锁冲突
+    CONTEXT_OVERFLOW = "CONTEXT_OVERFLOW"  # 触发压缩（与 MAX_TOKENS 不同）
+
+    # === ok ===
+    # "" 表示成功，不设枚举值
+
+
+# 哪些码视为 fatal（job 终止，不进入重试队列）
+FATAL_CODES = frozenset({
+    ErrorCode.VENDOR_ERROR,
+    ErrorCode.VENDOR_REFUSAL,
+    ErrorCode.PARSE_FAIL,
+    ErrorCode.ENV_BLOCK,
+    ErrorCode.SIGNAL_IGNORED,
+})
+
+
+_MISSING_CREDENTIAL = (
+    "no api key", "missing api key", "api key not found", "api key is required",
+    "missing environment variable", "missing env var",
+)
+_AUTH = (
+    "invalid api key", "invalid_api_key", "api key is invalid", "incorrect api key",
+    "api key not valid", "api key 不可用", "鉴权失败", "认证失败",
+    "unauthorized", "unauthorised", "authentication failed", "authentication error",
+    "authentication_error", "invalid token", "invalid_token", "http 401",
+    "status code 401",
+)
+_FORBIDDEN = ("forbidden", "request not allowed", "permission denied", "access denied")
+_RATE_LIMIT = ("rate limit", "rate_limit", "too many requests", "http 429", "status code 429",
+               "concurrent limit", "concurrency limit", "too many concurrent",
+               "并发限制", "并发超限", "限流")
+_QUOTA = ("insufficient balance", "insufficient quota", "insufficient credit",
+          "quota", "balance is insufficient", "credit balance", "billing", "arrears",
+          "payment required", "http 402", "status code 402", "欠费", "余额", "额度",
+          "usage limit", "limit exceeded", "quota exhausted")
+_SERVER = ("unexpected server error", "internal server error", "server error",
+           "overloaded", "temporarily", "unavailable", "service unavailable",
+           "internal error", "no available channel", "unknown model")
+_NETWORK = ("stream disconnected", "stream closed", "connection reset", "connection aborted",
+            "broken pipe", "socket hang up", "econnrefused", "enotfound", "getaddrinfo",
+            "name resolution", "network error", "network is unreachable", "reqwest error stream",
+            "error sending request for url", "no route to host", "connection error",
+            "reset by peer", "channel is closed", "name or service not known",
+            "输出停滞")
+_LOCAL_STATE = ("database is locked", "database locked", "sqlite busy",
+                "unable to open database file")
+_TRANSIENT_HINTS = ("stream closed before response.completed", "output stall", "timed out",
+                    "timeout", "output stalled", "ENOTFOUND", "Reconnecting...",
+                    "initialize", "[1211]")
+_REFUSAL = ("can't help with this", "couldn't help with this", '"stop_reason":"refusal"',
+            "content policy", "content filtering")
+
+
+def error_code_value(code):
+    """Return the stable wire value for ErrorCode or a string-like code."""
+    if isinstance(code, ErrorCode):
+        return code.value
+    return str(code or "")
+
+
+def classify_error_text(error):
+    """Map a vendor/CLI error message to the shared failure taxonomy.
+
+    Return ``None`` when text does not provide a reliable signal. Cancellation and
+    timeout flags still take precedence in the caller because text may be noisy.
+    More specific credential/permission/quota signals are checked before generic
+    server and transport wording.
+    """
+    text = str(error or "").lower()
+    if not text:
+        return None
+    if re.search(r"(?:http(?:/\d(?:\.\d)?)?\s*|status(?:\s+code)?\s*[:=]?\s*)401\b", text):
+        return ErrorCode.AUTH
+    # An explicit HTTP 403 is authoritative even when the provider describes the
+    # permission problem with credential wording (for example, "invalid API key
+    # for this model"). Do not cool down the key as an authentication failure.
+    if (re.search(r"(?:http(?:/\d(?:\.\d)?)?\s*|status(?:\s+code)?\s*[:=]?\s*|"
+                  r"code\s*[:=]?\s*)403\b", text)
+            or re.search(r"(?:^|:\s*)403(?=$|[\s:{])", text)):
+        return ErrorCode.FORBIDDEN
+    if any(marker in text for marker in _MISSING_CREDENTIAL):
+        return ErrorCode.MISSING_CREDENTIAL
+    if (any(marker in text for marker in _AUTH)
+            or re.search(r"(?:http(?:/\d(?:\.\d)?)?\s*|status(?:\s+code)?\s*[:=]?\s*)401\b", text)):
+        return ErrorCode.AUTH
+    if any(marker in text for marker in _FORBIDDEN):
+        return ErrorCode.FORBIDDEN
+    if (re.search(r"(?:http(?:/\d(?:\.\d)?)?\s*|status(?:\s+code)?\s*[:=]?\s*)429\b", text)
+            or any(marker in text for marker in _RATE_LIMIT)):
+        return ErrorCode.RATE_LIMIT
+    if (re.search(r"(?:http(?:/\d(?:\.\d)?)?\s*|status(?:\s+code)?\s*[:=]?\s*)402\b", text)
+            or any(marker in text for marker in _QUOTA)):
+        return ErrorCode.QUOTA
+    if any(marker in text for marker in _LOCAL_STATE):
+        return ErrorCode.LOCAL_STATE
+    if (re.search(r"(?<!\d)5\d\d(?!\d)", text)
+            or any(marker in text for marker in _SERVER)):
+        return ErrorCode.UPSTREAM_SERVER
+    if ("timed out" in text or "timeout" in text or "deadline exceeded" in text
+            or re.search(r"(?<!\d)408(?!\d)", text)):
+        return ErrorCode.TIMEOUT
+    if any(marker in text for marker in _NETWORK):
+        return ErrorCode.NETWORK
+    if any(marker in text for marker in _REFUSAL):
+        return ErrorCode.VENDOR_REFUSAL
+    return None
+
+
+def is_auth_error(error):
+    return classify_error_text(error) == ErrorCode.AUTH
+
+
+def is_forbidden_error(error):
+    return classify_error_text(error) == ErrorCode.FORBIDDEN
+
+
+def is_quota_error(error):
+    return classify_error_text(error) in (ErrorCode.QUOTA, ErrorCode.RATE_LIMIT)
+
+
+def is_transient_error(error):
+    text = str(error or "").lower()
+    return classify_error_text(text) in (
+        ErrorCode.NETWORK, ErrorCode.UPSTREAM_SERVER, ErrorCode.TIMEOUT,
+        ErrorCode.QUOTA, ErrorCode.RATE_LIMIT, ErrorCode.LOCAL_STATE,
+    ) or any(marker.lower() in text for marker in _TRANSIENT_HINTS)
+
+
+def is_rate_limited_error(error):
+    return classify_error_text(error) == ErrorCode.RATE_LIMIT
+
+
+_SAFE_ERROR_SUMMARIES = {
+    ErrorCode.VENDOR_ERROR: "供应商或 CLI 调用失败（未识别具体原因）",
+    ErrorCode.VENDOR_REFUSAL: "上游拒绝了本次请求",
+    ErrorCode.PARSE_FAIL: "模型响应无法解析",
+    ErrorCode.ENV_BLOCK: "本机环境或执行策略阻断",
+    ErrorCode.SIGNAL_IGNORED: "子进程未响应终止信号",
+    ErrorCode.TIMEOUT: "调用超时",
+    ErrorCode.CANCELLED: "调用已取消",
+    ErrorCode.MAX_TOKENS: "模型输出达到 token 上限",
+    ErrorCode.EMPTY: "模型未返回内容",
+    ErrorCode.NETWORK: "网络连接或数据流中断",
+    ErrorCode.UPSTREAM_SERVER: "上游服务端异常",
+    ErrorCode.MISSING_CREDENTIAL: "所选路由未找到可用 API 凭据",
+    ErrorCode.AUTH: "API 凭据认证失败",
+    ErrorCode.FORBIDDEN: "上游拒绝账号、模型或请求权限",
+    ErrorCode.QUOTA: "上游余额或调用额度不足",
+    ErrorCode.RATE_LIMIT: "上游触发速率或并发限制",
+    ErrorCode.LOCAL_STATE: "CLI 本地状态或数据库锁冲突",
+    ErrorCode.CONTEXT_OVERFLOW: "输入上下文超出模型容量",
+}
+
+
+def safe_error_summary(code):
+    """A content-free summary suitable for durable diagnostics and telemetry."""
+    try:
+        value = ErrorCode(code)
+    except (ValueError, TypeError):
+        value = ErrorCode.VENDOR_ERROR
+    return _SAFE_ERROR_SUMMARIES.get(value, _SAFE_ERROR_SUMMARIES[ErrorCode.VENDOR_ERROR])
+
+
+def is_fatal(code) -> bool:
+    """code 可以是 ErrorCode 或字符串（如来自 dict.get('error_code')）。"""
+    if not code:
+        return False
+    try:
+        return ErrorCode(code) in FATAL_CODES
+    except ValueError:
+        return False
+
+
+def is_retryable(code) -> bool:
+    """item 类（可重试）。"""
+    if not code:
+        return False
+    try:
+        ec = ErrorCode(code)
+        return ec not in FATAL_CODES and ec != ErrorCode.CANCELLED
+    except ValueError:
+        return False

package/app/core/errorlog.py

@@ -1,137 +1,137 @@
-# -*- coding: utf-8 -*-
-"""错误台账：把分散在各处的失败（步骤失败/超时/看门狗击杀、CLI 启动失败）
-统一落成结构化记录——「客户遇到了问题我们根本不知道」的第一块拼图。
-
-落盘：data/errors/errors-YYYYMM.jsonl（按月分文件，append-only，一行一记录）。
-与 usage.py 同一套纪律：
-- record() 任何异常都吞掉——埋点失败绝不能影响任务执行；
-- 落盘前先脱敏（scrub_text）：密钥/令牌/绝对路径/超长文本一概不进台账。
-
-隐私红线（上传侧在 telemetry.py 再兜一道底）：
-1. API KEY / 凭据绝不入台账；
-2. 任务正文 / 章节内容（用户版权内容）绝不入台账——detail 只允许失败原因摘录；
-3. 绝对路径剥成相对片段。
-"""
-from __future__ import annotations
-
-import json
-import threading
-import time
-import uuid
-
-from . import paths
-from .redact import scrub_text
-
-LOCK = threading.RLock()
-
-FIELDS = ("id", "ts", "day", "category", "reason", "detail", "provider",
-          "model", "tool", "role", "run_id", "task_id", "step",
-          "exit_code", "app_version", "os")
-
-# detail 摘录上限（字符）：失败原因足够，不要变成日志搬运
-DETAIL_LIMIT = 600
-
-
-# ---------------------------------------------------------------- 脱敏
-
-def _coerce_str(v, limit):
-    return str(v or "")[:limit]
-
-
-def _os_tag():
-    import sys
-    return {"win32": "windows", "darwin": "macos"}.get(sys.platform, sys.platform)
-
-
-def _version():
-    try:
-        from . import selfupdate
-        return str(selfupdate.package_version() or "dev")
-    except Exception:
-        return "dev"
-
-
-def record(category="", reason="", detail="", provider="", model="", tool="",
-           role="", run_id="", task_id="", step=0, exit_code=None, source=""):
-    """追加一条错误记录。缺省字段留空；任何异常都吞掉（埋点不拖垮业务）。
-
-    category: step / run / launch / service（枚举放宽，未知值原样入库）
-    reason:   失败原因码（error_codes.ErrorCode 的字符串值优先，如 TIMEOUT）
-    detail:   失败摘录（内部先过 scrub_text，绝不存原文）
-    """
-    try:
-        rec = {
-            "id": "%s-%s" % (time.strftime("%Y-%m-%d %H:%M:%S"),
-                             uuid.uuid4().hex[:8]),
-            "ts": time.strftime("%Y-%m-%d %H:%M:%S"),
-            "day": time.strftime("%Y-%m-%d"),
-            "category": _coerce_str(category, 24) or "step",
-            "reason": _coerce_str(reason, 40) or "UNKNOWN",
-            "detail": scrub_text(detail),
-            "provider": _coerce_str(provider, 60),
-            "model": _coerce_str(model, 80),
-            "tool": _coerce_str(tool, 24),
-            "role": _coerce_str(role, 40),
-            "run_id": _coerce_str(run_id, 64),
-            "task_id": _coerce_str(task_id, 64),
-            "step": int(step or 0),
-            "exit_code": exit_code,
-            "app_version": _version()[:24],
-            "os": _os_tag(),
-        }
-        line = json.dumps(rec, ensure_ascii=False)
-        day = rec["day"]
-        with LOCK:
-            paths.ERRORS_DIR.mkdir(parents=True, exist_ok=True)
-            with open(paths.ERRORS_DIR / ("errors-%s.jsonl" % day[:7].replace("-", "")),
-                      "a", encoding="utf-8") as f:
-                f.write(line + "\n")
-    except Exception:
-        pass
-
-
-def iter_records(days=0):
-    """按时间范围读取台账（days=0 表示全部）。返回按写入顺序的记录列表。"""
-    out = []
-    try:
-        files = sorted(paths.ERRORS_DIR.glob("errors-*.jsonl")) if paths.ERRORS_DIR.is_dir() else []
-    except Exception:
-        return out
-    since_day = ""
-    if days:
-        import datetime
-        since_day = (datetime.date.today()
-                     - datetime.timedelta(days=int(days) - 1)).isoformat()
-    for p in files:
-        try:
-            for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
-                line = line.strip()
-                if not line.startswith("{"):
-                    continue
-                try:
-                    r = json.loads(line)
-                except Exception:
-                    continue
-                if not isinstance(r, dict):
-                    continue
-                if since_day and str(r.get("day", "")) < since_day:
-                    continue
-                out.append(r)
-        except Exception:
-            continue
-    return out
-
-
-def pending_since(cursor, limit=200):
-    """取 cursor（上次上传到的记录 id）之后的记录，最多 limit 条。
-
-    id 以时间开头、uuid 尾巴保证唯一且字典序≈时间序；返回 (records, new_cursor)。
-    """
-    try:
-        recs = [r for r in iter_records(0) if str(r.get("id") or "") > str(cursor or "")]
-        recs.sort(key=lambda r: str(r.get("id") or ""))
-        batch = recs[:max(1, int(limit))]
-        new_cursor = str(batch[-1].get("id")) if batch else str(cursor or "")
-        return batch, new_cursor
-    except Exception:
-        return [], str(cursor or "")
+# -*- coding: utf-8 -*-
+"""错误台账：把分散在各处的失败（步骤失败/超时/看门狗击杀、CLI 启动失败）
+统一落成结构化记录——「客户遇到了问题我们根本不知道」的第一块拼图。
+
+落盘：data/errors/errors-YYYYMM.jsonl（按月分文件，append-only，一行一记录）。
+与 usage.py 同一套纪律：
+- record() 任何异常都吞掉——埋点失败绝不能影响任务执行；
+- 落盘前先脱敏（scrub_text）：密钥/令牌/绝对路径/超长文本一概不进台账。
+
+隐私红线（上传侧在 telemetry.py 再兜一道底）：
+1. API KEY / 凭据绝不入台账；
+2. 任务正文 / 章节内容（用户版权内容）绝不入台账——detail 只允许失败原因摘录；
+3. 绝对路径剥成相对片段。
+"""
+from __future__ import annotations
+
+import json
+import threading
+import time
+import uuid
+
+from . import paths
+from .redact import scrub_text
+
+LOCK = threading.RLock()
+
+FIELDS = ("id", "ts", "day", "category", "reason", "detail", "provider",
+          "model", "tool", "role", "run_id", "task_id", "step",
+          "exit_code", "app_version", "os")
+
+# detail 摘录上限（字符）：失败原因足够，不要变成日志搬运
+DETAIL_LIMIT = 600
+
+
+# ---------------------------------------------------------------- 脱敏
+
+def _coerce_str(v, limit):
+    return str(v or "")[:limit]
+
+
+def _os_tag():
+    import sys
+    return {"win32": "windows", "darwin": "macos"}.get(sys.platform, sys.platform)
+
+
+def _version():
+    try:
+        from . import selfupdate
+        return str(selfupdate.package_version() or "dev")
+    except Exception:
+        return "dev"
+
+
+def record(category="", reason="", detail="", provider="", model="", tool="",
+           role="", run_id="", task_id="", step=0, exit_code=None, source=""):
+    """追加一条错误记录。缺省字段留空；任何异常都吞掉（埋点不拖垮业务）。
+
+    category: step / run / launch / service（枚举放宽，未知值原样入库）
+    reason:   失败原因码（error_codes.ErrorCode 的字符串值优先，如 TIMEOUT）
+    detail:   失败摘录（内部先过 scrub_text，绝不存原文）
+    """
+    try:
+        rec = {
+            "id": "%s-%s" % (time.strftime("%Y-%m-%d %H:%M:%S"),
+                             uuid.uuid4().hex[:8]),
+            "ts": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "day": time.strftime("%Y-%m-%d"),
+            "category": _coerce_str(category, 24) or "step",
+            "reason": _coerce_str(reason, 40) or "UNKNOWN",
+            "detail": scrub_text(detail),
+            "provider": _coerce_str(provider, 60),
+            "model": _coerce_str(model, 80),
+            "tool": _coerce_str(tool, 24),
+            "role": _coerce_str(role, 40),
+            "run_id": _coerce_str(run_id, 64),
+            "task_id": _coerce_str(task_id, 64),
+            "step": int(step or 0),
+            "exit_code": exit_code,
+            "app_version": _version()[:24],
+            "os": _os_tag(),
+        }
+        line = json.dumps(rec, ensure_ascii=False)
+        day = rec["day"]
+        with LOCK:
+            paths.ERRORS_DIR.mkdir(parents=True, exist_ok=True)
+            with open(paths.ERRORS_DIR / ("errors-%s.jsonl" % day[:7].replace("-", "")),
+                      "a", encoding="utf-8") as f:
+                f.write(line + "\n")
+    except Exception:
+        pass
+
+
+def iter_records(days=0):
+    """按时间范围读取台账（days=0 表示全部）。返回按写入顺序的记录列表。"""
+    out = []
+    try:
+        files = sorted(paths.ERRORS_DIR.glob("errors-*.jsonl")) if paths.ERRORS_DIR.is_dir() else []
+    except Exception:
+        return out
+    since_day = ""
+    if days:
+        import datetime
+        since_day = (datetime.date.today()
+                     - datetime.timedelta(days=int(days) - 1)).isoformat()
+    for p in files:
+        try:
+            for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
+                line = line.strip()
+                if not line.startswith("{"):
+                    continue
+                try:
+                    r = json.loads(line)
+                except Exception:
+                    continue
+                if not isinstance(r, dict):
+                    continue
+                if since_day and str(r.get("day", "")) < since_day:
+                    continue
+                out.append(r)
+        except Exception:
+            continue
+    return out
+
+
+def pending_since(cursor, limit=200):
+    """取 cursor（上次上传到的记录 id）之后的记录，最多 limit 条。
+
+    id 以时间开头、uuid 尾巴保证唯一且字典序≈时间序；返回 (records, new_cursor)。
+    """
+    try:
+        recs = [r for r in iter_records(0) if str(r.get("id") or "") > str(cursor or "")]
+        recs.sort(key=lambda r: str(r.get("id") or ""))
+        batch = recs[:max(1, int(limit))]
+        new_cursor = str(batch[-1].get("id")) if batch else str(cursor or "")
+        return batch, new_cursor
+    except Exception:
+        return [], str(cursor or "")