codebee 0.1.66 → 0.1.67

package/app/core/operations.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+"""Append-only external operation ledger.
+
+External writes cannot promise exactly-once delivery.  Each write therefore
+has a stable operation id and an explicit state: pending, confirmed, failed or
+unknown.  Unknown means the caller must reconcile with the remote system before
+retrying; it is never treated as success.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import secrets
+import threading
+import time
+import urllib.error
+from pathlib import Path
+
+from . import paths
+
+LOCK = threading.RLock()
+STATUSES = ("pending", "confirmed", "failed", "unknown")
+PENDING_TTL_SECONDS = 15 * 60
+
+
+def _file(day=None) -> Path:
+    day = day or time.strftime("%Y-%m-%d")
+    return paths.DATA_DIR / "operations" / ("operations-%s.jsonl" % day[:7].replace("-", ""))
+
+
+def _read_all():
+    rows = []
+    root = paths.DATA_DIR / "operations"
+    try:
+        files = sorted(root.glob("operations-*.jsonl"))
+    except OSError:
+        files = []
+    for fp in files:
+        try:
+            for line in fp.read_text(encoding="utf-8").splitlines():
+                try:
+                    row = json.loads(line)
+                except (TypeError, ValueError):
+                    continue
+                if isinstance(row, dict) and row.get("operation_id"):
+                    rows.append(row)
+        except OSError:
+            continue
+    return rows
+
+
+def _write_all(rows):
+    """Test/support migration helper; production writes use append-only _append."""
+    root = paths.DATA_DIR / "operations"
+    root.mkdir(parents=True, exist_ok=True)
+    fp = _file()
+    fp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows),
+                  encoding="utf-8")
+
+
+def _append(row):
+    try:
+        fp = _file(row.get("submitted_at") and
+                   time.strftime("%Y-%m-%d", time.localtime(float(row["submitted_at"]))))
+        fp.parent.mkdir(parents=True, exist_ok=True)
+        with fp.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
+            handle.flush()
+    except Exception:
+        # Observability must not turn a remote write into a local failure.
+        pass
+
+
+def _request_hash(request) -> str:
+    raw = json.dumps(request or {}, ensure_ascii=False, sort_keys=True,
+                     separators=(",", ":"), default=str)
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def _operation_id(target: str) -> str:
+    stamp = time.strftime("%Y%m%d-%H%M%S")
+    return "op-%s-%s-%s" % (stamp, secrets.token_hex(4),
+                             hashlib.sha256(str(target).encode()).hexdigest()[:6])
+
+
+def _safe_meta(metadata):
+    if not isinstance(metadata, dict):
+        return {}
+    out = {}
+    for key, value in metadata.items():
+        name = str(key)[:40]
+        if any(token in name.lower() for token in ("key", "token", "password", "secret")):
+            continue
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            out[name] = str(value)[:300] if isinstance(value, str) else value
+    return out
+
+
+def begin(target, request=None, *, task_id="", run_id="", metadata=None):
+    operation_id = _operation_id(target)
+    now = time.time()
+    row = {
+        "operation_id": operation_id,
+        "target": str(target or "")[:160],
+        "request_hash": _request_hash(request),
+        "submitted_at": now,
+        "status": "pending",
+        "remote_receipt": "",
+        "error": "",
+        "last_reconciled_at": None,
+        "task_id": str(task_id or "")[:80],
+        "run_id": str(run_id or "")[:80],
+        "metadata": _safe_meta(metadata),
+    }
+    with LOCK:
+        _append(row)
+    return operation_id
+
+
+def update(operation_id, status, *, remote_receipt="", error="",
+           metadata=None, reconciled=False):
+    operation_id = str(operation_id or "").strip()
+    status = str(status or "").strip().lower()
+    if not operation_id or status not in STATUSES:
+        return False
+    with LOCK:
+        # Read and transition-check under the same lock as append.  Two
+        # concurrent callbacks must not race a pending operation into
+        # conflicting terminal states.
+        current = get(operation_id)
+        if current is None:
+            return False
+        current_status = str(current.get("status") or "pending")
+        # External effects are monotonic: a confirmed write must never be
+        # downgraded by cleanup/error handling.  Unknown/failed can only be
+        # promoted by an explicit remote reconciliation.
+        if current_status == "confirmed" and status != "confirmed":
+            return False
+        if current_status in ("unknown", "failed") and status == "confirmed" and not reconciled:
+            return False
+        if current_status in ("unknown", "failed") and status != current_status and not reconciled:
+            return False
+        if status == "confirmed" and not remote_receipt:
+            remote_receipt = current.get("remote_receipt") or ""
+        now = time.time()
+        row = {
+            "operation_id": operation_id,
+            "status": status,
+            "remote_receipt": str(remote_receipt or "")[:300],
+            "error": str(error or "")[:500],
+            "last_reconciled_at": now if reconciled else None,
+            "updated_at": now,
+        }
+        if metadata:
+            row["metadata"] = _safe_meta(metadata)
+        _append(row)
+    return True
+
+
+def confirm(operation_id, remote_receipt="", *, metadata=None):
+    return update(operation_id, "confirmed", remote_receipt=remote_receipt,
+                  metadata=metadata)
+
+
+def fail(operation_id, error="", *, metadata=None):
+    return update(operation_id, "failed", error=error, metadata=metadata)
+
+
+def mark_unknown(operation_id, error="", *, metadata=None):
+    return update(operation_id, "unknown", error=error, metadata=metadata)
+
+
+def reconcile(operation_id, status, remote_receipt="", error="", metadata=None):
+    """Record a remote lookup result for an unknown operation."""
+    return update(operation_id, status, remote_receipt=remote_receipt, error=error,
+                  metadata=metadata, reconciled=True)
+
+
+def get(operation_id):
+    operation_id = str(operation_id or "")
+    latest = None
+    with LOCK:
+        for row in _read_all():
+            if row.get("operation_id") == operation_id:
+                latest = dict(latest or {}, **row)
+    return latest
+
+
+def recent(*, target="", status="", limit=100):
+    latest = {}
+    with LOCK:
+        for row in _read_all():
+            if target and str(row.get("target") or "") != str(target):
+                continue
+            latest[row["operation_id"]] = dict(latest.get(row["operation_id"]) or {}, **row)
+    rows = list(latest.values())
+    if status:
+        rows = [r for r in rows if r.get("status") == status]
+    rows.sort(key=lambda r: float(r.get("updated_at") or r.get("submitted_at") or 0), reverse=True)
+    return rows[:max(1, min(1000, int(limit or 100)))]
+
+
+def recover_pending(*, now=None, ttl_seconds=PENDING_TTL_SECONDS):
+    """Turn stale pending writes into unknown so restart never replays blindly."""
+    now = time.time() if now is None else float(now)
+    changed = 0
+    for row in recent(status="pending", limit=1000):
+        submitted = float(row.get("submitted_at") or 0)
+        if submitted and now - submitted > max(1, float(ttl_seconds)):
+            if mark_unknown(row["operation_id"], "进程重启后未收到远端回执，请先对账"):
+                changed += 1
+    return changed
+
+
+def status_for_exception(error) -> str:
+    text = str(error or "").lower()
+    if isinstance(error, (TimeoutError, ConnectionError, urllib.error.URLError)) or any(x in text for x in
+            ("timeout", "timed out", "超时", "connection", "connect", "connection reset",
+             "断开", "断连", "连不上", "网络", "network", "unreachable", "disconnect")):
+        return "unknown"
+    return "failed"
+
+
+def finish_exception(operation_id, error):
+    status = status_for_exception(error)
+    if status == "unknown":
+        return mark_unknown(operation_id, str(error))
+    return fail(operation_id, str(error))

package/app/core/paihang.py

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 """扫榜选材：抓取公开排行榜页，产出选题分析素材。
 
-数据源（双源聚合，任一失败不影响另一源）：
-- 七猫：www.qimao.com/paihang/（公开可抓，2026-09-19 实测 200/106KB）
-- 番茄 Web 版：fanqienovel.com/rank（公开可抓，题材分类+书名在
-  中文 a 标签里；App 端接口有 SecuritySign/X-Argus 签名——那才是难路，
-  Web 版榜单页不需要签名，别走弯路）
+数据源（四源聚合，任一失败不影响其余源；2026-09-24 实测全 200）：
+- 七猫：www.qimao.com/paihang/（106KB）
+- 番茄 Web 版：fanqienovel.com/rank（题材分类+书名在中文 a 标签里；
+  App 端接口有 SecuritySign/X-Argus 签名——那才是难路，Web 版榜单页
+  不需要签名，别走弯路）
+- 起点移动版：m.qidian.com/rank（106KB，直接出书名；主站 www.qidian.com
+  被 WAF 拦 202/209B，移动页不需要签名）
+- 纵横：www.zongheng.com/rank（65KB，书名+统计标签混排，靠噪音词过滤）
 
 下载走 curl 子进程（与 covergen 同款），解析用宽松正则——页面结构变了
 宁可返回空（调用方回落普通直连提示词），不做脆弱的强解析。
@@ -19,16 +22,24 @@ log = logging.getLogger(__name__)
 
 QIMAO_RANK_URL = "https://www.qimao.com/paihang/"
 FANQIE_RANK_URL = "https://fanqienovel.com/rank"
+QIDIAN_M_RANK_URL = "https://m.qidian.com/rank/"
+ZONGHENG_RANK_URL = "https://www.zongheng.com/rank/"
 _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/126 Safari/537.36")
 
-# 页面 UI 噪音词（非书目/分类）：页面 chrome、导航、tab 名、登录/搜索等
+# 页面 UI 噪音词（非书目/分类）：页面 chrome、导航、tab 名、登录/搜索、
+# 榜单 tab 与统计标签（起点移动版 tab 名、纵横月票/人气/点击等）
 _NOISE = {"加入书架", "立即阅读", "开始阅读", "免费阅读", "全部", "分类", "排行",
           "排行榜", "完本", "连载中", "阅读", "书城", "男生", "女生",
           "帮助中心", "作家助手", "客服", "登录", "注册", "立即登录",
           "搜索", "热门作品排行", "大热榜", "新书榜", "完结榜", "收藏榜",
           "更新榜", "日榜", "月榜", "蝉联", "榜首", "热度", "免费", "畅销",
-          "下载", "客户端", "首页", "我的", "设置", "更多", "返回", "关于"}
+          "下载", "客户端", "首页", "我的", "设置", "更多", "返回", "关于",
+          "月票榜", "阅读榜", "书友榜", "推荐榜", "签约榜", "新人榜",
+          "畅销榜", "大家都在搜", "男生小说排行榜", "女生小说排行榜",
+          "男频阅读榜", "女频阅读榜", "男频新书榜", "番茄小说",
+          "登录后获得更多特色功能", "安装起点读书客户端", "看更多正版好书",
+          "月票", "人气", "点击", "万字", "推荐", "收藏", "鲜花", "打赏", "推荐票"}
 
 _CJK_RE = re.compile(r">([\u4e00-\u9fa5]{2,12})<")
 
@@ -70,39 +81,57 @@ def fetch_fanqie_rank(limit=30):
     return _extract_cjk(_curl_text(FANQIE_RANK_URL), limit)
 
 
+def fetch_qidian_rank(limit=30):
+    """起点移动版榜单：返回元素列表，抓取失败返回 []。"""
+    return _extract_cjk(_curl_text(QIDIAN_M_RANK_URL), limit)
+
+
+def fetch_zongheng_rank(limit=30):
+    """纵横排行榜：返回元素列表，抓取失败返回 []。"""
+    return _extract_cjk(_curl_text(ZONGHENG_RANK_URL), limit)
+
+
+# 源清单：顺序即素材拼接顺序（七猫/番茄为元老源保持在前）。
+# 存函数名、调用期 globals() 解析——mock.patch 换模块属性才能生效
+# （存函数引用会把定义期原函数钉死，测试会穿透打真网络）。
+_SOURCES = (
+    ("七猫排行榜素材", "fetch_qimao_rank"),
+    ("番茄排行榜素材", "fetch_fanqie_rank"),
+    ("起点排行榜素材", "fetch_qidian_rank"),
+    ("纵横排行榜素材", "fetch_zongheng_rank"),
+)
+
+
 def fetch_rank_items(limit=40):
-    """双源聚合：七猫+番茄各抓一份（去重合并，七猫在前）。
+    """四源聚合：各抓一份（跨源去重合并，按源清单顺序）。
 
-    两源都失败才返回 []——单源挂了另一源仍可用，扫榜永不因单点挡任务。"""
-    qm = fetch_qimao_rank(limit)
-    fq = fetch_fanqie_rank(limit)
-    merged, seen = [], set(qm)
-    for t in fq:
-        if t not in seen:
-            seen.add(t)
-            merged.append(t)
-    return qm + merged
+    全部失败才返回 []——单源挂了其余源仍可用，扫榜永不因单点挡任务。"""
+    merged, seen = [], set()
+    for _label, fname in _SOURCES:
+        for t in globals()[fname](limit):
+            if t not in seen:
+                seen.add(t)
+                merged.append(t)
+    return merged
 
 
 def rank_scan_prompt(goal):
-    """组装扫榜选材分析提示词。双源都空返回 None（调用方回落直连提示词）。"""
-    qm = fetch_qimao_rank(30)
-    fq = fetch_fanqie_rank(30)
-    qm = [t for t in qm if t not in fq]   # 双源重合的只留七猫份
-    if not qm and not fq:
+    """组装扫榜选材分析提示词。四源全空返回 None（调用方回落直连提示词）。"""
+    blocks, seen = [], set()
+    for label, fname in _SOURCES:
+        items = [t for t in globals()[fname](30) if t not in seen]
+        seen.update(items)
+        if items:
+            blocks.append("### %s\n%s" % (label, "、".join(items)))
+    if not blocks:
         return None
-    blocks = []
-    if qm:
-        blocks.append("### 七猫排行榜素材\n" + "、".join(qm))
-    if fq:
-        blocks.append("### 番茄排行榜素材\n" + "、".join(fq))
     material = "\n\n".join(blocks)
     return (
-        "你是网文选题分析师。以下是刚刚抓取的两个平台排行榜页面的书目与分类素材：\n\n"
+        "你是网文选题分析师。以下是刚刚抓取的各平台排行榜页面的书目与分类素材：\n\n"
         "## 榜单元素材\n%s\n\n"
         "## 用户想写的方向\n%s\n\n"
         "## 你的产出（Markdown 报告）\n"
-        "1. **热门题材 Top3**：各自的共同特征与上榜代表书目（跨平台重合的题材单独点出——双平台都热说明是真风口）\n"
+        "1. **热门题材 Top3**：各自的共同特征与上榜代表书目（跨平台重合的题材单独点出——多平台都热说明是真风口）\n"
         "2. **高频人设/套路总结**：3-5 条，点名反复出现的元素\n"
         "3. **差异化切入建议**：2-3 个，结合用户方向给出「题材+人设」组合与一句话理由\n"
         "直接输出分析报告，不要复述素材清单，不要输出与报告无关的内容。"