codebee 0.1.9 → 0.1.11

package/app/core/publish/flow.py

@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+"""数据驱动的发布流程解释器：按步骤表操作一个 CDP 页面。
+
+平台流程（建书/发章）描述为步骤数组，与代码分离：
+- 内置默认表在各平台模块（fanqie/qimao），是「待校准」的推测选择器；
+- data/publish/flows-<platform>.json 存在时整体覆盖内置表——平台改版/
+  首次校准只改数据文件，不用动代码；
+- 每步失败先落一张 fail-*.png 再抛人话错误，发布中断可回看卡在哪一步。
+
+步骤类型：
+  navigate  {url}                 打开地址（支持 {home} 等占位符，来自平台配置）
+  wait      {sel, timeout}        等元素出现
+  fill      {sel, key}            把 values[key] 填进输入框（React 安全 + 富文本）
+  click     {sel}                 点选择器命中的元素
+  click_text {text, scope, contains}  按「可见文本」点按钮/标签（无稳定 id 的弹层项）
+  shot      {name}                截图存证
+  probe     {note}                dump 表单元素清单（校准选择器用，结果进 log）
+  submit    {sel}                 终步：auto_submit=false 时跳过（留给人工确认），
+                                  跳过时补一张 ready-*.png，用户在浏览器窗口里自查提交
+  url_any   {any: [..]}           断言当前 URL 含任一标记，不含则报错（登录跳转检测）
+"""
+from __future__ import annotations
+
+import json
+import time
+
+from .browser import BrowserError
+
+
+class FlowError(Exception):
+    """流程失败：message 面向用户（含步骤序号与截图路径线索）。"""
+
+
+def _click_match_js():
+    # 点同时包含所有关键词的最小可见元素（长度最小者=最内层卡片）
+    return ("(keys,maxLen)=>{"
+            "const hit=[...document.querySelectorAll('div,li,section,label,span,a')].filter(e=>{"
+            "const x=(e.innerText||'').trim();"
+            "return x && x.length<=maxLen && keys.every(k=>x.includes(k));});"
+            "if(!hit.length)return{ok:false,err:'找不到同时含 '+keys.join('+')+' 的元素'};"
+            "hit.sort((a,b)=>(a.innerText||'').length-(b.innerText||'').length);"
+            "hit[0].scrollIntoView({block:'center'});hit[0].click();"
+            "return{ok:true,tag:hit[0].tagName};}")
+
+
+def _fill_label_js():
+    # Element UI 表单：在 .el-form-item（或 class 含 form-item 的容器）里按
+    # label 文本定位控件，走与 browser.fill 相同的 native setter 管道
+    return ("(labelText,text)=>{"
+            "const items=[...document.querySelectorAll('.el-form-item,[class*=form-item]')];"
+            "let target=null;"
+            "for(const it of items){"
+            "const lb=it.querySelector('[class*=label],[class*=label]');"
+            "const ltxt=((lb&&lb.innerText)||'').trim();"
+            "if(ltxt&&ltxt.includes(labelText)){target=it;break;}}"
+            "if(!target)return{ok:false,err:'找不到字段「'+labelText+'」'};"
+            "const el=target.querySelector('textarea,[contenteditable=true],input[type=text]')||"
+            "target.querySelector('input,textarea');"
+            "if(!el)return{ok:false,err:'字段「'+labelText+'」下没有输入控件'};"
+            "el.scrollIntoView({block:'center'});el.focus();"
+            "if(el.isContentEditable){"
+            "const r=document.createRange();r.selectNodeContents(el);"
+            "const g=getSelection();g.removeAllRanges();g.addRange(r);"
+            "document.execCommand('insertText',false,text);"
+            "return{ok:true};}"
+            "const proto=el.tagName==='TEXTAREA'?HTMLTextAreaElement.prototype:HTMLInputElement.prototype;"
+            "const d=Object.getOwnPropertyDescriptor(proto,'value');"
+            "(d&&d.set?d.set:function(v){el.value=v}).call(el,text);"
+            "el.dispatchEvent(new Event('input',{bubbles:true}));"
+            "el.dispatchEvent(new Event('change',{bubbles:true}));"
+            "return{ok:true};}")
+
+
+def _click_text_js():
+    # contains 模式下外层容器（整页文本）也会 includes 命中——候选按文本长度
+    # 升序取最短者=最内层最精确元素（「新建小说」按钮赢过包它的大 DIV）
+    return ("(t,scope,contains)=>{"
+            "const els=[...document.querySelectorAll(scope||'button,a,[role=button],span,li')];"
+            "const cands=els.filter(e=>{const x=(e.innerText||'').trim();"
+            "return x&&(contains?x.includes(t):x===t);});"
+            "if(!cands.length)return{ok:false,err:'页面上找不到文本为「'+t+'」的可点元素'};"
+            "cands.sort((a,b)=>((a.innerText||'').trim().length)-((b.innerText||'').trim().length));"
+            "const hit=cands[0];"
+            "hit.scrollIntoView({block:'center'});hit.click();return{ok:true};}")
+
+
+def run_flow(page, steps, values=None, config=None, auto_submit=False,
+             shot=None, log=None):
+    """跑一个流程。values：fill 取值字典；config：平台 URL 等占位符来源。
+
+    shot(name) → 截图落盘函数（manager 注入，路径含任务/平台维度）；
+    log(line)  → 步骤日志函数（进发布记录的 log 字段）。返回执行到的步数。"""
+    values = values or {}
+    config = config or {}
+    log = log or (lambda s: None)
+
+    def note(i, s):
+        log("步骤%d %s" % (i, s))
+
+    for i, st in enumerate(steps):
+        act = (st.get("do") or "").strip()
+        try:
+            if act == "navigate":
+                url = st["url"]
+                for k, v in config.items():
+                    url = url.replace("{%s}" % k, str(v))
+                note(i, "打开 %s" % url)
+                page.navigate(url)
+            elif act == "wait":
+                note(i, "等待 %s" % st["sel"])
+                page.wait_for(st["sel"], timeout=float(st.get("timeout") or 12))
+            elif act == "fill":
+                key = st.get("key") or ""
+                text = values.get(key, "")
+                if text in ("", None):
+                    continue                    # 空值字段跳过（如无第二主角）
+                note(i, "填入 %s（%d 字）" % (key, len(str(text))))
+                page.wait_for(st["sel"], timeout=8)
+                page.fill(st["sel"], str(text))
+            elif act == "click":
+                note(i, "点击 %s" % st["sel"])
+                page.wait_for(st["sel"], timeout=8)
+                page.click(st["sel"])
+            elif act == "click_text":
+                text = str(st.get("text") or "")
+                for k, v in (values or {}).items():     # {signing_mode} 等动态值
+                    text = text.replace("{%s}" % k, str(v))
+                if not text:
+                    continue
+                note(i, "点击「%s」" % text)
+                r = None
+                for _try in range(3):               # SPA 渲染慢：找不到先等再试
+                    r = page.call(_click_text_js(), text, st.get("scope") or "",
+                                  bool(st.get("contains")))
+                    if (r or {}).get("ok"):
+                        break
+                    time.sleep(0.9)
+                if not (r or {}).get("ok"):
+                    raise FlowError((r or {}).get("err") or text)
+            elif act == "click_match":
+                # 关键词选块（站点卡片等）：点同时包含所有关键词的最小元素
+                keys = [str(x) for x in (st.get("any") or []) if str(x).strip()]
+                note(i, "点击含 %s 的卡片" % keys)
+                r = None
+                for _try in range(4):               # 弹层渲染慢：找不到先等再试
+                    r = page.call(_click_match_js(), keys,
+                                  int(st.get("max_len") or 400))
+                    if (r or {}).get("ok"):
+                        break
+                    time.sleep(1.0)
+                if not (r or {}).get("ok"):
+                    raise FlowError((r or {}).get("err") or "卡片未找到")
+            elif act == "fill_label":
+                # 按字段标签填（Element UI form-item：label 与控件同容器）
+                key = st.get("key") or ""
+                label = str(st.get("label") or "")
+                text = str(values.get(key, "") or "")
+                if not text:
+                    continue
+                note(i, "填字段「%s」（%d 字）" % (label, len(text)))
+                page.wait_for("[class*=form]", timeout=8)
+                r = None
+                for _try in range(3):
+                    r = page.call(_fill_label_js(), label, text)
+                    if (r or {}).get("ok"):
+                        break
+                    time.sleep(0.9)
+                if not (r or {}).get("ok"):
+                    raise FlowError((r or {}).get("err") or "填字段失败")
+            elif act == "radio":
+                # 单选组：values[map[key]] 映射到 radio value 后点它。
+                # 隐藏 input（Element UI）点不到——点它的最近可见 label 祖先。
+                key = st.get("key") or ""
+                group = st.get("map") or {}
+                want = str(values.get(key, "")).strip()
+                val = group.get(want)
+                if val is None:
+                    if st.get("optional"):
+                        note(i, "跳过单选 %s（无值）" % key)
+                        continue
+                    raise FlowError("单选 %s：值「%s」不在映射 %s 里" % (key, want, list(group)))
+                note(i, "单选 %s=%s（value=%s）" % (key, want, val))
+                r = page.call(
+                    "(v)=>{const r=[...document.querySelectorAll('input[type=radio]')]"
+                    ".find(x=>x.value===v);if(!r)return{ok:false,err:'radio v='+v};"
+                    "const lab=r.closest('label')||(r.closest('.el-radio')||{}).firstElementChild||r;"
+                    "const t=lab.matches('label,.el-radio')?lab:(r.parentElement||r);"
+                    "t.scrollIntoView({block:'center'});t.click();return{ok:true};}", str(val))
+                if not (r or {}).get("ok"):
+                    raise FlowError((r or {}).get("err") or "单选点击失败")
+            elif act == "tags":
+                # 标签弹层逐个点选：manager 把标签清单放 values["_tags"]。
+                # 项为 [组名, 标签] 时先点左侧组名切换（组标签懒渲染）再点标签；
+                # 纯字符串直接点。弹层打开由前置 click_text「添加标签」负责。
+                tags = values.get("_tags") or []
+                if not tags:
+                    note(i, "无标签可点，跳过")
+                    continue
+                scope = st.get("scope") or "span,li,label,[class*=dialog] *,[class*=popper] *"
+                note(i, "点选标签 %d 项" % len(tags))
+                for item in tags:
+                    grp, tg = ("", str(item)) if isinstance(item, (str, int)) \
+                        else (str(item[0] or ""), str(item[1] or ""))
+                    if not tg:
+                        continue
+                    if grp:                             # 切到目标组
+                        r0 = None
+                        for _try in range(2):
+                            r0 = page.call(_click_text_js(), grp, scope, True)
+                            if (r0 or {}).get("ok"):
+                                break
+                            time.sleep(0.6)
+                        if not (r0 or {}).get("ok"):
+                            raise FlowError("标签组「%s」切换失败" % grp)
+                        time.sleep(0.3)
+                    r = None
+                    for _try in range(3):
+                        r = page.call(_click_text_js(), tg, scope, True)
+                        if (r or {}).get("ok"):
+                            break
+                        time.sleep(0.7)
+                    if not (r or {}).get("ok"):
+                        raise FlowError("标签「%s」点选失败：%s" % (tg, (r or {}).get("err")))
+            elif act == "shot":
+                note(i, "截图 %s" % st.get("name"))
+                if shot:
+                    shot(st.get("name") or "step")
+            elif act == "probe":
+                info = page.probe()
+                note(i, "表单探测 %s：%d 个可交互元素" % (st.get("note") or "", len(info)))
+                if log:
+                    log("PROBE " + json.dumps(info, ensure_ascii=False)[:4000])
+            elif act == "submit":
+                if not auto_submit:
+                    note(i, "已填好未提交——请人工检查后提交（auto_submit=false）")
+                    if shot:
+                        shot("ready-manual-submit")
+                    return i + 1
+                if st.get("text"):                    # 按按钮文本提交
+                    text = str(st["text"])
+                    note(i, "提交「%s」（真实鼠标事件）" % text)
+                    r = None
+                    for _try in range(3):
+                        r = page.real_click_text(
+                            text, st.get("scope") or "button,a,[class*=btn]",
+                            contains=True)
+                        if (r or {}).get("ok"):
+                            break
+                        time.sleep(0.9)
+                    if not (r or {}).get("ok"):
+                        raise FlowError((r or {}).get("err") or "提交按钮未找到")
+                else:
+                    note(i, "提交 %s" % st.get("sel") or "")
+                    page.wait_for(st["sel"], timeout=8)
+                    page.click(st["sel"])
+            elif act == "url_any":
+                u = str(page.url() or "")
+                marks = st.get("any") or []
+                if not any(m in u for m in marks):
+                    raise FlowError("当前页面 %s 不含预期标记 %s（可能未登录或改版）"
+                                    % (u[:90], marks))
+                note(i, "页面标记校验通过")
+            else:
+                raise FlowError("未知步骤类型：%s" % act)
+        except FlowError:
+            _fail_shot(shot, "step%d-%s" % (i, act))
+            raise
+        except BrowserError as e:
+            _fail_shot(shot, "step%d-%s" % (i, act))
+            raise FlowError("步骤%d（%s）失败：%s" % (i, act, e))
+        except KeyError as e:
+            raise FlowError("步骤%d 缺少字段 %s" % (i, e))
+    return len(steps)
+
+
+def _fail_shot(shot, name):
+    try:
+        if shot:
+            shot(name)
+    except Exception:
+        pass

package/app/core/publish/ledger.py

@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+"""发布台账：每次发布动作（连接/建书/传章节）追加一条 JSONL，append-only。
+
+落盘 data/publish/publish-YYYYMM.jsonl（按月分文件，与用量台账同范式）。
+幂等键 (task_id, platform, chapter_no)：章节发布成功后重发即跳过——
+断点续发（发布到一半挂了，重启后接着发没发完的）靠这一条。
+
+作品登记 books.json：task ↔ 平台作品的绑定（建书成功后写入 book_id/标题），
+原子写（tmp + os.replace）。与台账分工：台账只追加不改（审计/历史），
+可变状态（当前绑定的作品）进 books.json。
+
+截图存证：data/publish/shots/<platform>/<task_id>/<时间戳>-<步骤名>.png，
+每步动作后落一张，出错可回看卡在哪一步；旧截图按任务清理不自动做（量小）。
+"""
+from __future__ import annotations
+
+import json
+import re
+import threading
+import time
+
+from .. import paths
+
+LOCK = threading.RLock()
+
+FIELDS = ("ts", "day", "platform", "action", "task_id", "chapter_no",
+          "book_id", "title", "ok", "error", "shot")
+
+
+def _month_file(day):
+    return paths.PUBLISH_DIR / ("publish-%s.jsonl" % day[:7].replace("-", ""))
+
+
+def record(platform, action, task_id="", chapter_no=0, book_id="",
+           title="", ok=True, error="", shot=""):
+    """追加一条发布记录。异常全吞——记账失败绝不能影响发布主流程。"""
+    try:
+        rec = {
+            "ts": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "day": time.strftime("%Y-%m-%d"),
+            "platform": str(platform)[:16],
+            "action": str(action)[:24],
+            "task_id": str(task_id)[:64],
+            "chapter_no": int(chapter_no or 0),
+            "book_id": str(book_id)[:80],
+            "title": str(title)[:120],
+            "ok": bool(ok),
+            "error": str(error or "")[:300],
+            "shot": str(shot)[:200],
+        }
+        with LOCK:
+            paths.PUBLISH_DIR.mkdir(parents=True, exist_ok=True)
+            with open(_month_file(rec["day"]), "a", encoding="utf-8") as f:
+                f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+    except Exception:
+        pass
+
+
+def _iter_records(days=90):
+    """近 N 天的记录（月份文件粒度粗滤，行内再按 day 过滤）。"""
+    import datetime
+    cutoff = time.strftime("%Y-%m-%d", time.localtime(time.time() - days * 86400))
+    for fp in sorted(paths.PUBLISH_DIR.glob("publish-*.jsonl")):
+        try:
+            lines = fp.read_text(encoding="utf-8").splitlines()
+        except OSError:
+            continue
+        for ln in lines:
+            try:
+                r = json.loads(ln)
+            except ValueError:
+                continue
+            if str(r.get("day") or "") >= cutoff:
+                yield r
+
+
+def published_chapters(task_id, platform):
+    """该任务在该平台已成功发布的章节号集合（幂等跳过依据）。"""
+    out = set()
+    for r in _iter_records(3650):
+        if (r.get("action") == "upload_chapter" and r.get("ok")
+                and r.get("task_id") == task_id and r.get("platform") == platform):
+            n = r.get("chapter_no") or 0
+            if n > 0:
+                out.add(int(n))
+    return out
+
+
+def recent(task_id=None, platform=None, limit=50):
+    """最近记录（新在前），详情页发布历史用。"""
+    out = []
+    for r in _iter_records(90):
+        if task_id and r.get("task_id") != task_id:
+            continue
+        if platform and r.get("platform") != platform:
+            continue
+        out.append(r)
+    out.reverse()
+    return out[:limit]
+
+
+# ---------------------------------------------------------------- 作品登记
+_BOOKS_FILE = paths.PUBLISH_DIR / "books.json"
+
+
+def load_books():
+    try:
+        d = json.loads(_BOOKS_FILE.read_text(encoding="utf-8"))
+        return d if isinstance(d, dict) else {}
+    except Exception:
+        return {}
+
+
+def save_book(task_id, platform, info):
+    """登记/更新任务在某平台的作品绑定。info: {book_id, title, url?}。"""
+    with LOCK:
+        paths.PUBLISH_DIR.mkdir(parents=True, exist_ok=True)
+        books = load_books()
+        books.setdefault(str(task_id), {})[str(platform)] = {
+            "book_id": str(info.get("book_id") or ""),
+            "title": str(info.get("title") or "")[:120],
+            "url": str(info.get("url") or "")[:300],
+            "created_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+        }
+        tmp = _BOOKS_FILE.with_suffix(".tmp")
+        tmp.write_text(json.dumps(books, ensure_ascii=False, indent=1),
+                       encoding="utf-8")
+        tmp.replace(_BOOKS_FILE)
+
+
+def book_for(task_id, platform):
+    return ((load_books().get(str(task_id)) or {}).get(str(platform))) or None
+
+
+def shot_path(platform, task_id, step):
+    """截图存证路径（只算路径不落盘，由 browser.Page.screenshot 写）。"""
+    ts = time.strftime("%Y%m%d-%H%M%S")
+    return paths.PUBLISH_DIR / "shots" / str(platform) / str(task_id) / \
+        ("%s-%s.png" % (ts, step))
+
+
+_CHAPTER_RE = re.compile(r"第\s*([0-9０-９一二三四五六七八九十百千零两]+)\s*[章节回]")
+
+
+def parse_chapter_no(name):
+    """从章节标题/文件名解析章号（第12章/第十二章），失败返回 0。"""
+    m = _CHAPTER_RE.search(str(name or ""))
+    if not m:
+        return 0
+    s = m.group(1)
+    if s.isdigit():
+        return int(s)
+    cn = {"零": 0, "一": 1, "二": 2, "两": 2, "三": 3, "四": 4, "五": 5,
+          "六": 6, "七": 7, "八": 8, "九": 9}
+    # 正序解析（十二=12、二十三=23、一百零五=105）：数字暂存 num，
+    # 遇单位（十/百/千）把它乘上去并清零；万字内够用（章号没有更大的）
+    section, num = 0, 0
+    for ch in s:
+        if ch in cn:
+            num = cn[ch]
+        elif ch == "十":
+            section += (num or 1) * 10
+            num = 0
+        elif ch == "百":
+            section += (num or 1) * 100
+            num = 0
+        elif ch == "千":
+            section += (num or 1) * 1000
+            num = 0
+    return section + num
+
+
+# ---------------------------------------------------------------- 护栏统计
+def today_count(task_id, platform):
+    """该任务在该平台今日已成功发布的章数（每日上限护栏的计数口径）。"""
+    day = time.strftime("%Y-%m-%d")
+    return sum(1 for r in _iter_records(2)
+               if r.get("action") == "upload_chapter" and r.get("ok")
+               and r.get("task_id") == task_id and r.get("platform") == platform
+               and r.get("day") == day)
+
+
+def consecutive_failures(platform):
+    """该平台最近连续失败的发布动作数（连败退避护栏，跨任务口径）。
+
+    记录时间序旧→新，倒着数到第一条成功为止；连败大概率是风控或改版，
+    此时继续自动重试只会火上浇油，应转人工检查。
+    """
+    n = 0
+    for r in reversed(list(_iter_records(7))):
+        if r.get("platform") != platform:
+            continue
+        if r.get("action") not in ("upload_chapter", "create_book"):
+            continue
+        if r.get("ok"):
+            break
+        n += 1
+    return n