@cloud411716/fancy-webnovel 0.3.10 → 0.3.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloud411716/fancy-webnovel",
-  "version": "0.3.10",
+  "version": "0.3.12",
   "type": "module",
   "main": "index.js",
   "exports": {

package/plugins/fancy-scan/scripts/scrapers/fanqie-rank-scraper.cjs

@@ -7,13 +7,15 @@
  * 书名/作者/简介/题材/标签（番茄列表页有字体反爬，详情页 HTML 里是明文）。
  * 输出 Markdown 格式匹配 scan-output-format.md 规范。
  *
+ * 重试机制：每个品类/每个榜单 最多重试3次，每次间隔5秒。
+ *
  * 用法：
- *   node fanqie-rank-scraper.js --channel 1 --type 2              # 男频阅读榜
- *   node fanqie-rank-scraper.js --channel 0 --type 1              # 女频新书榜
- *   node fanqie-rank-scraper.js --channel all                     # 全部采集
- *   node fanqie-rank-scraper.js --channel 1 --top 15              # 每题材只取前 15 本
- *   node fanqie-rank-scraper.js --login-wait 60                   # 打开浏览器后等待用户登录（秒）
- *   node fanqie-rank-scraper.js --outdir ./                        # 指定输出目录
+ *   node fanqie-rank-scraper.cjs --channel 1 --type 2              # 男频阅读榜
+ *   node fanqie-rank-scraper.cjs --channel 0 --type 1            # 女频新书榜
+ *   node fanqie-rank-scraper.cjs --channel all                    # 全部采集
+ *   node fanqie-rank-scraper.cjs --channel 1 --top 15             # 每题材只取前 15 本
+ *   node fanqie-rank-scraper.cjs --login-wait 60                  # 打开浏览器后等待用户登录（秒）
+ *   node fanqie-rank-scraper.cjs --outdir ./                      # 指定输出目录
  */
 
 const fs = require("fs");
@@ -23,46 +25,78 @@ const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
 
 // 一次详情请求的并发批大小
 const DETAIL_CHUNK = 5;
-
-// ---------------------------------------------------------------------------
-// 工具函数
-// ---------------------------------------------------------------------------
+// 重试参数
+const MAX_RETRIES = 3;
+const RETRY_DELAY_MS = 5000;
 
 function sleep(ms) {
-  // 同步等待：Atomics.wait 在主线程阻塞，不占用事件循环
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+/**
+ * 重试包装器
+ * @param {Function} fn 异步任务函数，返回结果或抛异常
+ * @param {number} retries 最大重试次数（含首次）
+ * @param {number} delayMs 重试间隔（毫秒）
+ * @param {string} label 日志标签
+ * @returns {Promise<{_failed:boolean, err:Error, label:string}|*>} 失败时返回特殊标记对象
+ */
+async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
+  let lastErr;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < retries) {
+        console.log(`  ⏳ ${label} 第${attempt}次失败，${delayMs / 1000}s后重试...`);
+        sleep(delayMs);
+      }
+    }
+  }
+  // 全部重试失败
+  return { _failed: true, err: lastErr, label };
+}
+
 // ---------------------------------------------------------------------------
 // 页面提取
 // ---------------------------------------------------------------------------
 
 /** 连通性 + 页面就绪自检 */
 async function probePage(page) {
-  return page.evaluate(() => ({
-    host: location.host,
-    hasState: !!window.__INITIAL_STATE__,
-  }));
+  return page.evaluate(() => {
+    const s = window.__INITIAL_STATE__ || {};
+    const host = window.location.hostname;
+    const hasState = !!(s.rank || s.rankData || s.page);
+    return { host, hasState };
+  });
 }
 
-/** 提取侧边菜单品类链接 */
+/** 从 DOM 提取品类菜单 */
 async function extractCategories(page, channel, type) {
-  const prefix = `/rank/${channel}_${type}_`;
-  const cats = await page.evaluate((p) => {
-    const out = [];
-    const seen = {};
-    document.querySelectorAll("a").forEach((a) => {
-      const href = a.getAttribute("href") || "";
-      if (href.indexOf(p) === -1) return;
-      const name = (a.innerText || a.textContent || "").trim();
-      if (!name) return;
-      if (seen[href]) return;
-      seen[href] = 1;
-      out.push({ name, href });
-    });
-    return out;
-  }, prefix);
-  return cats;
+  return page.evaluate(({ channel, type }) => {
+    // 尝试从 __INITIAL_STATE__ 取品类列表
+    const s = window.__INITIAL_STATE__ || {};
+    let cats = null;
+    const cands = [
+      s.rank && s.rank.categories,
+      s.rank && s.rank.categoryList,
+      s.rankData && s.rankData.categories,
+      s.page && s.page.categories,
+    ];
+    for (const c of cands) {
+      if (Array.isArray(c) && c.length) { cats = c; break; }
+    }
+    if (!cats) {
+      // 降级：从 DOM 提取
+      const links = Array.from(document.querySelectorAll('a[href*="/rank/' + channel + '_' + type + '_"]'));
+      const seen = new Set();
+      cats = links
+        .map(a => ({ name: a.innerText.trim() || a.textContent.trim(), href: a.getAttribute('href') }))
+        .filter(c => c.name && c.href && !seen.has(c.href) && seen.add(c.href));
+    }
+    return cats.slice(0, 20);
+  }, { channel, type });
 }
 
 /** 从 __INITIAL_STATE__ 提取当前品类页的作品列表 */
@@ -81,8 +115,8 @@ async function extractBookList(page) {
       { timeout: 15000 }
     );
   } catch (waitErr) {
-    console.error(`  ✗ 等待页面数据超时（15s），可能是网络慢或页面结构变化：${waitErr.message}`);
-    return []; // 返回空数组而非抛异常，避免中断整个采集
+    // 超时返回空数组（由调用方判断是否重试）
+    return [];
   }
   return page.evaluate(() => {
     const s = window.__INITIAL_STATE__ || {};
@@ -115,9 +149,7 @@ async function extractBookList(page) {
         if (typeof o === "object") {
           for (var k in o) {
             if (found) break;
-            try {
-              walk(o[k], d + 1);
-            } catch (e) {}
+            try { walk(o[k], d + 1); } catch (e) {}
           }
         }
       })(s, 0);
@@ -135,126 +167,86 @@ async function extractBookList(page) {
             : b.status,
       lastChapterTitle: b.lastChapterTitle || b.last_chapter_title || b.lastChapter || "",
       category: b.category || b.categoryName || b.category_name || "",
-    })).filter((b) => b.bookId);
+    }));
   });
 }
 
-/** 批量解码详情：逐本同步 XHR 请求 /page/{id} */
+/** 分批解码，避免单次 eval 超时 */
 async function fetchDetailsChunk(page, ids) {
   return page.evaluate((ids) => {
-    var map = {};
-    function pick(h, res) {
-      for (var i = 0; i < res.length; i++) {
-        var m = h.match(res[i]);
-        if (m && m[1]) return m[1].trim();
-      }
-      return "";
-    }
-    for (var k = 0; k < ids.length; k++) {
-      var id = ids[k];
+    const base = "https://fanqienovel.com";
+    const map = {};
+    ids.forEach((id) => {
       try {
-        var x = new XMLHttpRequest();
-        x.open("GET", "/page/" + id, false);
-        x.send();
-        var h = x.responseText || "";
-        var title = pick(h, [
-          /"bookName"\s*:\s*"([^"]+)"/,
-          /<title>([^<]*?)(?:完整版|最新章节|在线阅读|_番茄小说|-番茄小说|_番茄|-番茄)/,
-          /<meta[^>]+property="og:title"[^>]+content="([^"]+)"/,
-          /<title>([^<|_]{1,40})/,
-        ]);
-        var author = pick(h, [
-          /"author"\s*:\s*"([^"]+)"/,
-          /"authorName"\s*:\s*"([^"]+)"/,
-          /<meta[^>]+property="og:novel:author"[^>]+content="([^"]+)"/,
-        ]);
-        var abs = pick(h, [/"abstract"\s*:\s*"([^"]{6,}?)"/]);
-        var desc =
-          abs ||
-          pick(h, [
-            /<meta[^>]+name="description"[^>]+content="([^"]+)"/,
-            /<meta[^>]+property="og:description"[^>]+content="([^"]+)"/,
-          ]);
-        var category = pick(h, [
-          /"categoryV2":"\\[\{[\\s\\S]*?\\"Name\\":\\"([^"\\]+)/,
-          /"category"\s*:\s*"([^"]{1,20})"/,
-          /<meta[^>]+property="og:novel:category"[^>]+content="([^"]+)"/,
-        ]);
-        var tags = "";
-        var bm = (abs || desc || "").match(/【\[]([^】\]]{2,40})[】\]]/);
-        if (bm) {
-          tags = bm[1]
-            .split(/[+、,\/\s]+/)
-            .filter(Boolean)
-            .slice(0, 6)
-            .join("、");
-        }
+        const el = document.querySelector(`[data-bookid="${id}"]`) ||
+                   document.querySelector(`[href*="/page/${id}"]`) ||
+                   document.querySelector(`[href*="/book/${id}"]`);
+        if (!el) { map[id] = {}; return; }
+        const card = el.closest(".rank-book-item") || el.closest(".book-card") || el;
+        const titleEl = card.querySelector(".book-title") || card.querySelector(".book-name") || card.querySelector("a");
+        const authorEl = card.querySelector(".author-name") || card.querySelector(".author") || card.querySelector(".writer");
+        const descEl = card.querySelector(".book-desc") || card.querySelector(".description");
+        const categoryEl = card.querySelector(".category") || card.querySelector(".tag");
+        const title = titleEl ? (titleEl.innerText || titleEl.textContent || "").trim() : "";
+        const author = authorEl ? (authorEl.innerText || authorEl.textContent || "").replace(/^作者：/, "").trim() : "";
+        const desc = descEl ? (descEl.innerText || descEl.textContent || "").trim() : "";
+        const category = categoryEl ? (categoryEl.innerText || categoryEl.textContent || "").trim() : "";
+        // 尝试从 href 提取标签
+        const tags = (desc.match(/#[^\s，,。]+/g) || []).slice(0, 6).join("、");
         map[id] = { title, author, desc, category, tags };
       } catch (e) {
         map[id] = { title: "", author: "", desc: "", category: "", tags: "", err: String(e && e.message || e) };
       }
-    }
+    });
     return map;
   }, ids);
 }
 
-/** 分批解码，避免单次 eval 超时 */
+/** 分批并行请求详情 */
 async function fetchDetails(page, bookIds) {
   const map = {};
   for (let i = 0; i < bookIds.length; i += DETAIL_CHUNK) {
     const chunk = bookIds.slice(i, i + DETAIL_CHUNK);
     const part = await fetchDetailsChunk(page, chunk);
     Object.assign(map, part);
-    sleep(300);
   }
   return map;
 }
 
-/** 滚动页面加载更多内容 */
 async function scrollLoad(page, times, interval = 1000) {
   for (let i = 0; i < times; i++) {
-    await page.evaluate(() => window.scrollBy(0, window.innerHeight));
-    sleep(interval);
+    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+    await page.waitForTimeout(interval);
   }
 }
 
-// ---------------------------------------------------------------------------
-// 格式化
-// ---------------------------------------------------------------------------
-
 function fmtReads(count) {
-  if (!count || count === "0") return "未知";
-  const n = parseInt(count, 10);
-  if (isNaN(n)) return "未知";
+  if (!count && count !== 0) return "";
+  const n = Number(count);
+  if (isNaN(n)) return String(count);
+  if (n >= 100000000) return (n / 100000000).toFixed(1) + "亿";
   if (n >= 10000) return (n / 10000).toFixed(1) + "万";
   return String(n);
 }
 
 function fmtWords(count) {
-  if (!count) return "未知";
-  const n = parseInt(count, 10);
-  if (isNaN(n)) return "未知";
-  if (n >= 10000) return (n / 10000).toFixed(1) + "万";
-  return String(n);
+  if (!count && count !== 0) return "";
+  const n = Number(count);
+  if (isNaN(n)) return String(count);
+  if (n >= 100000000) return (n / 100000000).toFixed(1) + "亿字";
+  if (n >= 10000) return (n / 10000).toFixed(0) + "万字";
+  return String(n) + "字";
 }
 
 function fmtStatus(s) {
-  const v = String(s);
-  if (v === "1") return "连载中";
-  if (v === "0" || v === "2") return "已完结";
-  return s ? String(s) : "未知";
+  if (s === 1 || s === "1" || s === "连载") return "连载";
+  if (s === 2 || s === "2" || s === "完结") return "完结";
+  return "未知";
 }
 
 function cleanDesc(raw) {
   if (!raw) return "";
-  let d = String(raw)
-    .replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16)))
-    .replace(/\\[nrt]/g, " ")
-    .replace(/\\"/g, '"')
-    .replace(/番茄小说[^。！？]*?(?:免费阅读|完整版|在线阅读)[^。！？]*[。！？]/g, "")
-    .replace(/番茄小说[^。！？]*?(?:免费阅读|完整版|在线阅读)[^。！？]*$/g, "")
-    .replace(/\s+/g, " ")
-    .trim();
+  let d = raw.replace(/\s+/g, " ").trim();
   if (d.length <= 100) return d;
   const cut = d.slice(0, 100);
   const m = cut.match(/[\s\S]*[。！？]/);
@@ -274,11 +266,25 @@ const TYPE = getArg(args, "--type") || "2";
 const TOP = parseInt(getArg(args, "--top") || "20", 10);
 
 function channelLabel(ch) {
-  return ch === "1" ? "男频" : "女频";
+  return ch === "1" ? "男频" : ch === "0" ? "女频" : ch;
 }
 
 function typeLabel(t) {
-  return t === "2" ? "阅读榜" : "新书榜";
+  return t === "2" ? "阅读榜" : t === "1" ? "新书榜" : t;
+}
+
+async function scrapeCategoryWithRetry(page, cat) {
+  return withRetry(async () => {
+    await page.goto(`https://fanqienovel.com${cat.href}`, { waitUntil: "networkidle" });
+    await page.waitForTimeout(2500);
+    await scrollLoad(page, 2);
+
+    const books = await extractBookList(page);
+    if (!books || !Array.isArray(books)) {
+      throw new Error("extractBookList 返回异常");
+    }
+    return books;
+  }, MAX_RETRIES, RETRY_DELAY_MS, `品类[${cat.name}]`);
 }
 
 async function scrapeChannel(ch, type) {
@@ -298,15 +304,25 @@ async function scrapeChannel(ch, type) {
     const cdp = await context.newCDPSession(page);
     const { windowId } = await cdp.send('Browser.getWindowForTarget', { target: page.target() });
     await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
-  } catch (_) {}
+  } catch (_) { }
 
   let scraped = null;
 
   try {
-    // 打开入口页
-    await page.goto(initUrl, { waitUntil: "networkidle" }).catch(gotoErr => {
-      console.error(`  ✗ 打开入口页失败：${gotoErr.message}`);
-    });
+    // 打开入口页（带重试）
+    const homeResult = await withRetry(async () => {
+      await page.goto(initUrl, { waitUntil: "networkidle" });
+      const probe = await probePage(page);
+      if (!probe.host || probe.host.indexOf("fanqie") === -1) {
+        throw new Error(`非番茄页面（host=${probe.host}），可能被重定向到登录/验证页`);
+      }
+      return probe;
+    }, MAX_RETRIES, RETRY_DELAY_MS, `打开入口页`);
+
+    if (homeResult._failed) {
+      console.log(`  ✗ 入口页采集连续失败：${homeResult.err.message}`);
+      return null;
+    }
 
     // 等待登录（用户可在此时手动登录）
     if (LOGIN_WAIT > 0) {
@@ -314,24 +330,11 @@ async function scrapeChannel(ch, type) {
       await page.waitForTimeout(LOGIN_WAIT * 1000);
     }
 
-    // 连通性自检
-    const probe = await probePage(page);
-    if (!probe.host || probe.host.indexOf("fanqie") === -1) {
-      console.error(
-        `  ✗ 当前页面非番茄（host=${probe.host}），可能被重定向到登录/验证页，已跳过。`
-      );
-      return null;
-    }
-    if (!probe.hasState) {
-      console.log(`  ⚠ 页面未挂载 __INITIAL_STATE__，将尝试兜底扫描，结果可能不完整。`);
-    }
-
     // ─── 调试模式：拦截所有请求，找出真实数据接口 ───
     if (PROBE) {
       const intercepted = [];
       const seen = new Set();
 
-      // 先注册拦截器，再导航到目标页（避免漏掉 initial 请求）
       context.on("request", req => {
         const url = req.url();
         if (seen.has(url)) return;
@@ -361,7 +364,6 @@ async function scrapeChannel(ch, type) {
         } catch {}
       });
 
-      // 直接导航到品类页（不用 waitUntil，避免卡住）
       const cats = await extractCategories(page, ch, type);
       if (!cats.length) {
         await page.goto(`https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`);
@@ -415,59 +417,61 @@ async function scrapeChannel(ch, type) {
     ];
 
     let totalBooks = 0;
-    let resolvedTitles = 0;
-    const bodyLines = [];
+    const failedCategories = []; // { name, err }
 
     for (let ci = 0; ci < categories.length; ci++) {
       const cat = categories[ci];
       console.log(`  [${ci + 1}/${categories.length}] ${cat.name}`);
 
-      try {
-        await page.goto(`https://fanqienovel.com${cat.href}`, { waitUntil: "networkidle" }).catch(gotoErr => {
-          console.error(`  ✗ 打开品类页失败：${gotoErr.message}`);
-        });
-        await page.waitForTimeout(2500);
-        await scrollLoad(page, 2);
-
-        let books = await extractBookList(page);
-        if (!Array.isArray(books) || !books.length) {
-          bodyLines.push(`## ${cat.name} — 0 本`, "", "---", "");
-          continue;
-        }
-        if (books.length > TOP) books = books.slice(0, TOP);
+      const booksResult = await scrapeCategoryWithRetry(page, cat);
 
-        const bookIds = books.map((b) => String(b.bookId));
-        const details = await fetchDetails(page, bookIds);
+      if (booksResult._failed) {
+        console.log(`  ✗ 品类[${cat.name}]连续${MAX_RETRIES}次失败：${booksResult.err.message}`);
+        failedCategories.push({ name: cat.name, err: booksResult.err.message });
+        lines.push(`## ${cat.name} — 采集失败（已重试${MAX_RETRIES}次）`, "", "---", "");
+        continue;
+      }
 
-        bodyLines.push(`## ${cat.name} — ${books.length} 本`, "");
+      let books = booksResult;
+      if (!books.length) {
+        lines.push(`## ${cat.name} — 0 本`, "", "---", "");
+        continue;
+      }
+      if (books.length > TOP) books = books.slice(0, TOP);
+
+      const bookIds = books.map((b) => String(b.bookId));
+      const details = await fetchDetails(page, bookIds);
+
+      lines.push(`## ${cat.name} — ${books.length} 本`, "");
+
+      for (let i = 0; i < books.length; i++) {
+        const b = books[i];
+        const info = details[String(b.bookId)] || {};
+        totalBooks++;
+        const title = info.title || "（标题待解析）";
+        const author = info.author || "未知";
+        const category = info.category || b.category || "";
+
+        lines.push(`书名：${title}`);
+        lines.push(`题材：${category}`);
+        lines.push(`作者：${author}`);
+        lines.push(`作品页：https://fanqienovel.com/page/${b.bookId}`);
+        lines.push("");
+      }
 
-        for (let i = 0; i < books.length; i++) {
-          const b = books[i];
-          const info = details[String(b.bookId)] || {};
-          totalBooks++;
-          const resolved = !!info.title;
+      lines.push("---", "");
+    }
 
-          const title = info.title || "（标题待解析）";
-          const author = info.author || "未知";
-          const category = info.category || b.category || "";
+    scraped = lines.join("\n");
 
-          bodyLines.push(`书名：${title}`);
-          bodyLines.push(`题材：${category}`);
-          bodyLines.push(`作者：${author}`);
-          bodyLines.push(`作品页：https://fanqienovel.com/page/${b.bookId}`);
-          bodyLines.push("");
-        }
-
-        bodyLines.push("---", "");
-      } catch (catErr) {
-        console.error(
-          `  [fanqie] 品类 ${cat.name} 处理出错，跳过: ${catErr && catErr.message ? catErr.message : catErr}`
-        );
-        bodyLines.push(`## ${cat.name} — 采集失败`, "", "---", "");
+    // 输出失败品类汇总
+    if (failedCategories.length > 0) {
+      console.log(`\n⚠️  以下品类采集失败（共 ${failedCategories.length} 项）：`);
+      for (const f of failedCategories) {
+        console.log(`  • ${f.name}：${f.err}`);
       }
     }
 
-    scraped = lines.concat(bodyLines).join("\n");
   } finally {
     await browser.close();
   }
@@ -485,12 +489,16 @@ async function main() {
   const channels = CHANNEL === "all" ? ["1", "0"] : [CHANNEL];
   const types = TYPE === "all" ? ["2", "1"] : [TYPE];
   let written = 0;
+  const failedChannels = []; // { ch, ty, err }
 
   for (const ch of channels) {
     for (const ty of types) {
       try {
         const content = await scrapeChannel(ch, ty);
-        if (!content) continue;
+        if (!content) {
+          failedChannels.push({ ch, ty, err: "采集返回空" });
+          continue;
+        }
 
         const date = localDateStamp();
         const filename = `番茄${channelLabel(ch)}${typeLabel(ty)}_全题材_${date}.md`;
@@ -500,12 +508,20 @@ async function main() {
         written++;
         console.log(`  ✓ 已保存: ${filepath}`);
       } catch (chErr) {
-        console.error(
-          `[fanqie] ${channelLabel(ch)}${typeLabel(ty)} 采集失败，跳过: ${chErr && chErr.message ? chErr.message : chErr}`
-        );
+        console.log(`  ✗ 频道${channelLabel(ch)}${typeLabel(ty)}采集失败：${chErr.message}`);
+        failedChannels.push({ ch, ty, err: chErr.message });
       }
     }
   }
+
+  // 输出失败榜单汇总
+  if (failedChannels.length > 0) {
+    console.log(`\n⚠️  以下榜单采集失败（共 ${failedChannels.length} 项）：`);
+    for (const f of failedChannels) {
+      console.log(`  • ${channelLabel(f.ch)}${typeLabel(f.ty)}：${f.err}`);
+    }
+  }
+
   return written;
 }