@cloud411716/fancy-webnovel 0.3.14 → 0.3.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloud411716/fancy-webnovel",
-  "version": "0.3.14",
+  "version": "0.3.16",
   "type": "module",
   "main": "index.js",
   "exports": {

package/plugins/fancy-scan/scripts/run-scan.js

@@ -33,6 +33,29 @@ const PLATFORM_CN = {
 // 工具函数
 // ---------------------------------------------------------------------------
 
+const MAX_RETRIES = 3;
+const RETRY_DELAY_MS = 5000;
+
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
+  let lastErr;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < retries) {
+        console.log(`  ⏳ ${label} 第${attempt}次失败，${delayMs / 1000}s后重试...`);
+        sleep(delayMs);
+      }
+    }
+  }
+  return { _failed: true, err: lastErr, label };
+}
+
 function nowIso() {
   return new Date().toISOString();
 }
@@ -64,31 +87,21 @@ async function scrapeQidian(outDir, ranks) {
 
   for (const rt of ranks) {
     const url = `${MOBILE_BASE}${rt.path}`;
-    let html = '';
-    try {
-      html = await fetch(url, { headers: MOBILE_HEADERS, signal: AbortSignal.timeout(15000) }).then(r => r.text());
-    } catch (e) {
-      console.error(`  ⚠ ${rt.label} 请求失败: ${e.message}`);
-      continue;
-    }
 
-    const m = html.match(/<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i);
-    if (!m) {
-      console.error(`  ⚠ ${rt.label} 未找到 pageContext`);
-      continue;
-    }
+    const doScrape = async () => {
+      const html = await fetch(url, { headers: MOBILE_HEADERS, signal: AbortSignal.timeout(15000) }).then(r => r.text());
+      const m = html.match(/<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i);
+      if (!m) throw new Error(`未找到 pageContext`);
+      const pageContext = JSON.parse(m[1]);
+      const records = pageContext?.pageContext?.pageProps?.pageData?.records || [];
+      if (!records.length) throw new Error(`无书籍数据`);
+      return records;
+    };
 
-    let pageContext;
-    try {
-      pageContext = JSON.parse(m[1]);
-    } catch (_) {
-      console.error(`  ⚠ ${rt.label} JSON 解析失败`);
-      continue;
-    }
+    const records = await withRetry(doScrape, MAX_RETRIES, RETRY_DELAY_MS, `起点${rt.label}`);
 
-    const records = pageContext?.pageContext?.pageProps?.pageData?.records || [];
-    if (!records.length) {
-      console.error(`  ⚠ ${rt.label} 无书籍数据`);
+    if (records._failed) {
+      console.error(`  ⚠ 起点${rt.label}连续${MAX_RETRIES}次失败：${records.err.message}`);
       continue;
     }
 

package/plugins/fancy-scan/scripts/scrapers/fanqie-rank-scraper.cjs

@@ -256,7 +256,7 @@ async function scrapeChannel(ch, type) {
   const initUrl   = `https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`;
 
   const browser = await chromium.launch({
-    headless: false,
+    headless: true,
     args: ['--no-sandbox', '--disable-dev-shm-usage', '--start-maximized'],
   });
   const context = await browser.newContext();

package/plugins/fancy-scan/scripts/scrapers/jjwxc-rank-scraper.cjs

@@ -38,6 +38,8 @@ const RANK_TYPES = [
 ];
 
 const DETAIL_CHUNK = 6;
+const MAX_RETRIES = 3;
+const RETRY_DELAY_MS = 5000;
 
 // ---------------------------------------------------------------------------
 // 工具函数
@@ -47,6 +49,26 @@ function sleep(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+/**
+ * 重试包装器
+ * @returns {{_failed:true, err, label}|*} 失败时返回 _failed=true 的标记对象
+ */
+async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
+  let lastErr;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < retries) {
+        console.log(`  ⏳ ${label} 第${attempt}次失败，${delayMs / 1000}s后重试...`);
+        sleep(delayMs);
+      }
+    }
+  }
+  return { _failed: true, err: lastErr, label };
+}
+
 function fmtWan(s, unit) {
   if (s == null || s === "") return "";
   const n = parseInt(String(s).replace(/[^0-9]/g, ""), 10);
@@ -114,7 +136,7 @@ async function scrapeRank(rankTypeId, channelId) {
   console.log(`\n→ 采集 晋江${rt.label}（${chLabel}）...`);
   console.log(`  URL: ${url}`);
 
-  const browser = await chromium.launch({ headless: false, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
+  const browser = await chromium.launch({ headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
   const context = await browser.newContext();
   const page = await context.newPage();
 
@@ -126,19 +148,15 @@ async function scrapeRank(rankTypeId, channelId) {
   } catch (_) {}
 
   let data = null;
-  try {
+  const pageLabel = `${rt.label}（${chLabel}）`;
+  const doScrape = async () => {
     await page.goto(url, { waitUntil: "networkidle" });
     await page.waitForTimeout(3000);
-
-    // 连通性自检
     const host = page.url();
     if (host.indexOf("jjwxc") === -1) {
-      console.error(`  ✗ 当前页面非晋江（url=${host}），可能被重定向，已跳过。`);
-      return null;
+      throw new Error(`非晋江页面（url=${host}），可能被重定向`);
     }
-
-    // 布局检测：先找含"序号"/"作者"/"作品"表头的表格，有则为布局A（表格），无为布局B（交替行）
-    data = await page.evaluate(() => {
+    return await page.evaluate(() => {
       // ── 布局 A：表格型（表头在第一行 <tr> 的 <td> 里，含"序号""作者""作品"） ──
       var tables = document.querySelectorAll('table');
       for (var ti = 0; ti < tables.length; ti++) {
@@ -228,17 +246,21 @@ async function scrapeRank(rankTypeId, channelId) {
       }
       return result;
     });
+  };
 
-    console.log(`  ✓ 布局检测：${data.layout === 'A' ? '表格型' : '交替行型'}`);
+  const dataResult = await withRetry(doScrape, MAX_RETRIES, RETRY_DELAY_MS, `晋江${pageLabel}`);
 
-    if (!data?.channels?.length) {
-      console.error(`[jjwxc] 采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开 ${url} 确认。`);
-      return null;
-    }
-  } finally {
+  if (dataResult._failed) {
+    console.log(`  ✗ 晋江${pageLabel}连续${MAX_RETRIES}次失败：${dataResult.err.message}`);
     await browser.close();
+    return null;
   }
 
+  data = dataResult;
+  await browser.close();
+
+  console.log(`  ✓ 布局检测：${data.layout === 'A' ? '表格型' : '交替行型'}`);
+
   let totalBooks = 0;
   data.channels.forEach(ch => {
     totalBooks += ch.books.length;

package/plugins/fancy-scan/scripts/scrapers/qimao-rank-scraper.cjs

@@ -42,10 +42,33 @@ const PERIODS = [
 // 工具函数
 // ---------------------------------------------------------------------------
 
+const MAX_RETRIES = 3;
+const RETRY_DELAY_MS = 5000;
+
 function sleep(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+/**
+ * 重试包装器
+ * @returns {{_failed:true, err, label}|*} 失败时返回 _failed=true 的标记对象
+ */
+async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
+  let lastErr;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < retries) {
+        console.log(`  ⏳ ${label} 第${attempt}次失败，${delayMs / 1000}s后重试...`);
+        sleep(delayMs);
+      }
+    }
+  }
+  return { _failed: true, err: lastErr, label };
+}
+
 function rankUrl(channelId, rankTypeId, periodId) {
   const channel = CHANNELS.find(c => c.id === channelId);
   const rt = RANK_TYPES.find(r => r.id === rankTypeId);
@@ -153,7 +176,7 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
   const url = rankUrl(channelId, rankTypeId, periodId);
   console.log(`\n→ 采集 七猫${ch.label}${rt.label}${period ? period.label : ""}...`);
 
-  const browser = await chromium.launch({ headless: false, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
+  const browser = await chromium.launch({ headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
   const context = await browser.newContext();
   const page = await context.newPage();
 
@@ -164,7 +187,7 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
   } catch (_) {}
 
-  // 拦截 XHR 响应，从 JSON 直接拿 bookId + 明文书名/作者，绕过字体加密
+  // 拦截 XHR 响应（只注册一次，放到 withRetry 外面）
   const xhrBookMap = new Map();
   page.on("response", async (resp) => {
     try {
@@ -176,13 +199,12 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
       if (!body || body.length < 100) return;
       let parsed;
       try { parsed = JSON.parse(body); } catch { return; }
-      // 遍历 JSON 结构，找含 bookId + title/author 的数组
       const found = findBookArrays(parsed);
       for (const b of found) {
         if (!b.bookId && !b.novel_id && !b.id) continue;
         const id = String(b.bookId || b.novel_id || b.id);
         xhrBookMap.set(id, {
-          id: id,
+          id,
           title: b.title || b.bookName || b.name || "",
           author: b.author || b.authorName || "",
         });
@@ -190,14 +212,14 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     } catch (_) {}
   });
 
-  let books, rawCount;
-  try {
+  // 单次采集+提取逻辑（包进 withRetry）
+  const pageLabel = `${ch.label}${rt.label}${period ? period.label : ""}`;
+  const doScrape = async () => {
     await page.goto(url, { waitUntil: "networkidle" });
     await page.waitForTimeout(3000);
 
     if (page.url().indexOf("qimao") === -1) {
-      console.error(`  ✗ 当前页面非七猫，可能被重定向，已跳过。`);
-      return null;
+      throw new Error(`非七猫页面（host=${page.url()}），可能被重定向`);
     }
 
     // 验证页面实际选中状态
@@ -224,13 +246,11 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
       observed.rankType !== rt.label ||
       (period && observed.period !== period.label)
     ) {
-      console.error(
-        `  ✗ 页面实际榜单与请求不一致（请求 ${ch.tab}/${rt.label}/${periodLabel}，` +
-        `实际 ${observed.channel || "?"}/${observed.rankType || "?"}/${observed.period || "?"}），已跳过。`
+      throw new Error(
+        `页面榜单不一致（请求 ${ch.tab}/${rt.label}/${periodLabel}，` +
+        `实际 ${observed.channel || "?"}/${observed.rankType || "?"}/${observed.period || "?"}）`
       );
-      return null;
     }
-    console.log(`  ✓ 已验证页面实际榜单：${observed.channel}/${observed.rankType}${observed.period ? "/" + observed.period : ""}`);
 
     // 滚动加载
     for (let i = 0; i < 5; i++) {
@@ -275,10 +295,21 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
       return books;
     });
 
-    rawCount = rawBooks.length;
-    books = rawBooks.filter(isUsableBook);
+    if (!rawBooks.length) throw new Error("innerText 解析结果为空");
+    return rawBooks;
+  };
+
+  const rawResult = await withRetry(doScrape, MAX_RETRIES, RETRY_DELAY_MS, `七猫${pageLabel}`);
+
+  let books = null;
+  if (rawResult._failed) {
+    console.log(`  ✗ 七猫${pageLabel}连续${MAX_RETRIES}次失败：${rawResult.err.message}`);
+    await browser.close();
+    return null;
+  } else {
+    const rawBooks = rawResult.filter(isUsableBook);
 
-    // 直接从 DOM 提取 /shuku/{id}/ 链接补到每本书上（不依赖 XHR 匹配）
+    // DOM 链接补全
     await page.evaluate(() => {
       if (window.__qimaoUrlMap) return;
       var urlMap = {};
@@ -296,18 +327,17 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     const rawUrlMap = await page.evaluate(() => window.__qimaoUrlMap || '{}');
     const urlMap = JSON.parse(rawUrlMap);
     const norm = s => (s || '').replace(/\s+/g, '');
-    for (const b of books) {
+    for (const b of rawBooks) {
       if (!b.url) {
         var found = Object.entries(urlMap).find(([t]) => norm(t) === norm(b.title));
         if (found) b.url = found[1];
       }
     }
 
-    // 用 XHR 数据覆盖明文书名/作者，绕过字体加密
+    // XHR 明文覆盖
     if (xhrBookMap.size > 0) {
       let updated = 0;
-      for (const b of books) {
-        // 尝试从 XHR map 中找匹配的明文数据（优先用 bookId，其次用书名归一匹配）
+      for (const b of rawBooks) {
         const xhrEntry = findXhrMatch(b, xhrBookMap);
         if (xhrEntry) {
           if (xhrEntry.title && xhrEntry.title !== b.title) {
@@ -315,24 +345,24 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
             updated++;
           }
           if (xhrEntry.author) b.author = xhrEntry.author;
-          // 用 XHR 数据里的 id 补 URL（如果有的话）
           const id = xhrEntry.id || findIdByTitle(b.title, xhrBookMap);
           if (id) b.url = `https://www.qimao.com/shuku/${id}/`;
         }
       }
-      console.log(`  ✓ XHR 明文更新 ${updated}/${books.length} 本`);
+      console.log(`  ✓ XHR 明文更新 ${updated}/${rawBooks.length} 本`);
     }
 
-  } finally {
-    await browser.close();
+    books = rawBooks;
   }
 
-  if (!books.length) {
+  await browser.close();
+
+  if (!books || !books.length) {
     console.error(`[qimao] 采集失败：页面结构可能已变。`);
     return null;
   }
 
-  const summary = summarizeQuality(books, rawCount);
+  const summary = summarizeQuality(books, books.length);
   console.log(`  ✓ 提取 ${books.length} 本（链接 ${summary.linked}/${books.length}，热度 ${summary.heated}/${books.length}）`);
 
   const periodLabel = period ? period.label : "";
@@ -358,6 +388,7 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
   return lines.join("\n");
 }
 
+
 function buildTargets(channel, rankType, period) {
   const channels = channel === "all" ? CHANNELS.map(c => c.id) : [channel];
   const rankTypes = rankType === "all" ? RANK_TYPES.map(r => r.id) : [rankType];