@cloud411716/fancy-webnovel 0.2.15 → 0.2.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloud411716/fancy-webnovel",
-  "version": "0.2.15",
+  "version": "0.2.16",
   "type": "module",
   "main": "index.js",
   "exports": {

package/plugins/fancy-scan/scripts/scrapers/qimao-rank-scraper.cjs

@@ -95,6 +95,52 @@ function summarizeQuality(books, rawCount) {
   return { linked, heated, problems, quality: problems.length ? "[存在问题]" : "[OK]" };
 }
 
+// 遍历 JSON 对象/数组，查找含 bookId + title/author 的数组
+function findBookArrays(obj, depth = 0) {
+  if (!obj || depth > 8) return [];
+  if (Array.isArray(obj)) {
+    for (const item of obj) {
+      if (item && typeof item === "object" && (item.bookId || item.novel_id || item.id)) {
+        if (item.title || item.bookName || item.author || item.authorName) {
+          return obj;
+        }
+      }
+    }
+    for (const item of obj) {
+      const found = findBookArrays(item, depth + 1);
+      if (found.length) return found;
+    }
+    return [];
+  }
+  if (typeof obj === "object") {
+    for (const key of Object.keys(obj)) {
+      const found = findBookArrays(obj[key], depth + 1);
+      if (found.length) return found;
+    }
+  }
+  return [];
+}
+
+// 从 xhrBookMap 中为一本 innerText 书匹配明文数据
+function findXhrMatch(book, xhrBookMap) {
+  // 先用归一化书名直接匹配
+  const norm = s => (s || "").replace(/\s+/g, "");
+  for (const [id, data] of xhrBookMap) {
+    if (norm(data.title) === norm(book.title)) {
+      return { ...data, id };
+    }
+  }
+  return null;
+}
+
+function findIdByTitle(title, xhrBookMap) {
+  const norm = s => (s || "").replace(/\s+/g, "");
+  for (const [id, data] of xhrBookMap) {
+    if (norm(data.title) === norm(title)) return id;
+  }
+  return null;
+}
+
 // ---------------------------------------------------------------------------
 // 采集
 // ---------------------------------------------------------------------------
@@ -119,7 +165,32 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
   } catch (_) {}
 
-  let books, urls, rawCount;
+  // 拦截 XHR 响应，从 JSON 直接拿 bookId + 明文书名/作者，绕过字体加密
+  const xhrBookMap = new Map();
+  page.on("response", async (resp) => {
+    try {
+      const respUrl = resp.url();
+      const ct = (resp.headers()["content-type"] || "").toLowerCase();
+      if (!ct.includes("json") && !respUrl.includes("/api/")) return;
+      if (/\.(css|woff|jpg|png|ico)/.test(respUrl)) return;
+      const body = await resp.text();
+      if (!body || body.length < 100) return;
+      let parsed;
+      try { parsed = JSON.parse(body); } catch { return; }
+      // 遍历 JSON 结构，找含 bookId + title/author 的数组
+      const found = findBookArrays(parsed);
+      for (const b of found) {
+        if (!b.bookId && !b.novel_id && !b.id) continue;
+        const id = String(b.bookId || b.novel_id || b.id);
+        xhrBookMap.set(id, {
+          title: b.title || b.bookName || b.name || "",
+          author: b.author || b.authorName || "",
+        });
+      }
+    } catch (_) {}
+  });
+
+  let books, rawCount;
   try {
     await page.goto(url, { waitUntil: "networkidle" });
     await page.waitForTimeout(3000);
@@ -207,22 +278,26 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     rawCount = rawBooks.length;
     books = rawBooks.filter(isUsableBook);
 
-    // 提取作品页链接
-    urls = await page.evaluate(() => {
-      var byId = {};
-      var order = [];
-      Array.from(document.querySelectorAll("a")).forEach(a => {
-        var h = (a.getAttribute("href") || a.href || "").match(/\/(?:shuku|book)\/([0-9]+)/);
-        if (!h) return;
-        var id = h[1];
-        var t = (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim();
-        if (!byId[id]) { byId[id] = ""; order.push(id); }
-        if (t && !/^[0-9]+$/.test(t) && !/^(最近更新、最新章节、最新)/.test(t)) {
-          if (t.length > byId[id].length) byId[id] = t;
+    // 用 XHR 数据覆盖明文书名/作者，绕过字体加密
+    if (xhrBookMap.size > 0) {
+      let updated = 0;
+      for (const b of books) {
+        // 尝试从 XHR map 中找匹配的明文数据（优先用 bookId，其次用书名归一匹配）
+        const xhrEntry = findXhrMatch(b, xhrBookMap);
+        if (xhrEntry) {
+          if (xhrEntry.title && xhrEntry.title !== b.title) {
+            b.title = xhrEntry.title;
+            updated++;
+          }
+          if (xhrEntry.author) b.author = xhrEntry.author;
+          // 用 XHR 数据里的 id 补 URL（如果有的话）
+          const id = xhrEntry.id || findIdByTitle(b.title, xhrBookMap);
+          if (id) b.url = `https://www.qimao.com/shuku/${id}/`;
         }
-      });
-      return order.map(id => ({ bookId: id, title: byId[id], url: "https://www.qimao.com/shuku/" + id + "/" }));
-    });
+      }
+      console.log(`  ✓ XHR 明文更新 ${updated}/${books.length} 本`);
+    }
+
   } finally {
     await browser.close();
   }
@@ -232,13 +307,6 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
     return null;
   }
 
-  // 按书名匹配 URL
-  const norm = s => (s || "").replace(/\s+/g, "");
-  for (const b of books) {
-    const matched = urls.find(u => norm(u.title) === norm(b.title));
-    if (matched) b.url = matched.url;
-  }
-
   const summary = summarizeQuality(books, rawCount);
   console.log(`  ✓ 提取 ${books.length} 本（链接 ${summary.linked}/${books.length}，热度 ${summary.heated}/${books.length}）`);