@cloud411716/fancy-webnovel 0.2.10 → 0.2.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloud411716/fancy-webnovel",
-  "version": "0.2.10",
+  "version": "0.2.11",
   "type": "module",
   "main": "index.js",
   "exports": {

package/plugins/fancy-scan/scripts/scrapers/fanqie-rank-scraper.cjs

@@ -312,49 +312,60 @@ async function scrapeChannel(ch, type) {
       const intercepted = [];
       const seen = new Set();
 
+      // 先注册拦截器，再导航到目标页（避免漏掉 initial 请求）
+      context.on("request", req => {
+        const url = req.url();
+        if (seen.has(url)) return;
+        if (/\.(css|woff|jpg|png|ico)/.test(url)) return;
+        if (!url.includes("fanqie")) return;
+        seen.add(url);
+        console.log(`[REQ] ${url}`);
+      });
+
       page.on("response", async (resp) => {
         const url = resp.url();
-        const ct = resp.headers()["content-type"] || "";
-        // 只关心 JSON 响应
-        if (!ct.includes("json") && !ct.includes("text")) return;
-        // 跳过静态资源
+        const ct = (resp.headers()["content-type"] || "").toLowerCase();
+        if (seen.has(url)) return;
+        if (!ct.includes("json") && !ct.includes("text") && !url.includes("api")) return;
         if (/\.(css|woff|jpg|png|ico)/.test(url)) return;
         try {
           const body = await resp.text();
           if (!body || body.length < 50) return;
-          // 尝试解析 JSON
           let parsed;
           try { parsed = JSON.parse(body); } catch { return; }
-          // 找含书籍数据的响应
           const str = JSON.stringify(parsed);
           if (str.includes("bookId") || str.includes("book_name") || str.includes("bookName") || str.includes("authorName")) {
-            if (!seen.has(url)) {
-              seen.add(url);
-              intercepted.push({ url, preview: str.slice(0, 400) });
-            }
+            console.log(`[JSON] ${url}`);
+            console.log(`  预览: ${str.slice(0, 300)}`);
+            intercepted.push({ url, preview: str.slice(0, 400) });
           }
         } catch {}
       });
 
-      // 触发几个品类页的请求
+      // 直接导航到品类页（不用 waitUntil，避免卡住）
       const cats = await extractCategories(page, ch, type);
       if (!cats.length) {
-        await scrollLoad(page, 2);
-        sleep(1000);
-      }
-      const targets = cats.length ? cats.slice(0, 3) : [{ name: "入口页", href: `/rank/${ch}_${type}_${initCatId}` }];
-      for (const c of targets) {
-        console.log(`  探测: ${c.name} → ${c.href}`);
-        await page.goto(`https://fanqienovel.com${c.href}`, { waitUntil: "networkidle" });
-        await page.waitForTimeout(2000);
+        await page.goto(`https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`);
+        await page.waitForTimeout(3000);
+        const moreCats = await extractCategories(page, ch, type);
+        if (moreCats.length) {
+          console.log(`  发现 ${moreCats.length} 个品类`);
+          for (const c of moreCats.slice(0, 3)) {
+            console.log(`  探测品类: ${c.name} → ${c.href}`);
+            await page.goto(`https://fanqienovel.com${c.href}`);
+            await page.waitForTimeout(2000);
+          }
+        }
+      } else {
+        console.log(`  发现 ${cats.length} 个品类`);
+        for (const c of cats.slice(0, 3)) {
+          console.log(`  探测品类: ${c.name} → ${c.href}`);
+          await page.goto(`https://fanqienovel.com${c.href}`);
+          await page.waitForTimeout(2000);
+        }
       }
 
-      console.log(`\n=== 拦截到的接口 (${intercepted.length}) ===\n`);
-      for (const { url, preview } of intercepted) {
-        console.log(`URL: ${url}`);
-        console.log(`预览: ${preview}`);
-        console.log("---");
-      }
+      console.log(`\n=== 共拦截到 ${intercepted.length} 个含书籍数据的接口 ===`);
       await browser.close();
       return null;
     }