@cloud411716/fancy-webnovel 0.1.76 → 0.1.78

package/plugins/fancy-scan/scripts/scrapers/zhihu-rank-scraper.cjs

@@ -2,8 +2,8 @@
 /**
  * 知乎短篇/长篇采集脚本
  *
+ * 使用 playwright-core 自己管理浏览器。
  * 入口：https://www.zhihu.com/fiore/h5/vip-web
- *
  * 短篇（4 个榜单）：推荐榜 / 热搜榜 / 热度榜 / 口碑榜
  * 长篇（1 个榜单）：长篇榜
  *
@@ -11,30 +11,27 @@
  *   node zhihu-rank-scraper.js --length short                 # 短篇 4 个榜单
  *   node zhihu-rank-scraper.js --length long                  # 长篇 1 个榜单
  *   node zhihu-rank-scraper.js --length short --boards rec,hot # 指定短篇子榜
+ *   node zhihu-rank-scraper.js --login-wait 60               # 等待手动登录
  *
- * 前置：
- *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
- *   知乎需要登录态才能看完整榜单，请先用 Chrome 手动登录 zhihu.com
- *
- * 输出：扫榜结果/知乎{长篇|短篇}{榜单名}_{YYYYMMDD}.md
+ * 注意：知乎需要登录态才能看完整榜单。
  */
 
 const fs = require("fs");
 const path = require("path");
-const { ab, evalJSONBase64, getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
+const { chromium } = require("playwright-core");
+const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
 
-const PORT = 9222;
 const OUTDIR = getArg(process.argv, "--outdir") || "扫榜结果";
 const ENTRY_URL = "https://www.zhihu.com/fiore/h5/vip-web";
-
 const LENGTH = getArg(process.argv, "--length") || "short";
+const LOGIN_WAIT = parseInt(getArg(process.argv, "--login-wait") || "0", 10);
 
 // 短篇 4 个榜单
 const SHORT_BOARDS = [
-  { id: "rec", label: "推荐榜", tabText: "推荐榜" },
-  { id: "hot", label: "热搜榜", tabText: "热搜榜" },
-  { id: "trending", label: "热度榜", tabText: "热度榜" },
-  { id: "reputation", label: "口碑榜", tabText: "口碑榜" },
+  { id: "rec",       label: "推荐榜", tabText: "推荐榜" },
+  { id: "hot",       label: "热搜榜", tabText: "热搜榜" },
+  { id: "trending",  label: "热度榜", tabText: "热度榜" },
+  { id: "reputation",label: "口碑榜", tabText: "口碑榜" },
 ];
 
 // 长篇 1 个榜单
@@ -44,83 +41,88 @@ const LONG_BOARDS = [
 
 const BOARDS = LENGTH === "long" ? LONG_BOARDS : SHORT_BOARDS;
 
-// 自定义子榜（可选）
 const customBoards = getArg(process.argv, "--boards");
 const targetBoards = customBoards
-  ? BOARDS.filter((b) => customBoards.split(",").map((s) => s.trim()).includes(b.id))
+  ? BOARDS.filter(b => customBoards.split(",").map(s => s.trim()).includes(b.id))
   : BOARDS;
 
-/**
- * 在页面里点指定 tab，等待内容刷新，再提取故事列表
- */
-async function scrapeBoard(port, board) {
+// ---------------------------------------------------------------------------
+// 工具函数
+// ---------------------------------------------------------------------------
+
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+// ---------------------------------------------------------------------------
+// 采集
+// ---------------------------------------------------------------------------
+
+async function scrapeBoard(board) {
   console.log(`  [${board.label}] 正在抓取...`);
 
-  // 导航到入口页（每次都重新进入首页，避免 SPA 状态污染）
-  const navResult = ab(port, "navigate", ENTRY_URL);
-  if (!navResult.ok) {
-    console.error(`  [${board.label}] 导航失败: ${navResult.error || "unknown"}`);
-    return null;
-  }
+  const browser = await chromium.launch({ headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage"] });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  try {
+    await page.goto(ENTRY_URL, { waitUntil: "networkidle" });
+
+    if (LOGIN_WAIT > 0) {
+      console.log(`  ⏳ 等待 ${LOGIN_WAIT}s 供用户登录...`);
+      await page.waitForTimeout(LOGIN_WAIT * 1000);
+    }
+
+    const host = page.url();
+    if (host.indexOf("zhihu") === -1) {
+      console.error(`  ✗ 当前页面非知乎（url=${host}），可能被重定向，已跳过。`);
+      return null;
+    }
 
-  // 等待页面加载
-  await new Promise((r) => setTimeout(r, 3000));
-
-  // 点击对应 tab
-  // 用包含 tabText 的元素点击（兼容多种 class 命名）
-  const clickJs = `
-    (function(){
-      const target = ${JSON.stringify(board.tabText)};
-      const candidates = Array.from(document.querySelectorAll('a, button, div, span, li'))
-        .filter((el) => {
-          const t = (el.innerText || '').trim();
+    // 点击 tab
+    const clicked = await page.evaluate((target) => {
+      const candidates = Array.from(document.querySelectorAll("a, button, div, span, li"))
+        .filter(el => {
+          const t = (el.innerText || "").trim();
           return t === target || t.startsWith(target);
         });
-      if (candidates.length === 0) return JSON.stringify({ok:false, reason:'tab_not_found', target:target});
+      if (!candidates.length) return false;
       candidates[0].click();
-      return JSON.stringify({ok:true, clicked:candidates[0].tagName});
-    })()
-  `;
-  const clickResult = evalJSONBase64(port, clickJs);
-  if (!clickResult || !clickResult.ok) {
-    console.error(`  [${board.label}] 点击 tab 失败: ${JSON.stringify(clickResult)}`);
-    return null;
-  }
+      return true;
+    }, board.tabText);
+
+    if (!clicked) {
+      console.error(`  [${board.label}] 点击 tab 失败`);
+      return null;
+    }
 
-  // 等待内容加载
-  await new Promise((r) => setTimeout(r, 2500));
+    await page.waitForTimeout(2500);
 
-  // 提取故事列表
-  // 兼容多种可能的选择器（知乎没有公开 class 文档，按常见模式）
-  const storiesJs = `
-    (function(){
+    // 提取故事列表
+    const stories = await page.evaluate(() => {
       const items = [];
       const seen = new Set();
-
-      // 策略 1：找所有含故事标题的 anchor
       const anchors = document.querySelectorAll('a[href*="/p/"], a[href*="/question/"], a[href*="/xen/"]');
       anchors.forEach((a, i) => {
-        const href = a.href || '';
+        const href = a.href || "";
         if (!href || seen.has(href)) return;
-        const text = (a.innerText || '').trim();
+        const text = (a.innerText || "").trim();
         if (!text || text.length < 4 || text.length > 200) return;
-        // 跳过导航/按钮文字
         if (/^(首页|发现|等你来答|登录|注册|更多|查看全部)$/.test(text)) return;
         seen.add(href);
-        items.push({rank: i + 1, title: text.split('\\n')[0].trim(), url: href});
+        items.push({ rank: i + 1, title: text.split("\n")[0].trim(), url: href });
       });
+      return items.slice(0, 100);
+    });
 
-      return JSON.stringify(items.slice(0, 100));
-    })()
-  `;
-
-  const stories = evalJSONBase64(port, storiesJs);
-  if (!stories || stories.length === 0) {
-    console.warn(`  [${board.label}] 未提取到故事`);
-    return [];
+    if (!stories || stories.length === 0) {
+      console.warn(`  [${board.label}] 未提取到故事`);
+      return [];
+    }
+    return stories;
+  } finally {
+    await browser.close();
   }
-
-  return stories;
 }
 
 function buildBoardMarkdown(board, stories, date) {
@@ -129,36 +131,35 @@ function buildBoardMarkdown(board, stories, date) {
     "",
     `> 数据来源：${ENTRY_URL} → ${board.label}`,
     `> 篇幅：${LENGTH === "long" ? "长篇" : "短篇"}`,
-    `> 知乎需要登录态才能看完整榜单，请确认 Chrome 已登录`,
+    "> 知乎需要登录态才能看完整榜单，请确认已登录",
     "",
     `## ${board.label}内容`,
     "",
   ];
-
-  if (stories.length === 0) {
-    lines.push("（未提取到数据，可能页面结构变化或需要登录）", "");
+  if (!stories.length) {
+    lines.push("（未提取到数据，可能页面结构变化或需要登录）", "", "---", "");
   } else {
-    stories.forEach((s) => {
+    stories.forEach(s => {
       lines.push(`### ${s.rank}. ${s.title || "(无标题)"}`);
       if (s.url) lines.push(`- 链接：${s.url}`);
       lines.push("", "---", "");
     });
   }
-
   return lines.join("\n");
 }
 
-function main() {
+// ---------------------------------------------------------------------------
+// 入口
+// ---------------------------------------------------------------------------
+
+async function main() {
   const date = localDateStamp();
   fs.mkdirSync(OUTDIR, { recursive: true });
 
   let written = 0;
   for (const board of targetBoards) {
-    const stories = scrapeBoard(PORT, board);
-    if (stories === null) {
-      // 采集失败（导航失败 / tab 找不到），跳过这个榜单继续下一个
-      continue;
-    }
+    const stories = await scrapeBoard(board);
+    if (stories === null) continue;
 
     const md = buildBoardMarkdown(board, stories, date);
     const filename = `知乎${LENGTH === "long" ? "长篇" : "短篇"}${board.label}_${date}.md`;
@@ -167,7 +168,6 @@ function main() {
     console.log(`  ✓ ${filename} (${stories.length} 条)`);
     written++;
   }
-
   return written;
 }