@cloud411716/fancy-webnovel 1.0.46 → 2.2.0

package/plugins/fancy-scan/scripts/scrapers/jjwxc-rank-scraper.js

@@ -0,0 +1,339 @@
+#!/usr/bin/env node
+/**
+ * 晋江文学城排行榜采集脚本
+ *
+ * 使用 playwright-core 直接控制浏览器采集。
+ * 采集策略：
+ *   1) topten.php 列表页（纯文本，频道名直接出现，书名/作者交替行）解出频道分组。
+ *   2) 从书名 anchor 取 novelid，逐本进 onebook.php 详情页补采核心指标
+ *      （收藏数/营养液/积分/字数/状态），满足规范对晋江的硬性要求。
+ * 晋江页面为 gb18030 编码：详情页用 fetch+arrayBuffer+TextDecoder('gb18030') 解码。
+ * 详情采集默认开启但有上限（每频道前 N + 总量上限），用 --list-only 可只采列表。
+ *
+ * 用法：
+ *   node jjwxc-rank-scraper.js --type 12                  # 收入金榜
+ *   node jjwxc-rank-scraper.js --type 12 --top 15         # 每频道补采前 15 本
+ *   node jjwxc-rank-scraper.js --type 12 --detail-limit 60 # 详情总量上限 60
+ *   node jjwxc-rank-scraper.js --type 12 --list-only      # 只采列表
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { pathToFileURL } from 'url';
+import { chromium } from 'playwright-core';
+import { getArg, localDateStamp, runCli } from '#scraper-utils';
+
+const BASE_URL = "https://www.jjwxc.net/topten.php";
+
+const RANK_TYPES = [
+  { id: "5",  label: "月榜" },
+  { id: "7",  label: "总分榜" },
+  { id: "4",  label: "季度榜" },
+  { id: "3",  label: "新晋作者榜" },
+];
+
+const DETAIL_CHUNK = 6;
+const DETAIL_LIMIT = 100;
+const TOP = 10;
+
+// ---------------------------------------------------------------------------
+// 命令行参数解析
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+const RANKTYPE = getArg(args, "--type") || "12";
+const LIST_ONLY = args.includes("--list-only");
+const OUTDIR = getArg(args, "--outdir") || '.';
+
+// ---------------------------------------------------------------------------
+// 列表页提取（使用 Playwright）
+// ---------------------------------------------------------------------------
+
+/**
+ * 提取晋江榜单数据（从表格提取）
+ */
+async function extractRankData(page) {
+  return await page.evaluate(() => {
+    var result = { channels: [{ name: '全站', books: [] }] };
+
+    // 从表格提取书籍数据
+    var tables = document.querySelectorAll('table');
+    for (var t = 0; t < tables.length; t++) {
+      var table = tables[t];
+      var rows = table.querySelectorAll('tr');
+      for (var i = 0; i < rows.length; i++) {
+        var row = rows[i];
+        var cells = row.querySelectorAll('td');
+        if (cells.length < 6) continue;
+
+        // 检查是否是书籍行（第一列是数字序号）
+        var rankCell = cells[0];
+        var rankNum = parseInt(rankCell.innerText.trim(), 10);
+        if (isNaN(rankNum)) continue;
+
+        var authorCell = cells[1];
+        var titleCell = cells[2];
+        var typeCell = cells[3];
+        var statusCell = cells[4];
+        var wordsCell = cells[5];
+        var scoreCell = cells[6];
+
+        var author = (authorCell.innerText || '').trim();
+        var title = (titleCell.innerText || '').trim();
+        var type = (typeCell.innerText || '').trim();
+        var status = (statusCell.innerText || '').trim();
+        var words = (wordsCell.innerText || '').trim();
+        var score = (scoreCell.innerText || '').trim();
+
+        // 排除标题行
+        if (author === '作者' || title === '作品') continue;
+
+        // 获取 novelid
+        var novelid = '';
+        var link = titleCell.querySelector('a');
+        if (link) {
+          var href = link.getAttribute('href') || '';
+          var m = href.match(/novelid=([0-9]+)/);
+          if (m) novelid = m[1];
+        }
+
+        result.channels[0].books.push({
+          title: title,
+          author: author,
+          novelid: novelid,
+          type: type,
+          status: status,
+          words: words,
+          score: score
+        });
+      }
+    }
+    return result;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// 详情页提取（gb18030 + itemprop 微数据）
+// ---------------------------------------------------------------------------
+
+async function fetchDetails(page, ids) {
+  // 使用 Playwright 的 page.evaluate 来执行详情页抓取
+  const details = await page.evaluate(async (ids) => {
+    const results = [];
+    for (const id of ids) {
+      try {
+        // redirect: 'error' 防止 onebook.php 重定向到外站造成 SSRF
+        const response = await fetch(`/onebook.php?novelid=${id}`, { redirect: 'error' });
+        const buffer = await response.arrayBuffer();
+        const decoder = new TextDecoder('gb18030');
+        const html = decoder.decode(buffer);
+
+        function prop(name) {
+          const m = html.match(new RegExp(`itemprop="${name}"[^>]*>([^<]*) <`));
+          return m ? m[1].trim() : '';
+        }
+
+        const statusMatch = html.match(/itemprop="updataStatus"[^>]*>\s*([^\s<]{1,6})/) || [];
+        const status = statusMatch[1] || (html.includes('连载中') ? '连载中' : html.includes('已完结') ? '已完结' : '');
+
+        results.push({
+          id: id,
+          collect: prop('collectedCount'),
+          nutrition: prop('nutritionCount'),
+          score: prop('scoreCount'),
+          review: prop('reviewCount'),
+          words: prop('wordCount'),
+          status: status
+        });
+      } catch (e) {
+        results.push({ id: id, err: String(e) });
+      }
+    }
+    return results;
+  }, ids);
+
+  const map = {};
+  for (const d of details) {
+    map[d.id] = d;
+  }
+  return map;
+}
+
+// ---------------------------------------------------------------------------
+// 采集单个榜单
+// ---------------------------------------------------------------------------
+
+async function scrapeRank(browser, rankTypeId, channelId) {
+  const rt = RANK_TYPES.find((r) => r.id === rankTypeId);
+  if (!rt) {
+    console.error(`  ⚠ 未知榜单类型: ${rankTypeId}`);
+    return null;
+  }
+
+  const url = `${BASE_URL}?orderstr=${rankTypeId}&t=${channelId}`;
+  const chLabel = channelId === "0" ? "全站" : `频道${channelId}`;
+
+  const page = await browser.newPage();
+
+  try {
+    await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
+
+    const data = await extractRankData(page);
+    if (!data?.channels?.length) {
+      console.error(`❌ 晋江-${rt.label}（${chLabel}）采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开 ${url} 确认。`);
+      return null;
+    }
+
+    let totalBooks = 0;
+    data.channels.forEach((ch) => {
+      totalBooks += ch.books.length;
+      const authors = new Set(ch.books.map((b) => b.author));
+      if (ch.books.length >= 5 && authors.size / ch.books.length < 0.2) {
+        console.error(`  ⚠ ${ch.name}：${ch.books.length} 本只有 ${authors.size} 个唯一作者，可能提取有误`);
+      }
+    });
+    console.error(`  ✓ 列表：${data.channels.length} 个频道，共 ${totalBooks} 本`);
+
+    // 构建输出内容
+    const now = new Date().toISOString();
+    const lines = [
+      `# 晋江 · ${rt.label}`,
+      "",
+      `- 来源：${url}`,
+      `- 抓取时间：${now}`,
+      `- 频道数：${data.channels.length}`,
+      `- 总条目数：${totalBooks}`,
+      "",
+      "---",
+      "",
+    ];
+
+    for (const ch of data.channels) {
+      lines.push(`## ${ch.name} — ${ch.books.length} 本`, "");
+      for (let i = 0; i < ch.books.length; i++) {
+        const b = ch.books[i];
+        lines.push(`### #${i + 1} ${b.title}`);
+        if (b.author) lines.push(`**作者：** ${b.author}`);
+        if (b.type) lines.push(`**题材：** ${b.type}`);
+        if (b.status) lines.push(`**状态：** ${b.status}`);
+        if (b.words) lines.push(`**字数：** ${b.words}`);
+        if (b.score) lines.push(`**积分：** ${b.score}`);
+        if (b.novelid) lines.push(`[作品页](https://www.jjwxc.net/onebook.php?novelid=${b.novelid})`);
+        lines.push("");
+      }
+      lines.push("---", "");
+    }
+
+    return {
+      content: lines.join("\n"),
+      partial: false,
+    };
+  } finally {
+    await page.close();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// 主函数
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const CHANNEL = getArg(args, "--channel") || "0";
+
+  if (RANKTYPE !== "all" && !RANK_TYPES.some((rank) => rank.id === RANKTYPE)) {
+    throw new Error(`未知 --type: ${RANKTYPE}`);
+  }
+  if (CHANNEL !== "0") {
+    throw new Error(`未知 --channel: ${CHANNEL}（当前仅支持 0=全站）`);
+  }
+
+  const rankTypes = RANKTYPE === "all" ? RANK_TYPES.map((r) => r.id) : [RANKTYPE];
+  const channels = [CHANNEL];
+
+  let written = 0;
+  let failed = 0;
+  let partial = false;
+  const partialReasons = [];
+  const files = [];
+
+  // 启动浏览器
+  let browser;
+  // 被 SIGTERM 杀时（上游超时或中断）主动关掉浏览器，
+  // 否则 chromium 会留在系统里继续占 CPU/内存。
+  // close 必须带超时兜底：浏览器正在导航时 close() 可能挂住，
+  // 无超时的话进程会卡住不退出（实测过）。
+  process.on('SIGTERM', async () => {
+    try {
+      if (browser) {
+        await Promise.race([
+          browser.close(),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('close timeout')), 5000)),
+        ]);
+      }
+    } catch (_) {}
+    process.exit(143);
+  });
+  try {
+    browser = await chromium.launch({ headless: true, timeout: 30000 });
+  } catch (launchErr) {
+    console.log(JSON.stringify({ ok: false, error: `晋江采集失败：浏览器启动失败：${launchErr.message}` }));
+    process.exit(1);
+  }
+  try {
+    for (const rt of rankTypes) {
+      for (const ch of channels) {
+        try {
+          const result = await scrapeRank(browser, rt, ch);
+          if (!result) {
+            failed++;
+            const rtInfo = RANK_TYPES.find((r) => r.id === rt);
+            partialReasons.push(`${rtInfo ? rtInfo.label : rt}: no usable data`);
+            continue;
+          }
+          if (result.partial) {
+            partial = true;
+          }
+
+          const rtInfo = RANK_TYPES.find((r) => r.id === rt);
+          const date = localDateStamp();
+          const chLabel = ch === "0" ? "全站" : `频道${ch}`;
+          const filename = `晋江${rtInfo.label}_${chLabel}_${date}.md`;
+          fs.mkdirSync(OUTDIR, { recursive: true });
+          const filepath = path.join(OUTDIR, filename);
+          fs.writeFileSync(filepath, result.content, "utf-8");
+          written++;
+          files.push(filepath);
+          console.error(`  ✓ 已保存: ${filepath}`);
+        } catch (rankErr) {
+          failed++;
+          const rtInfo = RANK_TYPES.find((r) => r.id === rt);
+          const message = rankErr && rankErr.message ? rankErr.message : String(rankErr);
+          partialReasons.push(`${rtInfo ? rtInfo.label : rt}: ${message}`);
+          console.error(`❌ 晋江-${rtInfo ? rtInfo.label : rt}采集失败：${message}`);
+        }
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  return {
+    planned: rankTypes.length * channels.length,
+    written,
+    failed,
+    partial: partial || failed > 0,
+    partialReasons,
+    files,
+  };
+}
+
+
+// import.meta.main 需要 Node >= 22.19（package.json 声明支持 node >=18.0.0）。
+// 旧版本上它是 undefined，必须回落到 argv[1] 比对 —— 否则整个脚本静默空跑：
+// exit 0、无 stdout、没有任何采集文件，调用方只看到"采集失败"。
+const isMain = import.meta.main ??
+  (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href);
+
+if (isMain) {
+  runCli(main(), "晋江采集");
+}