@cloud411716/fancy-webnovel 0.3.15 → 1.0.0

package/plugins/fancy-scan/scripts/scrapers/jjwxc-rank-scraper.cjs

@@ -1,384 +0,0 @@
-#!/usr/bin/env node
-/**
- * 晋江文学城排行榜采集脚本
- *
- * 使用 playwright-core 自己管理浏览器。
- * 采集策略：
- *   布局 A（表格型）：orderstr=5/7/4/15 等，表头含"序号/作者/作品"，直接解析表格。
- *   布局 B（交替行型）：orderstr=12/16/17 等，按频道分段、书名/作者交替行解析。
- *   详情补采：从书名 anchor 取 novelid，逐本进 onebook.php 补核心指标
- *   （收藏数/营养液/积分/字数/状态），满足规范对晋江的硬性要求。
- * 晋江页面为 gb18030 编码：详情页用 fetch+arrayBuffer+TextDecoder('gb18030') 解码。
- *
- * 用法：
- *   node jjwxc-rank-scraper.js --type 12                  # 收入金榜（默认含详情）
- *   node jjwxc-rank-scraper.js --type 12 --top 15         # 每频道补采前 15 本
- *   node jjwxc-rank-scraper.js --type 12 --detail-limit 60 # 详情总量上限 60
- *   node jjwxc-rank-scraper.js --type 12 --list-only      # 只采列表（快，无核心指标）
- *   node jjwxc-rank-scraper.js --type all                 # 全部榜单
- *   node jjwxc-rank-scraper.js --login-wait 60            # 等待手动登录
- */
-
-const fs = require("fs");
-const path = require("path");
-const { chromium } = require("playwright-core");
-const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
-
-const BASE_URL = "https://www.jjwxc.net/topten.php";
-const DETAIL_BASE = "https://www.jjwxc.net/onebook.php";
-
-const RANK_TYPES = [
-  { id: "5",  label: "月榜" },
-  { id: "7",  label: "总分榜" },
-  { id: "4",  label: "季度榜" },
-  { id: "12", label: "收入金榜" },
-  { id: "16", label: "完结金榜" },
-  { id: "15", label: "勤奋指数榜" },
-  { id: "17", label: "新手金榜" },
-];
-
-const DETAIL_CHUNK = 6;
-
-// ---------------------------------------------------------------------------
-// 工具函数
-// ---------------------------------------------------------------------------
-
-function sleep(ms) {
-  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
-}
-
-function fmtWan(s, unit) {
-  if (s == null || s === "") return "";
-  const n = parseInt(String(s).replace(/[^0-9]/g, ""), 10);
-  if (isNaN(n)) return "";
-  if (n >= 10000) return (n / 10000).toFixed(1) + "万" + (unit || "");
-  return n + (unit || "");
-}
-
-// ---------------------------------------------------------------------------
-// 详情页（Node.js fetch + gb18030 解码）
-// ---------------------------------------------------------------------------
-
-/** 单次详情请求（Node.js fetch，gb18030 解码） */
-async function fetchOneDetail(novelid) {
-  try {
-    const res = await fetch(`${DETAIL_BASE}?novelid=${novelid}`);
-    if (!res.ok) return { id: novelid, err: `HTTP ${res.status}` };
-    const buf = await res.arrayBuffer();
-    const h = new TextDecoder("gb18030").decode(new Uint8Array(buf));
-    function prop(n) {
-      const m = h.match(new RegExp(`itemprop="${n}"[^>]*>([^<]*)<`));
-      return m ? m[1].trim() : "";
-    }
-    const status =
-      (h.match(/itemprop="updataStatus"[^>]*>\s*([^\s<]{1,6})/) || [, ""])[1] ||
-      (h.match(/(连载中|已完结|完结)/) || [, ""])[1] ||
-      "";
-    return {
-      id: novelid,
-      collect: prop("collectedCount"),
-      nutrition: prop("nutritionCount"),
-      score: prop("scoreCount"),
-      review: prop("reviewCount"),
-      words: prop("wordCount"),
-      status,
-    };
-  } catch (e) {
-    return { id: novelid, err: String(e && e.message || e) };
-  }
-}
-
-/** 分批补采详情 */
-async function fetchDetails(ids) {
-  const map = {};
-  let failedChunks = 0;
-  for (let i = 0; i < ids.length; i += DETAIL_CHUNK) {
-    const chunk = ids.slice(i, i + DETAIL_CHUNK);
-    const results = await Promise.all(chunk.map(id => fetchOneDetail(id)));
-    for (const r of results) map[r.id] = r;
-    sleep(400);
-  }
-  return map;
-}
-
-// ---------------------------------------------------------------------------
-// 主流程
-// ---------------------------------------------------------------------------
-
-async function scrapeRank(rankTypeId, channelId) {
-  const rt = RANK_TYPES.find(r => r.id === rankTypeId);
-  if (!rt) { console.log(`  ⚠ 未知榜单类型: ${rankTypeId}`); return null; }
-
-  const url = `${BASE_URL}?orderstr=${rankTypeId}&t=${channelId}`;
-  const chLabel = channelId === "0" ? "全站" : `频道${channelId}`;
-  console.log(`\n→ 采集 晋江${rt.label}（${chLabel}）...`);
-  console.log(`  URL: ${url}`);
-
-  const browser = await chromium.launch({ headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
-  const context = await browser.newContext();
-  const page = await context.newPage();
-
-  // CDP 强制窗口最大化
-  try {
-    const cdp = await context.newCDPSession(page);
-    const { windowId } = await cdp.send('Browser.getWindowForTarget', { target: page.target() });
-    await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
-  } catch (_) {}
-
-  let data = null;
-  try {
-    await page.goto(url, { waitUntil: "networkidle" });
-    await page.waitForTimeout(3000);
-
-    // 连通性自检
-    const host = page.url();
-    if (host.indexOf("jjwxc") === -1) {
-      console.error(`  ✗ 当前页面非晋江（url=${host}），可能被重定向，已跳过。`);
-      return null;
-    }
-
-    // 布局检测：先找含"序号"/"作者"/"作品"表头的表格，有则为布局A（表格），无为布局B（交替行）
-    data = await page.evaluate(() => {
-      // ── 布局 A：表格型（表头在第一行 <tr> 的 <td> 里，含"序号""作者""作品"） ──
-      var tables = document.querySelectorAll('table');
-      for (var ti = 0; ti < tables.length; ti++) {
-        var firstRow = tables[ti].querySelector('tr');
-        if (!firstRow) continue;
-        var headerCells = firstRow.querySelectorAll('td,th');
-        var headerText = Array.from(headerCells).map(function(c) { return c.innerText.trim(); }).join('');
-        if (headerText.indexOf('序号') >= 0 && headerText.indexOf('作者') >= 0 && headerText.indexOf('作品') >= 0) {
-          var rows = tables[ti].querySelectorAll('tr');
-          var books = [];
-          for (var ri = 1; ri < rows.length; ri++) {
-            var cells = rows[ri].querySelectorAll('td');
-            if (cells.length < 3) continue;
-            var rankCell = cells[0].innerText.trim();
-            if (!/^\d+$/.test(rankCell)) continue;
-            var author = cells[1].innerText.trim();
-            var titleLink = cells[2].querySelector('a');
-            var title = titleLink ? titleLink.innerText.trim() : cells[2].innerText.trim();
-            var novelid = '';
-            if (titleLink) {
-              var m = (titleLink.getAttribute('href') || '').match(/novelid=([0-9]+)/);
-              if (m) novelid = m[1];
-            }
-            var typeText = cells.length > 3 ? cells[3].innerText.trim() : '';
-            var progress = cells.length > 4 ? cells[4].innerText.trim() : '';
-            var words = cells.length > 5 ? cells[5].innerText.trim() : '';
-            var score = cells.length > 6 ? cells[6].innerText.trim() : '';
-            var pubtime = cells.length > 7 ? cells[7].innerText.trim() : '';
-            books.push({ rank: parseInt(rankCell), title: title, author: author, novelid: novelid, type: typeText, progress: progress, words: words, score: score, pubtime: pubtime });
-          }
-          return { layout: 'A', channels: [{ name: '全站', books: books }] };
-        }
-      }
-      // ── 布局 B：交替行型（频道分段，书名/作者交替） ──
-      var result = { layout: 'B', channels: [] };
-      var text = document.body.innerText || "";
-      var lines = text.split(/\n/).map(function(l) { return l.trim(); }).filter(Boolean);
-
-      var idMap = {};
-      Array.from(document.querySelectorAll("a")).forEach(function(a) {
-        var hm = (a.getAttribute("href") || "").match(/novelid=([0-9]+)/);
-        if (!hm) return;
-        var t = (a.innerText || a.textContent || "").trim();
-        if (!t || t.indexOf("向《") > -1 || t.indexOf("投") > -1 || t.length > 30) return;
-        if (!idMap[t]) idMap[t] = hm[1];
-      });
-
-      var channels = ["古代言情","现代言情","古代穿越","现代都市纯爱","现代幻想纯爱","古代纯爱","衍生纯爱","幻想现言","奇幻言情","未来游戏悬疑","百合","无CP","二次元言情","衍生言情","衍生无cp","未来幻想纯爱","原创轻小说","多元"];
-      var channelSet = {};
-      channels.forEach(function(c) { channelSet[c] = true; });
-
-      var curChannel = "";
-      var channelBooks = {};
-      var expectTitle = true;
-      var pendingTitle = "";
-
-      for (var i = 0; i < lines.length; i++) {
-        var line = lines[i];
-        if (/上榜天数记录|榜单说明/.test(line)) { break; }
-        if (/^(免费强推|vip强推|新晋作者|月榜|季榜|半年榜|长生殿|总分榜|字数榜|收入金榜|霸王票|霸王总榜|勤奋指数|完结金榜|新手金榜|栽培月榜|驻站|完结高分|千字金榜|完结全订榜)$/.test(line)) { continue; }
-        if (line.length > 30 && line.indexOf("·") > 0) continue;
-        if (channelSet[line]) {
-          if (curChannel && channelBooks[curChannel]) channelBooks[curChannel]._finished = true;
-          curChannel = line;
-          if (!channelBooks[curChannel]) channelBooks[curChannel] = { books: [] };
-          expectTitle = true;
-          pendingTitle = "";
-          continue;
-        }
-        if (!curChannel) continue;
-        if (expectTitle) {
-          pendingTitle = line;
-          expectTitle = false;
-        } else {
-          if (pendingTitle) {
-            channelBooks[curChannel].books.push({ title: pendingTitle, author: line, novelid: idMap[pendingTitle] || "" });
-          }
-          expectTitle = true;
-          pendingTitle = "";
-        }
-      }
-
-      for (var name in channelBooks) {
-        if (channelBooks[name].books.length > 0) {
-          result.channels.push({ name: name, books: channelBooks[name].books });
-        }
-      }
-      return result;
-    });
-
-    console.log(`  ✓ 布局检测：${data.layout === 'A' ? '表格型' : '交替行型'}`);
-
-    if (!data?.channels?.length) {
-      console.error(`[jjwxc] 采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开 ${url} 确认。`);
-      return null;
-    }
-  } finally {
-    await browser.close();
-  }
-
-  let totalBooks = 0;
-  data.channels.forEach(ch => {
-    totalBooks += ch.books.length;
-    const authors = new Set(ch.books.map(b => b.author));
-    if (ch.books.length >= 5 && authors.size / ch.books.length < 0.2) {
-      console.log(`  ⚠ ${ch.name}：${ch.books.length} 本只有 ${authors.size} 个唯一作者，可能提取有误`);
-    }
-  });
-  console.log(`  ✓ 列表：${data.channels.length} 个频道，共 ${totalBooks} 本`);
-
-  // 补采详情
-  let detailMap = {};
-  let detailPlanned = 0;
-  let detailOk = 0;
-  let detailFailedChunks = 0;
-  if (!LIST_ONLY) {
-    const picked = [];
-    for (const ch of data.channels) {
-      let n = 0;
-      for (const b of ch.books) {
-        if (picked.length >= DETAIL_LIMIT) break;
-        if (n >= TOP) break;
-        if (b.novelid) { picked.push(b.novelid); n++; }
-      }
-      if (picked.length >= DETAIL_LIMIT) break;
-    }
-    detailPlanned = picked.length;
-    if (picked.length) {
-      console.log(`  → 补采详情 ${picked.length} 本...`);
-      detailMap = await fetchDetails(picked);
-      detailFailedChunks = Object.values(detailMap).filter(d => d.err).length;
-      detailOk = Object.values(detailMap).filter(d => d && d.collect).length;
-      console.log(`  ✓ 详情命中收藏数 ${detailOk}/${picked.length}`);
-    }
-  }
-
-  let quality = "[OK]";
-  const detailPartial = !LIST_ONLY && detailPlanned > 0 && (detailFailedChunks > 0 || detailOk < detailPlanned);
-  if (!LIST_ONLY && detailPlanned > 0 && detailOk === 0) {
-    quality = "[详情解析异常/登录态缺失]";
-    console.error(`  ⚠ 详情全部无收藏数：可能页面结构变动或需登录。`);
-  } else if (detailPartial) {
-    quality = "[部分详情缺失]";
-    console.error(`  ⚠ 详情仅命中 ${detailOk}/${detailPlanned}，已按部分结果标注。`);
-  } else if (LIST_ONLY) {
-    quality = "[仅列表-无核心指标]";
-  }
-
-  const now = new Date().toISOString();
-  const lines = [
-    `# 晋江 · ${rt.label}`,
-    "",
-    `- 来源：${url}`,
-    `- 抓取时间：${now}`,
-    `- 条目数：${totalBooks}`,
-    "",
-    "---",
-    "",
-  ];
-
-  for (const ch of data.channels) {
-    for (let i = 0; i < ch.books.length; i++) {
-      const b = ch.books[i];
-      var genre = '';
-      if (data.layout === 'A' && b.type) {
-        var parts = b.type.split('-');
-        genre = parts[1] || '';
-      } else {
-        genre = ch.name;
-      }
-      lines.push(`书名：${b.title || ""}`);
-      lines.push(`题材：${genre}`);
-      lines.push(`作者：${b.author || ""}`);
-      if (b.novelid) lines.push(`作品页：https://www.jjwxc.net/onebook.php?novelid=${b.novelid}`);
-      lines.push("");
-    }
-  }
-
-  return {
-    content: lines.join("\n"),
-    partial: detailPartial,
-    partialReason: detailPartial ? `${rt.label}: detail ${detailOk}/${detailPlanned}` : "",
-  };
-}
-
-// ---------------------------------------------------------------------------
-// 入口
-// ---------------------------------------------------------------------------
-
-const args = process.argv.slice(2);
-const OUTDIR = getArg(args, "--outdir") || ".";
-const RANKTYPE = getArg(args, "--type") || "12";
-const CHANNEL = getArg(args, "--channel") || "0";
-const TOP = parseInt(getArg(args, "--top") || "10", 10);
-const DETAIL_LIMIT = parseInt(getArg(args, "--detail-limit") || "100", 10);
-const LIST_ONLY = args.includes("--list-only");
-const LOGIN_WAIT = parseInt(getArg(args, "--login-wait") || "0", 10);
-
-async function main() {
-  const rankTypeIds = (RANKTYPE || "12").split(",").map(t => t.trim());
-  const invalid = rankTypeIds.find(id => id !== "all" && !RANK_TYPES.some(r => r.id === id));
-  if (invalid) {
-    throw new Error(`未知 --type: ${invalid}`);
-  }
-  if (CHANNEL !== "0") {
-    throw new Error(`未知 --channel: ${CHANNEL}（当前仅支持 0=全站）`);
-  }
-  const rankTypes = RANKTYPE === "all" ? RANK_TYPES.map(r => r.id) : rankTypeIds;
-  let written = 0, failed = 0, partial = false;
-  const partialReasons = [];
-
-  for (const rt of rankTypes) {
-    try {
-      const result = await scrapeRank(rt, CHANNEL);
-      if (!result) { failed++; continue; }
-      if (result.partial) {
-        partial = true;
-        if (result.partialReason) partialReasons.push(result.partialReason);
-      }
-      const rtInfo = RANK_TYPES.find(r => r.id === rt);
-      const date = localDateStamp();
-      const filename = `晋江${rtInfo.label}_全站_${date}.md`;
-      fs.mkdirSync(OUTDIR, { recursive: true });
-      const filepath = path.join(OUTDIR, filename);
-      fs.writeFileSync(filepath, result.content, "utf-8");
-      written++;
-      console.log(`  ✓ 已保存: ${filepath}`);
-    } catch (rankErr) {
-      failed++;
-      const message = rankErr && rankErr.message ? rankErr.message : String(rankErr);
-      partialReasons.push(`${rt}: ${message}`);
-      console.error(`[jjwxc] ${rt} 采集失败，跳过: ${message}`);
-    }
-  }
-
-  return { planned: rankTypes.length, written, failed, partial: partial || failed > 0, partialReasons };
-}
-
-if (require.main === module) {
-  runCli(main, "晋江采集");
-}
-
-module.exports = { fmtWan };

package/plugins/fancy-scan/scripts/scrapers/package-lock.json

@@ -1,27 +0,0 @@
-{
-  "name": "fancy-scan-scrapers",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "fancy-scan-scrapers",
-      "version": "1.0.0",
-      "dependencies": {
-        "playwright-core": "^1.40.0"
-      }
-    },
-    "node_modules/playwright-core": {
-      "version": "1.62.1",
-      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
-      "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
-      "license": "Apache-2.0",
-      "bin": {
-        "playwright-core": "cli.js"
-      },
-      "engines": {
-        "node": ">=20"
-      }
-    }
-  }
-}