@cloud411716/fancy-webnovel 0.1.44 → 0.1.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloud411716/fancy-webnovel",
-  "version": "0.1.44",
+  "version": "0.1.46",
   "type": "module",
   "main": "index.js",
   "exports": {

package/plugins/fancy-scan/scripts/run-scan.js

@@ -13,7 +13,8 @@
  */
 
 import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
-import { join } from 'path';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
 
 // ---------------------------------------------------------------------------
 // 平台配置
@@ -168,104 +169,74 @@ function renderQidianMarkdown(rankLabel, url, books) {
 }
 
 // ---------------------------------------------------------------------------
-// 占位符采集 — 浏览器平台
+// Browser scraper spawner（调用 scripts/scrapers/ 下的 CJS 脚本）
 // ---------------------------------------------------------------------------
 
-function scrapeBrowserStub(platform, length, outDir) {
-  // 这些平台需要 browser_* 工具，由 LLM 调用
-  // 这里生成指令文件，供 LLM 读取执行
-  const today = todayStr();
-  const pcn = PLATFORM_CN[platform] || platform;
-  const lenStr = length === 'long' ? '长篇' : '短篇';
-  const instrFile = join(outDir, `_browser_instructions_${platform}_${length}_${today}.json`);
-  const instructions = getBrowserInstructions(platform, length);
-
-  writeFileSync(instrFile, JSON.stringify(instructions, null, 2), 'utf-8');
-  console.error(`  ℹ ${pcn}（${lenStr}）需要 browser_* 工具，已生成指令文件：${instrFile}`);
-  console.error(`  ℹ 请让 LLM 读取并执行该指令文件中的 browser 操作`);
-  return { file: instrFile, instructions };
+function spawnScraper(platform, length, outDir) {
+  return new Promise(async (resolve) => {
+    const { spawn } = await import('child_process');
+    const args = scraperArgs(platform, length, outDir);
+    // 先尝试 9222 端口（Chrome CDP），没有 Chrome 则尝试 9223
+    const ports = ['9222', '9223'];
+    let portIdx = 0;
+
+    function tryPort(port) {
+      const scraperScript = join(dirname(fileURLToPath(import.meta.url)), 'scrapers', args.script);
+      const fullArgs = ['--port', port, ...args.extra];
+      const p = spawn('node', [scraperScript, ...fullArgs], {
+        timeout: 120000,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+      let out = '', err = '';
+      p.stdout.on('data', d => { out += d.toString(); });
+      p.stderr.on('data', d => { err += d.toString(); });
+      p.on('close', (code) => {
+        if (code === 0 && out.trim()) {
+          resolve({ ok: true, output: out, stderr: err });
+        } else if (portIdx < ports.length - 1) {
+          portIdx++;
+          tryPort(ports[portIdx]);
+        } else {
+          resolve({ ok: false, output: out, error: err || `exit ${code}` });
+        }
+      });
+      p.on('error', (e) => {
+        if (portIdx < ports.length - 1) { portIdx++; tryPort(ports[portIdx]); }
+        else resolve({ ok: false, output: '', error: e.message });
+      });
+    }
+    tryPort(ports[portIdx]);
+  });
 }
 
-function getBrowserInstructions(platform, length) {
-  // 返回给 LLM 的操作指令
-  const today = todayStr();
-  const pcn = PLATFORM_CN[platform] || platform;
-  const lenStr = length === 'long' ? '长篇' : '短篇';
-
-  if (platform === 'fanqie') {
-    return {
-      platform, length, pcn, lenStr, date: today,
-      action: 'scrape_fanqie',
-      steps: [
-        { tool: 'browser_navigate', url: 'https://fanqienovel.com/rankings?channel=1&type=2', label: '番茄男频阅读榜' },
-        { tool: 'sleep', seconds: 3 },
-        { tool: 'scroll', times: 3 },
-        { tool: 'extract', description: '提取页面书籍列表：排名、书名、作者、在读数、题材' },
-      ],
-      outputFile: `扫榜结果/番茄男频阅读榜_${today}.md`,
-      format: '见 scan-output-format.md',
-    };
-  }
-
-  if (platform === 'jinjiang') {
-    return {
-      platform, length, pcn, lenStr, date: today,
-      action: 'scrape_jinjiang',
-      steps: [
-        { tool: 'browser_navigate', url: 'https://www.jjwxc.net/topten.php?orderstr=12&t=0', label: '晋江金榜' },
-        { tool: 'sleep', seconds: 3 },
-        { tool: 'extract', description: '提取页面书籍列表：排名、书名、作者、收藏数' },
-      ],
-      outputFile: `扫榜结果/晋江金榜_${today}.md`,
-      format: '见 scan-output-format.md',
-    };
-  }
-
-  if (platform === 'zhihu') {
-    const subType = length === 'short' ? '短篇4榜' : '长篇榜';
-    return {
-      platform, length, pcn, lenStr, date: today,
-      action: 'scrape_zhihu',
-      steps: [
-        { tool: 'browser_navigate', url: 'https://www.zhihu.com/fiore/h5/vip-web', label: `知乎${subType}` },
-        { tool: 'sleep', seconds: 3 },
-        { tool: 'extract', description: '提取页面书籍列表' },
-      ],
-      outputFile: `扫榜结果/知乎${subType}_${today}.md`,
-      format: '见 scan-output-format.md',
-    };
+function scraperArgs(platform, length) {
+  switch (platform) {
+    case 'fanqie':
+      return {
+        script: 'fanqie-rank-scraper.js',
+        extra: ['--channel', 'all', '--top', '20'],
+      };
+    case 'jinjiang':
+      return {
+        script: 'jjwxc-rank-scraper.js',
+        extra: ['--type', '12'],
+      };
+    case 'zhihu':
+      return {
+        script: 'zhihu-rank-scraper.js',
+        extra: ['--length', length === 'short' ? 'short' : 'long'],
+      };
+    case 'qimao':
+      return {
+        script: 'qimao-rank-scraper.js',
+        extra: ['--channel', length === 'short' ? 'female' : 'male', '--type', 'hot'],
+      };
+    case 'dianzhong':
+      // 点众没有 scraper，降级为 browser stub
+      return { script: null, extra: [] };
+    default:
+      return { script: null, extra: [] };
   }
-
-  if (platform === 'dianzhong') {
-    return {
-      platform, length, pcn, lenStr, date: today,
-      action: 'scrape_dianzhong',
-      steps: [
-        { tool: 'browser_navigate', url: 'https://www.tiyi.cn/', label: '点众短篇榜' },
-        { tool: 'sleep', seconds: 3 },
-        { tool: 'extract', description: '提取页面书籍列表' },
-      ],
-      outputFile: `扫榜结果/点众短篇榜_${today}.md`,
-      format: '见 scan-output-format.md',
-    };
-  }
-
-  if (platform === 'qimao') {
-    const gender = length === 'long' ? '男频' : '女频';
-    return {
-      platform, length, pcn, lenStr, date: today,
-      action: 'scrape_qimao',
-      steps: [
-        { tool: 'browser_navigate', url: 'https://www.qimao.com/paihang', label: `七猫${gender}大热榜` },
-        { tool: 'sleep', seconds: 3 },
-        { tool: 'extract', description: '提取页面书籍列表：排名、书名、作者、热度' },
-      ],
-      outputFile: `扫榜结果/七猫${gender}大热榜_${today}.md`,
-      format: '见 scan-output-format.md',
-    };
-  }
-
-  return { error: `未知平台: ${platform}` };
 }
 
 // ---------------------------------------------------------------------------
@@ -414,16 +385,30 @@ async function main() {
     }
     scanFiles = scrapeFiles.map(f => f.file);
   } else {
-    // 其他平台：browser_* 工具（LLM 读取指令文件后执行）
-    console.error(`→ ${PLATFORM_CN[platform]}（browser_* 工具）...`);
-    const { file, instructions } = scrapeBrowserStub(platform, length, scanDir);
-    scrapeFiles.push({ file });
-    scanFiles.push(file);
-
-    // 输出指令供 LLM 读取
-    console.error('\n=== 浏览器采集指令 ===');
-    console.error(JSON.stringify(instructions, null, 2));
-    console.error('=== 浏览器采集指令 END ===\n');
+    // 其他平台：spawn scraper 进程
+    const cfg = scraperArgs(platform, length);
+    if (!cfg.script) {
+      console.error(`  ❌ 平台 ${platform} 暂不支持采集`);
+      process.exit(1);
+    }
+    console.error(`→ 采集 ${PLATFORM_CN[platform]}（Chrome CDP）...`);
+    const result = await spawnScraper(platform, length, scanDir);
+    if (!result.ok) {
+      console.error(`  ❌ ${PLATFORM_CN[platform]} 采集失败：${result.error}`);
+      process.exit(1);
+    }
+    // scraper 输出落在 stdout（JSON 摘要），找到它
+    const lines = result.output.split('\n');
+    const writtenFiles = [];
+    for (const line of lines) {
+      const m = line.match(/已保存:\s*(.+)/);
+      if (m) writtenFiles.push(m[1].trim());
+    }
+    if (!writtenFiles.length) {
+      console.error('  ❌ 未找到任何输出文件');
+      process.exit(1);
+    }
+    scanFiles.push(...writtenFiles);
   }
 
   // Step 6: 收集原始数据

package/plugins/fancy-scan/scripts/scrapers/cdp-utils.js

@@ -0,0 +1,242 @@
+/**
+ * CDP 工具函数 — 各平台采集脚本的公共依赖
+ *
+ * 使用方式：
+ *   const { ab, sleep, evalJSON, evalJSONBase64, scrollLoad, getArg, safeStr, localDateStamp } = require("./cdp-utils");
+ *
+ * 前置：
+ *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
+ */
+
+const { execFileSync } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * On Windows `agent-browser` is an npm shim (agent-browser.cmd/.ps1) that
+ * forwards to the real target — the native agent-browser-win32-*.exe or a
+ * bundled Node CLI. Node refuses to execFile the `.cmd` without a shell
+ * (CVE-2024-27980), and routing the argv array through a shell mangles it: the
+ * `.cmd`'s `%*` is re-tokenized by cmd.exe (splitting on spaces, breaking on
+ * & | ^), and calling the shim by bare name from powershell.exe collapses the
+ * whole array into a single space-joined argument. The exact locus differs by
+ * runtime, so instead of hardening any one shell path we bypass shells entirely:
+ * read the `.cmd` shim, recover the real program plus its fixed leading args,
+ * and execFile that target directly with the argv array — verbatim, no shell.
+ */
+function resolveWindowsAgentBrowser(argv) {
+  const dirs = String(process.env.PATH || "").split(path.delimiter);
+  let cmdPath = null;
+  for (const dir of dirs) {
+    if (!dir) continue;
+    const candidate = path.join(dir, "agent-browser.cmd");
+    if (fs.existsSync(candidate)) {
+      cmdPath = candidate;
+      break;
+    }
+  }
+  if (!cmdPath) return { file: "agent-browser", args: argv };
+  const dir = path.dirname(cmdPath);
+  const forwardLine =
+    fs
+      .readFileSync(cmdPath, "utf8")
+      .split(/\r?\n/)
+      .find((line) => line.includes("%*")) || "";
+  const tokens = [...forwardLine.matchAll(/"([^"]*)"/g)]
+    .map((m) => m[1])
+    .map((t) =>
+      t
+        .replace(/%~dp0/gi, () => dir + path.sep)
+        .replace(/%dp0%/gi, () => dir + path.sep)
+    );
+  const jsIndex = tokens.findIndex((t) => /\.[cm]?js$/i.test(t));
+  if (jsIndex >= 0) {
+    return { file: process.execPath, args: [...tokens.slice(jsIndex), ...argv] };
+  }
+  if (tokens.length > 0) {
+    return { file: tokens[0], args: [...tokens.slice(1), ...argv] };
+  }
+  return { file: "agent-browser", args: argv };
+}
+
+/**
+ * Build a shell-free invocation. POSIX runs the native `agent-browser` binary
+ * directly; Windows resolves the npm `.cmd` shim to that native target so the
+ * argument array is passed verbatim, never routed through cmd.exe/PowerShell.
+ */
+function buildAgentBrowserInvocation(port, args, platform = process.platform) {
+  const argv = ["--cdp", String(port), ...args.map(String)];
+  if (platform !== "win32") {
+    return { file: "agent-browser", args: argv };
+  }
+  return resolveWindowsAgentBrowser(argv);
+}
+
+// ---------------------------------------------------------------------------
+// agent-browser 工具函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 调用 agent-browser CLI
+ * @param {number} port - CDP 端口
+ * @param  {...string} args - agent-browser 参数
+ * @returns {string} stdout（trim 后）
+ */
+function ab(port, ...args) {
+  const invocation = buildAgentBrowserInvocation(port, args);
+  try {
+    return execFileSync(
+      invocation.file,
+      invocation.args,
+      {
+        encoding: "utf-8",
+        timeout: 20000,
+        stdio: ["pipe", "pipe", "pipe"],
+        windowsHide: true,
+      }
+    ).trim();
+  } catch (error) {
+    const stderr = error && error.stderr ? String(error.stderr).trim() : "";
+    const stdout = error && error.stdout ? String(error.stdout).trim() : "";
+    const detail = stderr || stdout || (error && error.message) || "unknown error";
+    throw new Error(`agent-browser failed: ${detail}`, { cause: error });
+  }
+}
+
+/** 等待 ms 毫秒（跨平台，不依赖系统 sleep 命令） */
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function parseJSONResult(raw) {
+  if (!raw || raw === "ERR") {
+    throw new Error("agent-browser returned no JSON result");
+  }
+  try {
+    let parsed = JSON.parse(raw);
+    if (typeof parsed === "string") {
+      try { parsed = JSON.parse(parsed); } catch {}
+    }
+    return parsed;
+  } catch (error) {
+    throw new Error(`agent-browser returned invalid JSON: ${String(raw).slice(0, 160)}`, {
+      cause: error,
+    });
+  }
+}
+
+/**
+ * 在浏览器内执行 JS，并解析 JSON 返回值。
+ * 一律走 base64（-b）：正文提取用的 JS 常含引号、反斜杠等，作为命令行参数时在 Windows 上
+ * 无法逐字透传（.cmd 的 %* 与 PowerShell 都会二次解析）。base64 让参数只含 [A-Za-z0-9+/=]，
+ * 和各采集脚本已在用的 evalJSONBase64 走同一条安全通道。
+ */
+function evalJSON(port, js) {
+  return evalJSONBase64(port, js);
+}
+
+/**
+ * 通过 agent-browser 的 base64 参数执行复杂 JS，避免命令行转义和参数边界问题。
+ */
+function evalJSONBase64(port, js) {
+  const encoded = Buffer.from(String(js), "utf8").toString("base64");
+  return parseJSONResult(ab(port, "eval", "-b", encoded));
+}
+
+/**
+ * 安全地将值插入浏览器 eval 字符串。
+ * 使用 JSON.stringify 确保值不会因特殊字符（引号、反斜杠等）破坏 eval 字符串。
+ * @param {*} val - 要插入的值
+ * @returns {string} JSON 字符串表示（含引号）
+ */
+function safeStr(val) {
+  return JSON.stringify(String(val));
+}
+
+/**
+ * 滚动页面加载更多内容
+ * @param {number} port - CDP 端口
+ * @param {number} times - 滚动次数
+ * @param {number} [interval=1000] - 每次滚动间隔（ms）
+ */
+function scrollLoad(port, times, interval = 1000) {
+  for (let i = 0; i < times; i++) {
+    ab(port, "eval", "window.scrollBy(0, window.innerHeight)");
+    sleep(interval);
+  }
+}
+
+/** 解析 --xxx 参数 */
+function getArg(args, name) {
+  const i = args.indexOf(name);
+  if (i >= 0) return i + 1 < args.length ? args[i + 1] : null;
+  const prefix = `${name}=`;
+  const inline = args.find((arg) => String(arg).startsWith(prefix));
+  return inline === undefined ? null : String(inline).slice(prefix.length);
+}
+
+/**
+ * 输出文件名用的日期戳（YYYYMMDD），一律取**本地日历日**。
+ * 不能用 new Date().toISOString().slice(0,10)：那是 UTC 日期，比 UTC+8 晚 8 小时。
+ * 文件名是各采集脚本唯一的去重键（一个榜单一天一份），北京时间 00:00-08:00 之间的采集
+ * 会退回「昨天」的文件名，静默覆盖前一晚采到的同名报告，且这份数据被标成前一天。
+ * @param {Date} [date] - 默认当前时间
+ * @returns {string} YYYYMMDD
+ */
+function localDateStamp(date) {
+  const d = date instanceof Date ? date : new Date();
+  const y = String(d.getFullYear()).padStart(4, "0");
+  const m = String(d.getMonth() + 1).padStart(2, "0");
+  const day = String(d.getDate()).padStart(2, "0");
+  return `${y}${m}${day}`;
+}
+
+/**
+ * Run a scraper entrypoint and turn empty/partial output into machine-readable
+ * CLI status. Legacy entrypoints may return an integer; multi-target scrapers
+ * return {planned,written,failed,partial,partialReasons}.
+ */
+function runCli(main, label) {
+  Promise.resolve()
+    .then(main)
+    .then((result) => {
+      const outcome = Number.isInteger(result)
+        ? { planned: result, written: result, failed: 0, partial: false, partialReasons: [] }
+        : result;
+      if (!outcome || !Number.isInteger(outcome.written) || outcome.written < 1) {
+        throw new Error("no output was written");
+      }
+      const failed = Number.isInteger(outcome.failed) ? outcome.failed : 0;
+      const planned = Number.isInteger(outcome.planned)
+        ? outcome.planned
+        : outcome.written + failed;
+      const reasons = Array.isArray(outcome.partialReasons)
+        ? outcome.partialReasons.filter(Boolean).map(String)
+        : [];
+      if (outcome.partial || failed > 0) {
+        const details = [`wrote ${outcome.written}/${planned}`];
+        if (failed > 0) details.push(`failed ${failed}`);
+        details.push(...reasons);
+        console.error(`${label} partial: ${details.join("; ")}`);
+        process.exitCode = 2;
+      }
+    })
+    .catch((error) => {
+      const message = error && error.message ? error.message : String(error);
+      console.error(`${label} failed: ${message}`);
+      process.exitCode = 1;
+    });
+}
+
+module.exports = {
+  ab,
+  sleep,
+  evalJSON,
+  evalJSONBase64,
+  buildAgentBrowserInvocation,
+  safeStr,
+  scrollLoad,
+  getArg,
+  localDateStamp,
+  runCli,
+};