@lemoncat7/dsh-partner 2.8.0 → 2.8.2

package/resources/skills/story-creation/upstream/skills/story-long-scan/scripts/cdp-utils.js

@@ -0,0 +1,242 @@
+/**
+ * CDP 工具函数 — 各平台采集脚本的公共依赖
+ *
+ * 使用方式：
+ *   const { ab, sleep, evalJSON, evalJSONBase64, scrollLoad, getArg, safeStr, localDateStamp } = require("./cdp-utils");
+ *
+ * 前置：
+ *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
+ */
+
+const { execFileSync } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * On Windows `agent-browser` is an npm shim (agent-browser.cmd/.ps1) that
+ * forwards to the real target — the native agent-browser-win32-*.exe or a
+ * bundled Node CLI. Node refuses to execFile the `.cmd` without a shell
+ * (CVE-2024-27980), and routing the argv array through a shell mangles it: the
+ * `.cmd`'s `%*` is re-tokenized by cmd.exe (splitting on spaces, breaking on
+ * & | ^), and calling the shim by bare name from powershell.exe collapses the
+ * whole array into a single space-joined argument. The exact locus differs by
+ * runtime, so instead of hardening any one shell path we bypass shells entirely:
+ * read the `.cmd` shim, recover the real program plus its fixed leading args,
+ * and execFile that target directly with the argv array — verbatim, no shell.
+ */
+function resolveWindowsAgentBrowser(argv) {
+  const dirs = String(process.env.PATH || "").split(path.delimiter);
+  let cmdPath = null;
+  for (const dir of dirs) {
+    if (!dir) continue;
+    const candidate = path.join(dir, "agent-browser.cmd");
+    if (fs.existsSync(candidate)) {
+      cmdPath = candidate;
+      break;
+    }
+  }
+  if (!cmdPath) return { file: "agent-browser", args: argv };
+  const dir = path.dirname(cmdPath);
+  const forwardLine =
+    fs
+      .readFileSync(cmdPath, "utf8")
+      .split(/\r?\n/)
+      .find((line) => line.includes("%*")) || "";
+  const tokens = [...forwardLine.matchAll(/"([^"]*)"/g)]
+    .map((m) => m[1])
+    .map((t) =>
+      t
+        .replace(/%~dp0/gi, () => dir + path.sep)
+        .replace(/%dp0%/gi, () => dir + path.sep)
+    );
+  const jsIndex = tokens.findIndex((t) => /\.[cm]?js$/i.test(t));
+  if (jsIndex >= 0) {
+    return { file: process.execPath, args: [...tokens.slice(jsIndex), ...argv] };
+  }
+  if (tokens.length > 0) {
+    return { file: tokens[0], args: [...tokens.slice(1), ...argv] };
+  }
+  return { file: "agent-browser", args: argv };
+}
+
+/**
+ * Build a shell-free invocation. POSIX runs the native `agent-browser` binary
+ * directly; Windows resolves the npm `.cmd` shim to that native target so the
+ * argument array is passed verbatim, never routed through cmd.exe/PowerShell.
+ */
+function buildAgentBrowserInvocation(port, args, platform = process.platform) {
+  const argv = ["--cdp", String(port), ...args.map(String)];
+  if (platform !== "win32") {
+    return { file: "agent-browser", args: argv };
+  }
+  return resolveWindowsAgentBrowser(argv);
+}
+
+// ---------------------------------------------------------------------------
+// agent-browser 工具函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 调用 agent-browser CLI
+ * @param {number} port - CDP 端口
+ * @param  {...string} args - agent-browser 参数
+ * @returns {string} stdout（trim 后）
+ */
+function ab(port, ...args) {
+  const invocation = buildAgentBrowserInvocation(port, args);
+  try {
+    return execFileSync(
+      invocation.file,
+      invocation.args,
+      {
+        encoding: "utf-8",
+        timeout: 20000,
+        stdio: ["pipe", "pipe", "pipe"],
+        windowsHide: true,
+      }
+    ).trim();
+  } catch (error) {
+    const stderr = error && error.stderr ? String(error.stderr).trim() : "";
+    const stdout = error && error.stdout ? String(error.stdout).trim() : "";
+    const detail = stderr || stdout || (error && error.message) || "unknown error";
+    throw new Error(`agent-browser failed: ${detail}`, { cause: error });
+  }
+}
+
+/** 等待 ms 毫秒（跨平台，不依赖系统 sleep 命令） */
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function parseJSONResult(raw) {
+  if (!raw || raw === "ERR") {
+    throw new Error("agent-browser returned no JSON result");
+  }
+  try {
+    let parsed = JSON.parse(raw);
+    if (typeof parsed === "string") {
+      try { parsed = JSON.parse(parsed); } catch {}
+    }
+    return parsed;
+  } catch (error) {
+    throw new Error(`agent-browser returned invalid JSON: ${String(raw).slice(0, 160)}`, {
+      cause: error,
+    });
+  }
+}
+
+/**
+ * 在浏览器内执行 JS，并解析 JSON 返回值。
+ * 一律走 base64（-b）：正文提取用的 JS 常含引号、反斜杠等，作为命令行参数时在 Windows 上
+ * 无法逐字透传（.cmd 的 %* 与 PowerShell 都会二次解析）。base64 让参数只含 [A-Za-z0-9+/=]，
+ * 和各采集脚本已在用的 evalJSONBase64 走同一条安全通道。
+ */
+function evalJSON(port, js) {
+  return evalJSONBase64(port, js);
+}
+
+/**
+ * 通过 agent-browser 的 base64 参数执行复杂 JS，避免命令行转义和参数边界问题。
+ */
+function evalJSONBase64(port, js) {
+  const encoded = Buffer.from(String(js), "utf8").toString("base64");
+  return parseJSONResult(ab(port, "eval", "-b", encoded));
+}
+
+/**
+ * 安全地将值插入浏览器 eval 字符串。
+ * 使用 JSON.stringify 确保值不会因特殊字符（引号、反斜杠等）破坏 eval 字符串。
+ * @param {*} val - 要插入的值
+ * @returns {string} JSON 字符串表示（含引号）
+ */
+function safeStr(val) {
+  return JSON.stringify(String(val));
+}
+
+/**
+ * 滚动页面加载更多内容
+ * @param {number} port - CDP 端口
+ * @param {number} times - 滚动次数
+ * @param {number} [interval=1000] - 每次滚动间隔（ms）
+ */
+function scrollLoad(port, times, interval = 1000) {
+  for (let i = 0; i < times; i++) {
+    ab(port, "eval", "window.scrollBy(0, window.innerHeight)");
+    sleep(interval);
+  }
+}
+
+/** 解析 --xxx 参数 */
+function getArg(args, name) {
+  const i = args.indexOf(name);
+  if (i >= 0) return i + 1 < args.length ? args[i + 1] : null;
+  const prefix = `${name}=`;
+  const inline = args.find((arg) => String(arg).startsWith(prefix));
+  return inline === undefined ? null : String(inline).slice(prefix.length);
+}
+
+/**
+ * 输出文件名用的日期戳（YYYYMMDD），一律取**本地日历日**。
+ * 不能用 new Date().toISOString().slice(0,10)：那是 UTC 日期，比 UTC+8 晚 8 小时。
+ * 文件名是各采集脚本唯一的去重键（一个榜单一天一份），北京时间 00:00-08:00 之间的采集
+ * 会退回「昨天」的文件名，静默覆盖前一晚采到的同名报告，且这份数据被标成前一天。
+ * @param {Date} [date] - 默认当前时间
+ * @returns {string} YYYYMMDD
+ */
+function localDateStamp(date) {
+  const d = date instanceof Date ? date : new Date();
+  const y = String(d.getFullYear()).padStart(4, "0");
+  const m = String(d.getMonth() + 1).padStart(2, "0");
+  const day = String(d.getDate()).padStart(2, "0");
+  return `${y}${m}${day}`;
+}
+
+/**
+ * Run a scraper entrypoint and turn empty/partial output into machine-readable
+ * CLI status. Legacy entrypoints may return an integer; multi-target scrapers
+ * return {planned,written,failed,partial,partialReasons}.
+ */
+function runCli(main, label) {
+  Promise.resolve()
+    .then(main)
+    .then((result) => {
+      const outcome = Number.isInteger(result)
+        ? { planned: result, written: result, failed: 0, partial: false, partialReasons: [] }
+        : result;
+      if (!outcome || !Number.isInteger(outcome.written) || outcome.written < 1) {
+        throw new Error("no output was written");
+      }
+      const failed = Number.isInteger(outcome.failed) ? outcome.failed : 0;
+      const planned = Number.isInteger(outcome.planned)
+        ? outcome.planned
+        : outcome.written + failed;
+      const reasons = Array.isArray(outcome.partialReasons)
+        ? outcome.partialReasons.filter(Boolean).map(String)
+        : [];
+      if (outcome.partial || failed > 0) {
+        const details = [`wrote ${outcome.written}/${planned}`];
+        if (failed > 0) details.push(`failed ${failed}`);
+        details.push(...reasons);
+        console.error(`${label} partial: ${details.join("; ")}`);
+        process.exitCode = 2;
+      }
+    })
+    .catch((error) => {
+      const message = error && error.message ? error.message : String(error);
+      console.error(`${label} failed: ${message}`);
+      process.exitCode = 1;
+    });
+}
+
+module.exports = {
+  ab,
+  sleep,
+  evalJSON,
+  evalJSONBase64,
+  buildAgentBrowserInvocation,
+  safeStr,
+  scrollLoad,
+  getArg,
+  localDateStamp,
+  runCli,
+};

package/resources/skills/story-creation/upstream/skills/story-long-scan/scripts/ciweimao-rank-scraper.js

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+/**
+ * 刺猬猫阅读排行榜采集脚本
+ *
+ * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
+ * 采集策略：刺猬猫 rank-index 页面单页展示所有榜单，文本解析提取结构化数据。
+ * 输出 Markdown 格式见 references/platform-ciweimao.md；aggregate-rank.js 按此格式聚合。
+ *
+ * 用法：
+ *   node ciweimao-rank-scraper.js --type click       # 点击榜
+ *   node ciweimao-rank-scraper.js --type monthly      # 月票榜
+ *   node ciweimao-rank-scraper.js --type all           # 全部榜单
+ *
+ * 前置：
+ *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { ab, sleep, evalJSONBase64, scrollLoad, getArg, localDateStamp, runCli } = require("./cdp-utils");
+
+const RANK_URL = "https://www.ciweimao.com/rank-index";
+
+/** 连通性 + 页面就绪自检 */
+function probePage(port) {
+  return evalJSONBase64(
+    port,
+    "JSON.stringify({host:location.host,len:(document.body&&document.body.innerText||'').length})"
+  );
+}
+
+const RANK_TYPES = [
+  { id: "click", label: "点击榜", header: "点击榜" },
+  { id: "favor", label: "收藏榜", header: "收藏榜" },
+  { id: "recommend", label: "推荐榜", header: "推荐榜" },
+  { id: "subscribe", label: "订阅榜", header: "订阅榜" },
+  { id: "monthly", label: "月票榜", header: "月票榜" },
+  { id: "tsukkomi", label: "吐槽榜", header: "吐槽榜" },
+  { id: "newbook", label: "新书榜", header: "新书榜" },
+  { id: "blade", label: "刀片榜", header: "刀片榜" },
+  { id: "update", label: "更新榜", header: "更新榜" },
+];
+
+// ---------------------------------------------------------------------------
+// 页面提取
+// ---------------------------------------------------------------------------
+
+/**
+ * 从 rank-index 单页解析所有榜单。
+ * 页面结构：每个榜单有标题行（如"点击榜"），后跟 NO.1 特殊条目 + #2-10 普通条目。
+ * NO.1 格式：标题 / 作者 / 指标值（三行）
+ * #2-10 格式：N[题材]书名 / 指标值（两行）
+ */
+function extractAllRanks(port) {
+  const js =
+    "JSON.stringify((()=>{" +
+    "var text=document.body.innerText||'';" +
+    "var lines=text.split(/\\n/).map(function(l){return l.trim()}).filter(Boolean);" +
+    "var headers=['点击榜','收藏榜','推荐榜','订阅榜','月票榜','吐槽榜','新书榜','刀片榜','更新榜'];" +
+    "var sections=[];var curName='';var curEntries=[];" +
+    "for(var i=0;i<lines.length;i++){" +
+    "  var line=lines[i];" +
+    // 检测新 section
+    "  var headerIdx=headers.indexOf(line);" +
+    "  if(headerIdx>=0){" +
+    "    if(curName&&curEntries.length)sections.push({name:curName,entries:curEntries});" +
+    "    curName=headers[headerIdx];curEntries=[];continue" +
+    "  }" +
+    "  if(!curName)continue;" +
+    // 跳过周期 tab 和 UI 文字
+    "  if(/^(周榜|月榜|总榜)$/.test(line))continue;" +
+    // NO.1 条目
+    "  if(line==='NO.1'&&i+3<lines.length){" +
+    "    var t=lines[i+1]||'';var a=lines[i+2]||'';var v=lines[i+3]||'';" +
+    "    if(headers.indexOf(v)>=0)continue;" +
+    "    curEntries.push({rank:1,title:t,author:a,genre:'',metric:v});" +
+    "    i+=2;continue" +
+    "  }" +
+    // #2-10 条目：N[题材]书名
+    "  var rm=line.match(/^(\\d{1,2})\\[(.+?)\\](.+)$/);" +
+    "  if(rm){" +
+    "    var nextVal=i+1<lines.length?lines[i+1]:'';" +
+    "    var metric='';" +
+    "    if(/^[\\d.]+(万)?$/.test(nextVal)){metric=nextVal;i++}" +
+    "    curEntries.push({rank:parseInt(rm[1]),title:rm[3],author:'',genre:rm[2],metric:metric});" +
+    "    continue" +
+    "  }" +
+    "}" +
+    "if(curName&&curEntries.length)sections.push({name:curName,entries:curEntries});" +
+    "return sections" +
+    "})())";
+  return evalJSONBase64(port, js) || [];
+}
+
+/**
+ * 从 DOM 获取书籍链接。每本书常有封面图 anchor（textContent 为空）和书名 anchor，
+ * 按 bookId 聚合后取最长的非空文本作为书名，避免空封面 anchor 覆盖书名导致回填全失败。
+ */
+function extractBookUrls(port) {
+  const js = `JSON.stringify((function(){
+    function clean(t){return t.replace(/^[0-9]+\\[[^\\]]*\\]/,'').replace(/\\s+[0-9.]+(?:万|亿)?$/,'').trim();}
+    var byId={};var order=[];
+    Array.from(document.querySelectorAll('a[href*="/book/"]')).forEach(function(a){
+      var h=a.getAttribute('href')||a.href||'';
+      var m=h.match(/\\/book\\/([0-9]+)/);
+      if(!m)return; var id=m[1];
+      var t=clean((a.innerText||a.textContent||'').replace(/\\s+/g,' ').trim());
+      if(!byId[id]){byId[id]='';order.push(id);}
+      if(t&&t.length>byId[id].length)byId[id]=t;
+    });
+    return order.map(function(id){return {bookId:id,title:byId[id],url:'https://www.ciweimao.com/book/'+id};});
+  })())`;
+  return evalJSONBase64(port, js) || [];
+}
+
+// ---------------------------------------------------------------------------
+// 主流程
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+const PORT = parseInt(getArg(args, "--port") || "9222", 10);
+const OUTDIR = getArg(args, "--outdir") || ".";
+const RANKTYPE = getArg(args, "--type") || "all";
+
+function main() {
+  console.log("\n→ 采集 刺猬猫排行榜...");
+  console.log(`  URL: ${RANK_URL}`);
+
+  let sections, urls;
+  try {
+    ab(PORT, "open", RANK_URL);
+    sleep(4000);
+
+    // 连通性自检：CDP 未起/被重定向时给可操作报错，而非误报"结构已变"
+    const probe = probePage(PORT);
+    if (!probe) {
+      console.error(
+        `  ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome（端口 ${PORT}），且 agent-browser 可用。`
+      );
+      return 0;
+    }
+    if (probe.host && probe.host.indexOf("ciweimao") === -1) {
+      console.error(`  ✗ 当前页面非刺猬猫（host=${probe.host}），可能被重定向，已跳过。`);
+      return 0;
+    }
+
+    scrollLoad(PORT, 3);
+    sleep(1000);
+
+    sections = extractAllRanks(PORT);
+    if (!sections.length) {
+      // 懒加载可能未触发，再滚动重试一次
+      scrollLoad(PORT, 2);
+      sleep(1000);
+      sections = extractAllRanks(PORT);
+    }
+    if (!sections.length) {
+      console.error("[ciweimao] 采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开榜单页确认。");
+      return 0;
+    }
+
+    urls = extractBookUrls(PORT);
+  } catch (err) {
+    console.error(`[ciweimao] 采集失败（页面加载或提取阶段）: ${err.message}`);
+    return 0;
+  }
+
+  console.log(`  ✓ 提取 ${sections.length} 个榜单，${urls.length} 个书籍链接`);
+
+  // 筛选需要的榜单类型
+  const targetTypes =
+    RANKTYPE === "all"
+      ? RANK_TYPES
+      : RANK_TYPES.filter((r) => r.id === RANKTYPE);
+
+  let written = 0;
+  for (const rt of targetTypes) {
+    try {
+      const section = sections.find((s) => s.name === rt.header);
+      if (!section || !section.entries.length) {
+        console.log(`  ⚠ ${rt.label} 无数据，跳过`);
+        continue;
+      }
+
+      const now = new Date().toISOString();
+      const norm = (s) => (s || "").replace(/\s+/g, "");
+      const linked = section.entries.filter((e) =>
+        urls.some((u) => norm(u.title) === norm(e.title))
+      ).length;
+      const lines = [
+        `# 刺猬猫 · ${rt.label}`,
+        "",
+        `- 来源：${RANK_URL}`,
+        `- 抓取时间：${now}`,
+        `- 条目数：${section.entries.length}`,
+        `- 作品页链接：${linked} / ${section.entries.length}`,
+        "",
+        "---",
+        "",
+      ];
+
+      for (const entry of section.entries) {
+        try {
+          lines.push(`### #${entry.rank} ${entry.title}`);
+          const meta = [
+            entry.author,
+            entry.genre,
+            entry.metric || "",
+          ].filter(Boolean).join(" · ");
+          if (meta) lines.push(`*${meta}*`);
+
+          // 按标题匹配书籍链接（归一后比对）
+          const matched = urls.find((u) => norm(u.title) === norm(entry.title));
+          if (matched) {
+            lines.push(`[作品页](${matched.url})`);
+          }
+
+          lines.push("", "---", "");
+        } catch (entryErr) {
+          console.error(`[ciweimao] ${rt.label} 条目处理出错（#${entry.rank} ${entry.title}）: ${entryErr.message}`);
+          lines.push("", "---", "");
+        }
+      }
+
+      const filename = `刺猬猫${rt.label}_${localDateStamp()}.md`;
+      fs.mkdirSync(OUTDIR, { recursive: true });
+      const filepath = path.join(OUTDIR, filename);
+      fs.writeFileSync(filepath, lines.join("\n"), "utf-8");
+      written++;
+      console.log(`  ✓ ${rt.label}：${section.entries.length} 条 → ${filepath}`);
+    } catch (rankErr) {
+      console.error(`[ciweimao] ${rt.label} 处理出错，跳过: ${rankErr.message}`);
+    }
+  }
+  return written;
+}
+
+if (require.main === module) {
+  runCli(main, "刺猬猫采集");
+}