@lemoncat7/dsh-partner 2.8.1 → 2.8.2

package/resources/skills/story-creation/upstream/skills/story-short-scan/scripts/cdp-utils.js

@@ -0,0 +1,242 @@
+/**
+ * CDP 工具函数 — 各平台采集脚本的公共依赖
+ *
+ * 使用方式：
+ *   const { ab, sleep, evalJSON, evalJSONBase64, scrollLoad, getArg, safeStr, localDateStamp } = require("./cdp-utils");
+ *
+ * 前置：
+ *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
+ */
+
+const { execFileSync } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * On Windows `agent-browser` is an npm shim (agent-browser.cmd/.ps1) that
+ * forwards to the real target — the native agent-browser-win32-*.exe or a
+ * bundled Node CLI. Node refuses to execFile the `.cmd` without a shell
+ * (CVE-2024-27980), and routing the argv array through a shell mangles it: the
+ * `.cmd`'s `%*` is re-tokenized by cmd.exe (splitting on spaces, breaking on
+ * & | ^), and calling the shim by bare name from powershell.exe collapses the
+ * whole array into a single space-joined argument. The exact locus differs by
+ * runtime, so instead of hardening any one shell path we bypass shells entirely:
+ * read the `.cmd` shim, recover the real program plus its fixed leading args,
+ * and execFile that target directly with the argv array — verbatim, no shell.
+ */
+function resolveWindowsAgentBrowser(argv) {
+  const dirs = String(process.env.PATH || "").split(path.delimiter);
+  let cmdPath = null;
+  for (const dir of dirs) {
+    if (!dir) continue;
+    const candidate = path.join(dir, "agent-browser.cmd");
+    if (fs.existsSync(candidate)) {
+      cmdPath = candidate;
+      break;
+    }
+  }
+  if (!cmdPath) return { file: "agent-browser", args: argv };
+  const dir = path.dirname(cmdPath);
+  const forwardLine =
+    fs
+      .readFileSync(cmdPath, "utf8")
+      .split(/\r?\n/)
+      .find((line) => line.includes("%*")) || "";
+  const tokens = [...forwardLine.matchAll(/"([^"]*)"/g)]
+    .map((m) => m[1])
+    .map((t) =>
+      t
+        .replace(/%~dp0/gi, () => dir + path.sep)
+        .replace(/%dp0%/gi, () => dir + path.sep)
+    );
+  const jsIndex = tokens.findIndex((t) => /\.[cm]?js$/i.test(t));
+  if (jsIndex >= 0) {
+    return { file: process.execPath, args: [...tokens.slice(jsIndex), ...argv] };
+  }
+  if (tokens.length > 0) {
+    return { file: tokens[0], args: [...tokens.slice(1), ...argv] };
+  }
+  return { file: "agent-browser", args: argv };
+}
+
+/**
+ * Build a shell-free invocation. POSIX runs the native `agent-browser` binary
+ * directly; Windows resolves the npm `.cmd` shim to that native target so the
+ * argument array is passed verbatim, never routed through cmd.exe/PowerShell.
+ */
+function buildAgentBrowserInvocation(port, args, platform = process.platform) {
+  const argv = ["--cdp", String(port), ...args.map(String)];
+  if (platform !== "win32") {
+    return { file: "agent-browser", args: argv };
+  }
+  return resolveWindowsAgentBrowser(argv);
+}
+
+// ---------------------------------------------------------------------------
+// agent-browser 工具函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 调用 agent-browser CLI
+ * @param {number} port - CDP 端口
+ * @param  {...string} args - agent-browser 参数
+ * @returns {string} stdout（trim 后）
+ */
+function ab(port, ...args) {
+  const invocation = buildAgentBrowserInvocation(port, args);
+  try {
+    return execFileSync(
+      invocation.file,
+      invocation.args,
+      {
+        encoding: "utf-8",
+        timeout: 20000,
+        stdio: ["pipe", "pipe", "pipe"],
+        windowsHide: true,
+      }
+    ).trim();
+  } catch (error) {
+    const stderr = error && error.stderr ? String(error.stderr).trim() : "";
+    const stdout = error && error.stdout ? String(error.stdout).trim() : "";
+    const detail = stderr || stdout || (error && error.message) || "unknown error";
+    throw new Error(`agent-browser failed: ${detail}`, { cause: error });
+  }
+}
+
+/** 等待 ms 毫秒（跨平台，不依赖系统 sleep 命令） */
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function parseJSONResult(raw) {
+  if (!raw || raw === "ERR") {
+    throw new Error("agent-browser returned no JSON result");
+  }
+  try {
+    let parsed = JSON.parse(raw);
+    if (typeof parsed === "string") {
+      try { parsed = JSON.parse(parsed); } catch {}
+    }
+    return parsed;
+  } catch (error) {
+    throw new Error(`agent-browser returned invalid JSON: ${String(raw).slice(0, 160)}`, {
+      cause: error,
+    });
+  }
+}
+
+/**
+ * 在浏览器内执行 JS，并解析 JSON 返回值。
+ * 一律走 base64（-b）：正文提取用的 JS 常含引号、反斜杠等，作为命令行参数时在 Windows 上
+ * 无法逐字透传（.cmd 的 %* 与 PowerShell 都会二次解析）。base64 让参数只含 [A-Za-z0-9+/=]，
+ * 和各采集脚本已在用的 evalJSONBase64 走同一条安全通道。
+ */
+function evalJSON(port, js) {
+  return evalJSONBase64(port, js);
+}
+
+/**
+ * 通过 agent-browser 的 base64 参数执行复杂 JS，避免命令行转义和参数边界问题。
+ */
+function evalJSONBase64(port, js) {
+  const encoded = Buffer.from(String(js), "utf8").toString("base64");
+  return parseJSONResult(ab(port, "eval", "-b", encoded));
+}
+
+/**
+ * 安全地将值插入浏览器 eval 字符串。
+ * 使用 JSON.stringify 确保值不会因特殊字符（引号、反斜杠等）破坏 eval 字符串。
+ * @param {*} val - 要插入的值
+ * @returns {string} JSON 字符串表示（含引号）
+ */
+function safeStr(val) {
+  return JSON.stringify(String(val));
+}
+
+/**
+ * 滚动页面加载更多内容
+ * @param {number} port - CDP 端口
+ * @param {number} times - 滚动次数
+ * @param {number} [interval=1000] - 每次滚动间隔（ms）
+ */
+function scrollLoad(port, times, interval = 1000) {
+  for (let i = 0; i < times; i++) {
+    ab(port, "eval", "window.scrollBy(0, window.innerHeight)");
+    sleep(interval);
+  }
+}
+
+/** 解析 --xxx 参数 */
+function getArg(args, name) {
+  const i = args.indexOf(name);
+  if (i >= 0) return i + 1 < args.length ? args[i + 1] : null;
+  const prefix = `${name}=`;
+  const inline = args.find((arg) => String(arg).startsWith(prefix));
+  return inline === undefined ? null : String(inline).slice(prefix.length);
+}
+
+/**
+ * 输出文件名用的日期戳（YYYYMMDD），一律取**本地日历日**。
+ * 不能用 new Date().toISOString().slice(0,10)：那是 UTC 日期，比 UTC+8 晚 8 小时。
+ * 文件名是各采集脚本唯一的去重键（一个榜单一天一份），北京时间 00:00-08:00 之间的采集
+ * 会退回「昨天」的文件名，静默覆盖前一晚采到的同名报告，且这份数据被标成前一天。
+ * @param {Date} [date] - 默认当前时间
+ * @returns {string} YYYYMMDD
+ */
+function localDateStamp(date) {
+  const d = date instanceof Date ? date : new Date();
+  const y = String(d.getFullYear()).padStart(4, "0");
+  const m = String(d.getMonth() + 1).padStart(2, "0");
+  const day = String(d.getDate()).padStart(2, "0");
+  return `${y}${m}${day}`;
+}
+
+/**
+ * Run a scraper entrypoint and turn empty/partial output into machine-readable
+ * CLI status. Legacy entrypoints may return an integer; multi-target scrapers
+ * return {planned,written,failed,partial,partialReasons}.
+ */
+function runCli(main, label) {
+  Promise.resolve()
+    .then(main)
+    .then((result) => {
+      const outcome = Number.isInteger(result)
+        ? { planned: result, written: result, failed: 0, partial: false, partialReasons: [] }
+        : result;
+      if (!outcome || !Number.isInteger(outcome.written) || outcome.written < 1) {
+        throw new Error("no output was written");
+      }
+      const failed = Number.isInteger(outcome.failed) ? outcome.failed : 0;
+      const planned = Number.isInteger(outcome.planned)
+        ? outcome.planned
+        : outcome.written + failed;
+      const reasons = Array.isArray(outcome.partialReasons)
+        ? outcome.partialReasons.filter(Boolean).map(String)
+        : [];
+      if (outcome.partial || failed > 0) {
+        const details = [`wrote ${outcome.written}/${planned}`];
+        if (failed > 0) details.push(`failed ${failed}`);
+        details.push(...reasons);
+        console.error(`${label} partial: ${details.join("; ")}`);
+        process.exitCode = 2;
+      }
+    })
+    .catch((error) => {
+      const message = error && error.message ? error.message : String(error);
+      console.error(`${label} failed: ${message}`);
+      process.exitCode = 1;
+    });
+}
+
+module.exports = {
+  ab,
+  sleep,
+  evalJSON,
+  evalJSONBase64,
+  buildAgentBrowserInvocation,
+  safeStr,
+  scrollLoad,
+  getArg,
+  localDateStamp,
+  runCli,
+};

package/resources/skills/story-creation/upstream/skills/story-short-scan/scripts/dz-browse-scraper.js

@@ -0,0 +1,232 @@
+#!/usr/bin/env node
+/**
+ * 点众阅读短篇采集脚本
+ *
+ * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
+ * 采集策略：以 /book/{id} 链接为骨架，按 bookId 聚合每本书的多个 anchor
+ * （封面/书名+评分/简介各一个），从中解出书名、评分、简介、作品页，再从卡片
+ * 容器文本里解出 作者·标签·状态·字数 与最新章节。避免纯 innerText 行序解析把
+ * UI 文字或简介误当书名。
+ * 输出 Markdown 格式。
+ *
+ * 用法：
+ *   node dz-browse-scraper.js --channel male              # 男频
+ *   node dz-browse-scraper.js --channel female             # 女频
+ *   node dz-browse-scraper.js --channel all                # 全部
+ *
+ * 前置：
+ *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { ab, sleep, evalJSONBase64, safeStr, scrollLoad, getArg, localDateStamp, runCli } = require("./cdp-utils");
+
+const CHANNELS = [
+  { id: "male", label: "男频", tab: "男频", url: "https://www.ishugui.com/browse" },
+  { id: "female", label: "女频", tab: "女频", url: "https://www.ishugui.com/browse/on3" },
+];
+
+// ---------------------------------------------------------------------------
+// 页面操作
+// ---------------------------------------------------------------------------
+
+/** 连通性 + 页面就绪自检 */
+function probePage(port) {
+  return evalJSONBase64(
+    port,
+    "JSON.stringify({host:location.host,len:(document.body&&document.body.innerText||'').length})"
+  );
+}
+
+/** 点击指定文本的 tab */
+function clickTab(port, text) {
+  const js =
+    "JSON.stringify((function(){" +
+    "var all=document.querySelectorAll('div,span,a,button,li');" +
+    "var el=Array.from(all).find(function(e){return (e.textContent||'').trim()===" + safeStr(text) + "});" +
+    "if(el){el.click();return true}return false" +
+    "})())";
+  return evalJSONBase64(port, js);
+}
+
+/**
+ * 以 /book/{id} 链接为骨架聚合每本书的字段。
+ * 书名取自“书名+评分”anchor（去掉尾部 X.X分），简介取自最长 anchor，
+ * 作者/标签/状态/字数从卡片容器文本里正则提取。
+ */
+function buildStoriesJS() {
+  return `JSON.stringify((function(){
+    var anchors=Array.from(document.querySelectorAll('a')).filter(function(a){
+      return /\\/book\\/[0-9]+/.test(a.getAttribute('href')||'');
+    });
+    var byId={};var order=[];
+    anchors.forEach(function(a){
+      var m=(a.getAttribute('href')||'').match(/\\/book\\/([0-9]+)/);if(!m)return;
+      var id=m[1];var txt=(a.innerText||a.textContent||'').replace(/\\s+/g,' ').trim();
+      if(!byId[id]){byId[id]={id:id,texts:[],node:a};order.push(id);}
+      if(txt)byId[id].texts.push(txt);
+    });
+    var out=[];
+    order.forEach(function(id){
+      var g=byId[id];
+      var title='',score='';
+      for(var i=0;i<g.texts.length;i++){
+        var tm=g.texts[i].match(/^(.+?)\\s*([0-9]+(?:\\.[0-9]+)?)分$/);
+        if(tm){title=tm[1].trim();score=tm[2]+'分';break;}
+      }
+      if(!title){
+        var cand=g.texts.filter(Boolean).slice().sort(function(a,b){return a.length-b.length;});
+        title=cand.length?cand[0]:'';
+      }
+      // 简介：最长的、不是“书名+评分”的 anchor 文本
+      var desc='';
+      g.texts.forEach(function(t){ if(/分$/.test(t))return; if(t.length>desc.length)desc=t; });
+      // 卡片容器：从任一 anchor 向上找到含“字”的祖先
+      var el=g.node;
+      for(var j=0;j<6;j++){ if(el.parentElement){el=el.parentElement; if((el.innerText||'').indexOf('字')>-1)break;} }
+      var card=(el.innerText||'').replace(/\\s+/g,' ');
+      var tail=(desc&&card.indexOf(desc)>-1)?card.slice(card.indexOf(desc)+desc.length):card;
+      var meta=tail.match(/([^·]{1,20}?)\\s*·\\s*([^·]{1,20}?)\\s*·\\s*(完结|完本|连载)\\s*·\\s*([0-9]+)\\s*字/);
+      var author=meta?meta[1].trim():'';
+      var tag=meta?meta[2].trim():'';
+      var status=meta?meta[3]:'';
+      var words=meta?meta[4]+'字':'';
+      var um=card.match(/最新章节[:：\\s]*([^·]{1,40})/);
+      var update=um?um[1].trim():'';
+      out.push({rank:out.length+1,bookId:id,title:title,score:score,author:author,tag:tag,status:status,words:words,update:update,desc:desc.slice(0,200),url:'https://www.ishugui.com/book/'+id});
+    });
+    return out;
+  })())`;
+}
+
+function extractStories(port) {
+  const list = evalJSONBase64(port, buildStoriesJS());
+  return Array.isArray(list) ? list : [];
+}
+
+// ---------------------------------------------------------------------------
+// 主流程
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+const PORT = parseInt(getArg(args, "--port") || "9222", 10);
+const OUTDIR = getArg(args, "--outdir") || ".";
+const CHANNEL = getArg(args, "--channel") || "male";
+
+function scrapeChannel(port, channelId) {
+  const ch = CHANNELS.find((c) => c.id === channelId);
+  if (!ch) return null;
+
+  console.log(`\n→ 采集 点众${ch.label}短篇...`);
+
+  let stories;
+  try {
+    ab(port, "open", ch.url);
+    sleep(3000);
+
+    // 连通性自检：CDP 未起/被重定向时给可操作报错，而非静默产空
+    const probe = probePage(port);
+    if (!probe) {
+      console.error(
+        `  ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome（端口 ${port}），且 agent-browser 可用。`
+      );
+      return null;
+    }
+    if (probe.host && probe.host.indexOf("ishugui") === -1) {
+      console.error(`  ✗ 当前页面非点众（host=${probe.host}），可能被重定向，已跳过。`);
+      return null;
+    }
+
+    // 切换频道（female 已有独立 URL，tab 失败不致命）
+    try {
+      if (clickTab(port, ch.tab)) {
+        console.log(`  ✓ 切换到${ch.tab}`);
+        sleep(2000);
+      }
+    } catch (tabErr) {
+      console.error(`[dz] ${ch.label} tab切换出错，继续采集: ${tabErr.message}`);
+    }
+
+    scrollLoad(port, 8);
+    sleep(1000);
+
+    stories = extractStories(port);
+  } catch (err) {
+    console.error(`[dz] ${ch.label} 页面加载或提取出错: ${err.message}`);
+    return null;
+  }
+
+  if (!stories.length) {
+    console.error(
+      `[dz] 采集失败：未解析到书目（页面结构可能变动或未加载）。请人工打开 ${ch.url} 确认页面正常。`
+    );
+    return null;
+  }
+
+  // 质量门：书名命中率是点众采集成败的核心信号
+  const titled = stories.filter((s) => s.title).length;
+  const ratio = titled / stories.length;
+  const quality = ratio < 0.5 ? "[书名解析异常]" : "[OK]";
+  console.log(`  ✓ 提取 ${stories.length} 条（书名 ${titled}/${stories.length}）`);
+  if (ratio < 0.5) {
+    console.error(`  ⚠ 书名解析率偏低（${titled}/${stories.length}），结果质量已标注。`);
+  }
+
+  const now = new Date().toISOString();
+  const lines = [
+    `# 点众 · ${ch.label}短篇`,
+    "",
+    `- 来源：${ch.url}`,
+    `- 抓取时间：${now}`,
+    `- 条目数：${stories.length}`,
+    `- 书名解析：${titled} / ${stories.length}`,
+    `- 数据质量：${quality}`,
+    "",
+    "---",
+    "",
+  ];
+
+  stories.forEach((s, i) => {
+    try {
+      lines.push(`### #${i + 1} ${s.title || "（书名待解析）"}`);
+      const meta = [s.author, s.tag, s.status, s.words, s.score].filter(Boolean).join(" · ");
+      if (meta) lines.push(`*${meta}*`);
+      if (s.update) lines.push(`**最新：** ${s.update}`);
+      if (s.url) lines.push(`[作品页](${s.url})`);
+      if (s.desc) {
+        lines.push("");
+        lines.push(`> ${s.desc.substring(0, 150)}${s.desc.length > 150 ? "..." : ""}`);
+      }
+      lines.push("", "---", "");
+    } catch (storyErr) {
+      console.error(`[dz] ${ch.label} 第${i + 1}条处理出错: ${storyErr.message}`);
+      lines.push("", "---", "");
+    }
+  });
+
+  return lines.join("\n");
+}
+
+function main() {
+  const channels = CHANNEL === "all" ? CHANNELS.map((c) => c.id) : [CHANNEL];
+  let written = 0;
+
+  for (const ch of channels) {
+    const content = scrapeChannel(PORT, ch);
+    if (!content) continue;
+
+    const chInfo = CHANNELS.find((c) => c.id === ch);
+    const filename = `点众${chInfo.label}短篇_${localDateStamp()}.md`;
+    fs.mkdirSync(OUTDIR, { recursive: true });
+    const filepath = path.join(OUTDIR, filename);
+    fs.writeFileSync(filepath, content, "utf-8");
+    written++;
+    console.log(`  ✓ 已保存: ${filepath}`);
+  }
+  return written;
+}
+
+if (require.main === module) {
+  runCli(main, "点众采集");
+}