@cloud411716/fancy-webnovel 2.2.0 → 3.0.3

package/plugins/fancy-douyin/scripts/start-chrome.cjs

@@ -0,0 +1,203 @@
+'use strict';
+
+/**
+ * Start, reuse, switch or stop the workspace-local Chrome.
+ *
+ * - headed: needed for the first manual login / captcha.
+ * - headless: used on later runs when login-state.json has auth cookies.
+ * - --stop: close the Chrome belonging to this workspace.
+ *
+ * The profile is the primary login state. Cookies are additionally mirrored to
+ * login-state.json by the page scripts.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const net = require('net');
+const http = require('http');
+const { spawn } = require('child_process');
+const crypto = require('crypto');
+const { parseArgs, ensureDir, readJson, writeJsonAtomic, printJson, fail, log } = require('./lib.cjs');
+
+function getVersion(port, timeout = 1500) {
+  return new Promise((resolve) => {
+    const req = http.get({ host: '127.0.0.1', port, path: '/json/version', timeout }, (res) => {
+      let body = '';
+      res.setEncoding('utf8');
+      res.on('data', (chunk) => { body += chunk; });
+      res.on('end', () => {
+        try {
+          resolve({ ok: true, data: JSON.parse(body) });
+        } catch {
+          resolve({ ok: false });
+        }
+      });
+    });
+    req.on('timeout', () => { req.destroy(); resolve({ ok: false }); });
+    req.on('error', () => resolve({ ok: false }));
+  });
+}
+
+function portFree(port) {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once('error', () => resolve(false));
+    server.listen(port, '127.0.0.1', () => {
+      server.close(() => resolve(true));
+    });
+  });
+}
+
+async function pickPort(preferred) {
+  if (await portFree(preferred)) return preferred;
+  for (let port = 9333; port <= 9399; port += 1) {
+    if (await portFree(port)) return port;
+  }
+  throw new Error('没有可用的本地端口');
+}
+
+async function waitForCdp(port, timeoutMs = 40000) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const result = await getVersion(port);
+    if (result.ok) return result.data;
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  return null;
+}
+
+async function closeChrome(port) {
+  const version = await getVersion(port);
+  if (!version.ok) return false;
+  const wsUrl = version.data && version.data.webSocketDebuggerUrl;
+  if (!wsUrl || typeof WebSocket !== 'function') return false;
+  await new Promise((resolve) => {
+    let finished = false;
+    const finish = () => {
+      if (finished) return;
+      finished = true;
+      try { ws.close(); } catch { /* ignore */ }
+      resolve();
+    };
+    let ws;
+    try {
+      ws = new WebSocket(wsUrl);
+    } catch {
+      resolve();
+      return;
+    }
+    ws.onopen = () => {
+      try { ws.send(JSON.stringify({ id: 1, method: 'Browser.close' })); } catch { /* ignore */ }
+      setTimeout(finish, 2500);
+    };
+    ws.onmessage = finish;
+    ws.onerror = finish;
+    setTimeout(finish, 5000);
+  });
+  for (let i = 0; i < 20; i += 1) {
+    if (!(await getVersion(port)).ok) return true;
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  return false;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const workspace = path.resolve(String(args.workspace || process.cwd()));
+  const configFile = path.resolve(String(args.config || path.join(workspace, '.fancy', 'douyin', 'chrome.json')));
+  const profile = path.resolve(String(args.profile || path.join(workspace, '.fancy', 'douyin', 'chrome-profile')));
+  const chrome = String(args.chrome || process.env.FANCY_DOUYIN_CHROME || process.env.CHROME_PATH || '/usr/bin/google-chrome');
+  const mode = String(args.mode || 'headed').toLowerCase() === 'headless' ? 'headless' : 'headed';
+  const existing = readJson(configFile);
+
+  if (args.stop) {
+    const port = Number(existing && existing.port ? existing.port : 0);
+    if (!port) {
+      printJson({ ok: true, stopped: false, reason: '没有 chrome.json 端口记录' });
+      return;
+    }
+    const stopped = await closeChrome(port);
+    printJson({ ok: true, stopped, port });
+    return;
+  }
+
+  if (!fs.existsSync(chrome)) {
+    throw new Error('找不到 Chrome：' + chrome + '。请安装 google-chrome，或设置 FANCY_DOUYIN_CHROME。');
+  }
+
+  const hash = crypto.createHash('sha256').update(workspace).digest();
+  const workspacePort = 9400 + (hash[0] % 200);
+  const preferred = Number(args.port || process.env.FANCY_DOUYIN_CDP_PORT || (existing && existing.port) || workspacePort);
+
+  if (existing && Number(existing.port) > 0) {
+    const version = await getVersion(Number(existing.port));
+    const existingMode = existing.mode || 'headed';
+    if (version.ok && existingMode === mode) {
+      const port = Number(existing.port);
+      printJson({
+        ok: true,
+        reused: true,
+        mode,
+        port,
+        cdp: 'http://127.0.0.1:' + port,
+        profile,
+        browser: version.data?.Browser || '',
+      });
+      return;
+    }
+    if (version.ok) {
+      log('[chrome] mode switch ' + existingMode + ' -> ' + mode + ', closing old browser');
+      await closeChrome(Number(existing.port));
+    }
+  }
+
+  const port = await pickPort(Number.isFinite(preferred) && preferred > 0 ? preferred : workspacePort);
+  ensureDir(profile);
+  ensureDir(path.dirname(configFile));
+
+  const chromeArgs = [
+    '--remote-debugging-port=' + port,
+    '--remote-allow-origins=*',
+    '--user-data-dir=' + profile,
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-blink-features=AutomationControlled',
+    '--window-size=1400,1000',
+    '--lang=zh-CN',
+  ];
+  if (mode === 'headless') {
+    chromeArgs.push('--headless=new');
+  }
+  chromeArgs.push('about:blank');
+
+  log('[chrome] starting', chrome, 'mode=' + mode, 'port=' + port, 'profile=' + profile);
+  const child = spawn(chrome, chromeArgs, {
+    detached: true,
+    stdio: 'ignore',
+  });
+  child.unref();
+
+  const version = await waitForCdp(port);
+  if (!version) {
+    throw new Error('Chrome 已启动，但 CDP 在 40 秒内没有就绪。请检查 DISPLAY 是否能打开图形窗口；如果该 profile 已被另一个 Chrome 窗口占用，请先关闭那个窗口再重试。');
+  }
+
+  writeJsonAtomic(configFile, {
+    port,
+    profile,
+    mode,
+    started_at: new Date().toISOString(),
+  }, 0o600);
+
+  printJson({
+    ok: true,
+    reused: false,
+    mode,
+    port,
+    cdp: 'http://127.0.0.1:' + port,
+    profile,
+    browser: version.Browser || '',
+  });
+}
+
+main().catch(fail);

package/plugins/fancy-douyin/state.js

@@ -0,0 +1,197 @@
+/**
+ * Workspace state for fancy-douyin.
+ *
+ * Accounts, crawl history, the Chrome profile and downloaded screenshots all
+ * live under <workspace>/.fancy/douyin. The deployed crawler scripts live under
+ * <workspace>/.fancy-tools/fancy-douyin.
+ *
+ * The crawl history is keyed by the note id parsed from /note/<id>. A note is
+ * written only after its screenshot analysis has been imported into the
+ * material library, so failed notes are retried on the next run.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { atomicWriteJson, readJsonFile } from '#workspace';
+
+export const DOUYIN_DIR = path.join('.fancy', 'douyin');
+
+export function douyinPaths(root) {
+  const base = path.join(path.resolve(root), DOUYIN_DIR);
+  return {
+    base,
+    accounts: path.join(base, 'accounts.json'),
+    crawled: path.join(base, 'crawled.json'),
+    loginState: path.join(base, 'login-state.json'),
+    chrome: path.join(base, 'chrome.json'),
+    profile: path.join(base, 'chrome-profile'),
+    captchaLog: path.join(base, 'captcha.log'),
+    output: path.join(base, 'output'),
+  };
+}
+
+export function ensureDouyinDir(root) {
+  const paths = douyinPaths(root);
+  fs.mkdirSync(paths.base, { recursive: true, mode: 0o700 });
+  return paths;
+}
+
+/**
+ * Normalize a Douyin account homepage URL.
+ *
+ * We only accept /user/<id> pages. Share links and individual note links are
+ * not account homepages and cannot be scanned.
+ *
+ * @param {string} raw
+ * @returns {string}
+ */
+export function normalizeAccountUrl(raw) {
+  let text = String(raw || '').trim();
+  if (!text) throw new Error('缺少抖音账号主页地址');
+  if (!/^https?:\/\//i.test(text)) text = 'https://' + text;
+  let url;
+  try {
+    url = new URL(text);
+  } catch {
+    throw new Error('不是合法的 URL：' + raw);
+  }
+  const host = url.hostname.toLowerCase();
+  if (host !== 'douyin.com' && !host.endsWith('.douyin.com')
+    && host !== 'iesdouyin.com' && !host.endsWith('.iesdouyin.com')) {
+    throw new Error('不是抖音地址：' + raw);
+  }
+  if (!url.pathname.startsWith('/user/') || url.pathname.length <= '/user/'.length) {
+    throw new Error('请填写抖音账号主页地址（包含 /user/），不要填写单篇作品或短链：' + raw);
+  }
+  url.hash = '';
+  url.search = '';
+  return url.toString();
+}
+
+export function accountSlug(url) {
+  try {
+    const parsed = new URL(url);
+    const id = parsed.pathname.split('/').filter(Boolean).pop() || 'account';
+    return id.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 80) || 'account';
+  } catch {
+    return 'account';
+  }
+}
+
+export function noteIdFromUrl(url) {
+  const match = String(url || '').match(/\/note\/(\d+)/);
+  if (match) return match[1];
+  return String(url || '').replace(/[?#].*$/, '').split('/').filter(Boolean).pop() || '';
+}
+
+export function isNoteUrl(url) {
+  return /\/note\/\d+/.test(String(url || ''));
+}
+
+export function readAccounts(root) {
+  const paths = douyinPaths(root);
+  const value = readJsonFile(paths.accounts);
+  const accounts = Array.isArray(value?.accounts) ? value.accounts : [];
+  return accounts
+    .filter((item) => item && typeof item.url === 'string')
+    .map((item) => ({
+      url: item.url,
+      added_at: item.added_at || '',
+    }));
+}
+
+export function writeAccounts(root, accounts) {
+  const paths = ensureDouyinDir(root);
+  atomicWriteJson(paths.accounts, {
+    version: 1,
+    accounts: accounts.map((item) => ({
+      url: item.url,
+      added_at: item.added_at || new Date().toISOString(),
+    })),
+  }, { mode: 0o600 });
+}
+
+/**
+ * @param {string} root
+ * @param {string} rawUrl
+ * @returns {{added: boolean, url: string, accounts: Array}}
+ */
+export function addAccount(root, rawUrl) {
+  const url = normalizeAccountUrl(rawUrl);
+  const accounts = readAccounts(root);
+  if (accounts.some((item) => item.url === url)) {
+    return { added: false, url, accounts };
+  }
+  accounts.push({ url, added_at: new Date().toISOString() });
+  writeAccounts(root, accounts);
+  return { added: true, url, accounts };
+}
+
+/**
+ * Remove an account by 1-based list index or by homepage URL.
+ *
+ * @param {string} root
+ * @param {string} selector
+ * @returns {{removed: boolean, url: string, accounts: Array}}
+ */
+export function removeAccount(root, selector) {
+  const accounts = readAccounts(root);
+  const text = String(selector || '').trim();
+  if (!text) throw new Error('缺少要删除的账号序号或主页地址');
+  let index = -1;
+  if (/^\d+$/.test(text)) {
+    index = Number(text) - 1;
+  } else {
+    const url = normalizeAccountUrl(text);
+    index = accounts.findIndex((item) => item.url === url);
+  }
+  if (index < 0 || index >= accounts.length) {
+    throw new Error('没有找到要删除的账号：' + selector);
+  }
+  const [removed] = accounts.splice(index, 1);
+  writeAccounts(root, accounts);
+  return { removed: true, url: removed.url, accounts };
+}
+
+export function readCrawled(root) {
+  const value = readJsonFile(douyinPaths(root).crawled);
+  return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+
+export function writeCrawled(root, state) {
+  const paths = ensureDouyinDir(root);
+  atomicWriteJson(paths.crawled, state, { mode: 0o600 });
+}
+
+export function markCrawled(root, noteId, data) {
+  const state = readCrawled(root);
+  state[noteId] = {
+    ...data,
+    crawled_at: new Date().toISOString(),
+  };
+  writeCrawled(root, state);
+  return state[noteId];
+}
+
+export function accountListText(root) {
+  const accounts = readAccounts(root);
+  if (!accounts.length) {
+    return [
+      '还没有记录抖音账号。',
+      '先添加账号主页：/fancy-douyin add https://www.douyin.com/user/...',
+    ].join('\n');
+  }
+  const crawled = readCrawled(root);
+  const counts = new Map();
+  for (const item of Object.values(crawled)) {
+    if (!item || typeof item !== 'object' || !item.account_url) continue;
+    counts.set(item.account_url, (counts.get(item.account_url) || 0) + 1);
+  }
+  return [
+    '已记录的抖音账号：',
+    ...accounts.map((item, index) => {
+      const count = counts.get(item.url) || 0;
+      return `${index + 1}. ${item.url}（已采集 ${count} 篇）`;
+    }),
+  ].join('\n');
+}

package/plugins/fancy-douyin/templates.js

@@ -0,0 +1,127 @@
+/**
+ * fancy-douyin command text and LLM prompt rendering.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PROMPT_FILE = path.join(__dirname, 'prompts', '01-analysis-prompt.md');
+
+export function helpText() {
+  return `🎬 抖音图文采集 用法
+
+/fancy-douyin add <主页地址>      记录一个抖音账号主页
+/fancy-douyin list               查看已记录的账号
+/fancy-douyin remove <序号|地址> 删除已记录的账号
+/fancy-douyin                    依次采集所有账号下未处理的图文作品
+/fancy-douyin help               显示本帮助
+
+说明：
+- 只采集 /note/ 图文作品，跳过视频。
+- 每篇作品会下载轮播原图并转成 JPG（不是滚动中的页面截图），然后交给 LLM 提取素材。
+- 素材按 material JSONL 格式写入数据库，source 固定为 other。
+- 已成功入库的作品记录在 <工作区>/.fancy/douyin/crawled.json，后续跳过。
+- 登录态保存在 <工作区>/.fancy/douyin/chrome-profile/，并镜像到 login-state.json。
+- 首次需要登录时用有头 Chrome；登录态有效后后续运行自动使用无头 Chrome。
+- 每次运行结束会自动关闭 Chrome，释放资源。
+- 需要登录或安全验证时会弹窗，请在 Chrome 窗口手动完成，然后选择「已登陆」。
+- 浏览器脚本部署在 <工作区>/.fancy-tools/fancy-douyin/。`;
+}
+
+export function loginQuestion(paths) {
+  return {
+    id: 'douyin-login',
+    header: '抖音需要登录',
+    question: '请在弹出的 Chrome 窗口完成抖音登录或安全验证。完成后点「已登陆」继续。',
+    detail: [
+      '登录态会保存在工作区的 Chrome profile 和 login-state.json 中，下次运行可复用。',
+      '如果当前没有看到 Chrome 窗口，请检查桌面环境/DISPLAY 是否可用。',
+      '点「取消」会停止本次采集，已导入的素材和已保存的登录态不会撤销。',
+      '',
+      'profile：' + paths.profile,
+      'cookie 镜像：' + paths.loginState,
+    ].join('\n'),
+    options: [
+      { label: '已登陆', description: '已经完成登录/验证，继续采集' },
+      { label: '取消', description: '停止本次采集' },
+    ],
+  };
+}
+
+function read(value, fallback = '') {
+  const text = String(value == null ? '' : value).trim();
+  return text || fallback;
+}
+
+export function analysisPrompt({
+  accountUrl,
+  note,
+  screenshots,
+  targetFile,
+  workspace,
+  materialKinds,
+  queryScript,
+  validateScript,
+}) {
+  const template = fs.readFileSync(PROMPT_FILE, 'utf8');
+  const list = (screenshots || [])
+    .map((shot, index) => `${index + 1}. \`${shot.file}\``)
+    .join('\n');
+  const vars = {
+    ACCOUNT_URL: read(accountUrl),
+    NOTE_URL: read(note.url),
+    NOTE_TITLE: read(note.title, '（无标题）'),
+    NOTE_DESCRIPTION: read(note.description, '（无描述）'),
+    NOTE_BODY: read(note.body, '（无正文摘录）').slice(0, 3000),
+    NOTE_DIR: read(note.dir),
+    SCREENSHOT_LIST: list || '（没有截图）',
+    TARGET_FILE: targetFile,
+    MATERIAL_KINDS: (materialKinds || []).join(', '),
+    QUERY_SCRIPT: queryScript,
+    VALIDATE_SCRIPT: validateScript,
+    WORKSPACE: workspace,
+  };
+  let text = template;
+  for (const [key, value] of Object.entries(vars)) {
+    text = text.split('{{' + key + '}}').join(value);
+  }
+  return text.trim() + '\n';
+}
+
+export function summaryText({
+  accounts,
+  scannedAccounts,
+  foundNotes,
+  skippedNotes,
+  processedNotes,
+  inserted,
+  skippedMaterials,
+  failures,
+  outputDir,
+  stateFile,
+  aborted,
+  mode,
+}) {
+  const lines = [
+    aborted ? '⚠️ 抖音采集已中断' : '✅ 抖音采集完成',
+    '账号：' + accounts + ' 个（本次扫描 ' + scannedAccounts + ' 个）',
+    '浏览器：' + (mode === 'headless' ? '无头' : '有头'),
+    '发现图文：' + foundNotes + ' 篇',
+    '已处理过并跳过：' + skippedNotes + ' 篇',
+    '本次处理：' + processedNotes + ' 篇',
+    '新增素材：' + inserted + ' 条（重复跳过 ' + skippedMaterials + ' 条）',
+  ];
+  if (failures.length) {
+    lines.push('失败：' + failures.length + ' 篇');
+    for (const item of failures.slice(0, 10)) {
+      lines.push('  - ' + (item.id || item.url || '未知') + '：' + item.reason);
+    }
+    if (failures.length > 10) lines.push('  - 其余 ' + (failures.length - 10) + ' 篇省略');
+    lines.push('失败作品不会写入采集记录，下次运行会重试。');
+  }
+  lines.push('输出目录：' + outputDir);
+  lines.push('采集记录：' + stateFile);
+  return lines.join('\n');
+}