@cloud411716/fancy-webnovel 2.2.0 → 3.0.0

package/plugins/fancy-douyin/scripts/crawl-note.cjs

@@ -0,0 +1,194 @@
+'use strict';
+
+/**
+ * Open one Douyin /note/ work, capture its image slides as JPG screenshots and
+ * write note.json. stdout is one JSON object; Chrome stays open.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const {
+  log, parseArgs, connect, restoreCookies, saveCookies,
+  gateResult, printJson, fail, ensureDir,
+} = require('./lib.cjs');
+
+function noteId(url) {
+  const match = String(url).match(/\/note\/(\d+)/);
+  if (match) return match[1];
+  return String(url).replace(/[?#].*$/, '').split('/').filter(Boolean).pop() || '';
+}
+
+async function readMetadata(page) {
+  return await page.evaluate(() => {
+    const meta = (name) => {
+      const el = document.querySelector('meta[name="' + name + '"]');
+      return el && el.content ? el.content : '';
+    };
+    return {
+      url: location.href,
+      title: document.title || '',
+      description: meta('description'),
+      body: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 6000),
+    };
+  });
+}
+
+const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';
+
+async function collectSlideUrls(page) {
+  const seen = new Set();
+  const urls = [];
+  for (let round = 0; round < 14; round += 1) {
+    const batch = await page.evaluate(() => {
+      const imgs = [...document.querySelectorAll('img')]
+        .filter((img) => img.src && img.src.includes('tplv-dy-aweme-images'))
+        .map((img) => ({ src: img.src, x: Math.round(img.getBoundingClientRect().x) }));
+      const localSeen = new Set();
+      const unique = [];
+      for (const img of imgs.sort((a, b) => a.x - b.x)) {
+        if (localSeen.has(img.src)) continue;
+        localSeen.add(img.src);
+        unique.push(img.src);
+      }
+      return unique;
+    });
+    let added = 0;
+    for (const src of batch) {
+      if (seen.has(src)) continue;
+      seen.add(src);
+      urls.push(src);
+      added += 1;
+    }
+    if (added === 0 && round > 0) break;
+    await page.keyboard.press('ArrowRight');
+    await page.waitForTimeout(900);
+  }
+  return urls;
+}
+
+async function downloadAsJpeg(context, converterPage, src, file) {
+  const response = await context.request.get(src, {
+    headers: {
+      Referer: 'https://www.douyin.com/',
+      'User-Agent': UA,
+    },
+    timeout: 45000,
+    failOnStatusCode: false,
+  });
+  if (!response.ok()) throw new Error('下载图片失败：HTTP ' + response.status());
+  const body = await response.body();
+  const type = String(response.headers()['content-type'] || 'image/jpeg').split(';')[0].trim() || 'image/jpeg';
+  const dataUrl = 'data:' + type + ';base64,' + body.toString('base64');
+  const jpegDataUrl = await converterPage.evaluate(async (url) => {
+    const image = new Image();
+    image.src = url;
+    if (typeof image.decode === 'function') await image.decode();
+    else await new Promise((resolve, reject) => { image.onload = resolve; image.onerror = reject; });
+    const canvas = document.createElement('canvas');
+    canvas.width = image.naturalWidth;
+    canvas.height = image.naturalHeight;
+    const context2d = canvas.getContext('2d');
+    context2d.drawImage(image, 0, 0);
+    return canvas.toDataURL('image/jpeg', 0.9);
+  }, dataUrl);
+  const base64 = String(jpegDataUrl).replace(/^data:image\/jpeg;base64,/, '');
+  fs.writeFileSync(file, Buffer.from(base64, 'base64'));
+}
+
+async function captureSlides(page, context, outDir) {
+  const urls = await collectSlideUrls(page);
+  const screenshots = [];
+  if (urls.length) {
+    // Convert the original carousel images, not viewport screenshots. This
+    // avoids partial captures while Douyin is scrolling or lazy-loading.
+    const converter = await context.newPage();
+    try {
+      for (let index = 0; index < urls.length; index += 1) {
+        const file = path.join(outDir, 'slide_' + String(index + 1).padStart(2, '0') + '.jpg');
+        try {
+          await downloadAsJpeg(context, converter, urls[index], file);
+          screenshots.push({ index: index + 1, file, src: urls[index] });
+        } catch (err) {
+          log('[note] slide download failed: ' + (err && err.message ? err.message : String(err)));
+        }
+      }
+    } finally {
+      try { await converter.close(); } catch { /* ignore */ }
+    }
+  }
+
+  if (!screenshots.length) {
+    const file = path.join(outDir, 'note.jpg');
+    try {
+      await page.screenshot({ path: file, type: 'jpeg', quality: 90, fullPage: false });
+      screenshots.push({ index: 1, file, src: '' });
+    } catch (err) {
+      throw new Error('没有找到图文图片，且页面截图失败：' + (err && err.message ? err.message : String(err)));
+    }
+  }
+  return screenshots;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const workspace = path.resolve(String(args.workspace || process.cwd()));
+  const noteUrl = String(args.note || '');
+  const cdp = String(args.cdp || 'http://127.0.0.1:9222');
+  const outDir = path.resolve(String(args.out || path.join(workspace, '.fancy', 'douyin', 'output', 'note')));
+  const loginState = path.resolve(String(args['login-state'] || path.join(workspace, '.fancy', 'douyin', 'login-state.json')));
+  if (!/\/note\//.test(noteUrl)) throw new Error('不是抖音图文作品地址：' + noteUrl);
+  ensureDir(outDir);
+
+  const { context } = await connect(cdp);
+  let page = null;
+  let keepPage = false;
+  try {
+    await restoreCookies(context, loginState);
+    page = await context.newPage();
+    await page.goto(noteUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
+    await page.waitForTimeout(4500);
+
+    const firstGate = await gateResult(page, outDir);
+    if (firstGate) {
+      keepPage = true;
+      await saveCookies(context, loginState, page.url());
+      printJson(firstGate);
+      return;
+    }
+
+    const meta = await readMetadata(page);
+    const screenshots = await captureSlides(page, context, outDir);
+
+    const afterGate = await gateResult(page, outDir);
+    if (afterGate) {
+      keepPage = true;
+      await saveCookies(context, loginState, page.url());
+      printJson(afterGate);
+      return;
+    }
+
+    await saveCookies(context, loginState, page.url());
+    const note = {
+      id: noteId(noteUrl),
+      url: noteUrl,
+      final_url: meta.url,
+      title: meta.title,
+      description: meta.description,
+      body: meta.body,
+      screenshots,
+      crawled_at: new Date().toISOString(),
+      dir: outDir,
+    };
+    const noteFile = path.join(outDir, 'note.json');
+    fs.writeFileSync(noteFile, JSON.stringify(note, null, 2) + '\n', 'utf8');
+    printJson({ ok: true, login_required: false, note, note_file: noteFile });
+  } finally {
+    if (page && !keepPage) {
+      try { await page.close(); } catch { /* ignore */ }
+    }
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => { fail(err); process.exit(0); });

package/plugins/fancy-douyin/scripts/lib.cjs

@@ -0,0 +1,197 @@
+'use strict';
+
+/**
+ * Shared helpers for the deployed Douyin browser scripts.
+ *
+ * These scripts run outside the plugin package, so they only require
+ * playwright-core through NODE_PATH (set by the plugin) plus Node built-ins.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+function log(...args) {
+  process.stderr.write(args.map((value) => String(value)).join(' ') + '\n');
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (!arg.startsWith('--')) continue;
+    const key = arg.slice(2);
+    const next = argv[i + 1];
+    if (next === undefined || next.startsWith('--')) {
+      out[key] = true;
+    } else {
+      out[key] = next;
+      i += 1;
+    }
+  }
+  return out;
+}
+
+function requirePlaywright() {
+  try {
+    return require('playwright-core');
+  } catch (err) {
+    throw new Error('无法加载 playwright-core。请由 /fancy-douyin 启动脚本（NODE_PATH 未设置）。' + (err && err.message ? ' ' + err.message : ''));
+  }
+}
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeJsonAtomic(file, value, mode = 0o600) {
+  ensureDir(path.dirname(file));
+  const tmp = file + '.' + process.pid + '.' + Date.now() + '.tmp';
+  fs.writeFileSync(tmp, JSON.stringify(value, null, 2) + '\n', { encoding: 'utf8', mode });
+  fs.renameSync(tmp, file);
+  try { fs.chmodSync(file, mode); } catch { /* best effort */ }
+}
+
+async function connect(cdp) {
+  const { chromium } = requirePlaywright();
+  const browser = await chromium.connectOverCDP(cdp);
+  const context = browser.contexts()[0];
+  if (!context) throw new Error('Chrome 没有可用的浏览器上下文');
+  return { browser, context };
+}
+
+async function restoreCookies(context, loginStateFile) {
+  const data = readJson(loginStateFile);
+  const cookies = Array.isArray(data?.cookies) ? data.cookies : [];
+  if (!cookies.length) return 0;
+  // The persistent profile is authoritative. Do not overwrite freshly logged-in
+  // cookies with an older mirror; only restore when the profile has no Douyin
+  // auth cookie at all.
+  try {
+    const current = await context.cookies('https://www.douyin.com');
+    const authNames = new Set(['sessionid', 'sessionid_ss', 'passport_auth_status', 'sid_tt', 'sid_guard']);
+    if (current.some((cookie) => authNames.has(cookie.name))) return 0;
+  } catch { /* fall through to restore */ }
+  try {
+    await context.addCookies(cookies);
+    return cookies.length;
+  } catch (err) {
+    log('[cookies] 恢复失败：' + (err && err.message ? err.message : String(err)));
+    return 0;
+  }
+}
+
+async function saveCookies(context, loginStateFile, url) {
+  try {
+    const cookies = await context.cookies();
+    writeJsonAtomic(loginStateFile, {
+      saved_at: new Date().toISOString(),
+      url: url || '',
+      cookies,
+    }, 0o600);
+    return cookies.length;
+  } catch (err) {
+    log('[cookies] 保存失败：' + (err && err.message ? err.message : String(err)));
+    return 0;
+  }
+}
+
+async function detectGate(page) {
+  const info = await page.evaluate(() => {
+    const body = (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 8000);
+    const modal = document.querySelector('.douyin_login_new_class');
+    let modalVisible = false;
+    try {
+      modalVisible = Boolean(modal && modal.getBoundingClientRect().width > 0);
+    } catch { /* ignore */ }
+    const loginText = body.includes('登录后免费畅享高清视频')
+      || body.includes('登录后查看更多')
+      || body.includes('登录后查看')
+      || body.includes('请先登录')
+      || body.includes('扫码登录')
+      || body.includes('密码登录');
+    const captchaKeys = ['请完成下列验证后继续', '滑块', '拖动', '安全验证', '图形验证', '请输入验证码'];
+    const captcha = captchaKeys.some((key) => body.includes(key));
+    return {
+      modalVisible,
+      loginText,
+      captcha,
+      url: location.href,
+      title: document.title,
+      bodyHead: body.slice(0, 400),
+    };
+  });
+  const required = Boolean(info.modalVisible || info.loginText || info.captcha);
+  return {
+    ...info,
+    required,
+    reason: info.captcha ? 'captcha' : (required ? 'login' : ''),
+  };
+}
+
+async function saveGateScreenshot(page, outDir) {
+  try {
+    ensureDir(outDir);
+    const file = path.join(outDir, 'gate.jpg');
+    await page.screenshot({ path: file, type: 'jpeg', quality: 85, fullPage: false });
+    return file;
+  } catch {
+    return '';
+  }
+}
+
+async function gateResult(page, outDir, extra = {}) {
+  const gate = await detectGate(page);
+  if (!gate.required) return null;
+  const screenshot = await saveGateScreenshot(page, outDir);
+  return {
+    ok: true,
+    login_required: true,
+    reason: gate.reason,
+    screenshot,
+    url: gate.url,
+    title: gate.title,
+    body_head: gate.bodyHead,
+    ...extra,
+  };
+}
+
+function printJson(value) {
+  process.stdout.write(JSON.stringify(value) + '\n');
+}
+
+function fail(err) {
+  printJson({
+    ok: false,
+    error: err && err.message ? err.message : String(err),
+  });
+}
+
+module.exports = {
+  log,
+  sleep,
+  parseArgs,
+  requirePlaywright,
+  ensureDir,
+  readJson,
+  writeJsonAtomic,
+  connect,
+  restoreCookies,
+  saveCookies,
+  detectGate,
+  saveGateScreenshot,
+  gateResult,
+  printJson,
+  fail,
+};

package/plugins/fancy-douyin/scripts/scan-account.cjs

@@ -0,0 +1,128 @@
+'use strict';
+
+/**
+ * Scan one Douyin account homepage and return its /note/ image-text works.
+ *
+ * stdout is one JSON object. Progress goes to stderr. The script never closes
+ * Chrome; it closes only its own page, so login state stays in the profile.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const {
+  log, parseArgs, connect, restoreCookies, saveCookies,
+  gateResult, printJson, fail, ensureDir,
+} = require('./lib.cjs');
+
+function normalize(url) {
+  if (String(url).startsWith('//')) return 'https:' + String(url);
+  return String(url);
+}
+
+async function collectCards(page) {
+  const cards = new Map();
+  let stable = 0;
+  for (let round = 1; round <= 30; round += 1) {
+    const batch = await page.evaluate(() => {
+      const rows = [];
+      for (const a of document.querySelectorAll('a[href*="/note/"], a[href*="/video/"]')) {
+        const li = a.closest('li') || a.parentElement;
+        const img = li ? li.querySelector('img') : null;
+        rows.push({
+          href: a.href,
+          kind: a.href.includes('/note/') ? 'note' : 'video',
+          text: li && li.innerText ? li.innerText.trim().slice(0, 300) : '',
+          alt: img && img.alt ? img.alt : '',
+          cover: img && img.src ? img.src : '',
+        });
+      }
+      return rows;
+    });
+    const before = cards.size;
+    for (const card of batch) {
+      const href = normalize(card.href);
+      if (!cards.has(href)) cards.set(href, { ...card, href });
+    }
+    log('[scan] round ' + round + ': ' + cards.size + ' cards');
+    if (cards.size === before) stable += 1;
+    else stable = 0;
+    if (stable >= 4 && cards.size > 0) break;
+    await page.evaluate(() => {
+      const candidates = [...document.querySelectorAll('div')].filter((el) => {
+        const style = getComputedStyle(el);
+        return style.overflowY === 'auto'
+          && el.scrollHeight > el.clientHeight + 100
+          && el.clientHeight > 500;
+      });
+      const target = candidates.sort((a, b) => b.clientHeight - a.clientHeight)[0];
+      if (target) target.scrollTop = target.scrollHeight;
+      else window.scrollTo(0, document.body.scrollHeight);
+    });
+    await page.waitForTimeout(2200 + Math.floor(Math.random() * 1600));
+  }
+  return [...cards.values()];
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const workspace = path.resolve(String(args.workspace || process.cwd()));
+  const account = String(args.account || '');
+  const cdp = String(args.cdp || 'http://127.0.0.1:9222');
+  const outDir = path.resolve(String(args.out || path.join(workspace, '.fancy', 'douyin', 'output', 'scan')));
+  const loginState = path.resolve(String(args['login-state'] || path.join(workspace, '.fancy', 'douyin', 'login-state.json')));
+  if (!account) throw new Error('缺少 --account 主页地址');
+  ensureDir(outDir);
+
+  const { context } = await connect(cdp);
+  let page = null;
+  let keepPage = false;
+  try {
+    await restoreCookies(context, loginState);
+    page = await context.newPage();
+    await page.goto(account, { waitUntil: 'domcontentloaded', timeout: 60000 });
+    await page.waitForTimeout(3500);
+
+    const firstGate = await gateResult(page, outDir);
+    if (firstGate) {
+      keepPage = true;
+      await saveCookies(context, loginState, page.url());
+      printJson(firstGate);
+      return;
+    }
+
+    const cards = await collectCards(page);
+    const afterGate = await gateResult(page, outDir);
+    if (afterGate) {
+      keepPage = true;
+      await saveCookies(context, loginState, page.url());
+      printJson(afterGate);
+      return;
+    }
+
+    await saveCookies(context, loginState, page.url());
+    const cardsFile = path.join(outDir, 'profile_cards.json');
+    fs.writeFileSync(cardsFile, JSON.stringify(cards, null, 2) + '\n', 'utf8');
+
+    const notes = cards.filter((card) => card.kind === 'note');
+    const videos = cards.filter((card) => card.kind === 'video');
+    printJson({
+      ok: true,
+      login_required: false,
+      account,
+      title: await page.title(),
+      notes,
+      videos: videos.length,
+      cards_file: cardsFile,
+    });
+  } finally {
+    if (page && !keepPage) {
+      try { await page.close(); } catch { /* ignore */ }
+    }
+    // Deliberately do not call browser.close(): Chrome stays open for the
+    // next note and preserves the workspace-local login profile.
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => { fail(err); process.exit(0); });