@cloud411716/fancy-webnovel 2.2.0 → 3.0.0

package/plugins/fancy-douyin/index.js

@@ -0,0 +1,551 @@
+/**
+ * fancy-douyin — crawl Douyin image-text works into the material library.
+ *
+ * Commands:
+ *   /fancy-douyin                  crawl every recorded account
+ *   /fancy-douyin add <主页地址>    record an account
+ *   /fancy-douyin list             list recorded accounts
+ *   /fancy-douyin remove <序号|地址> remove an account
+ *   /fancy-douyin help
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { RecordId } from 'surrealdb';
+
+import { askUserQuestion, llmFollowup, notify, spawnScript } from '#infra';
+import { readJsonFile, resolveWorkspace } from '#workspace';
+import { connectMaterial, ensureMaterialDatabase } from '#material-db';
+import { deployMaterialTools } from '../fancy-material/tool-deploy.js';
+import { MATERIAL_KINDS } from '../fancy-material/kinds.js';
+import { makeWaitForTurn } from '../fancy-story-builder/writing-driver.js';
+import { deployDouyinTools } from './deploy.js';
+import {
+  accountListText,
+  accountSlug,
+  addAccount,
+  douyinPaths,
+  ensureDouyinDir,
+  markCrawled,
+  noteIdFromUrl,
+  readAccounts,
+  readCrawled,
+  removeAccount,
+} from './state.js';
+import { analysisPrompt, helpText, loginQuestion, summaryText } from './templates.js';
+
+export const inject = ['commands', 'userQuestions'];
+
+const ANALYSIS_TIMEOUT_MS = 20 * 60 * 1000;
+const ANALYSIS_IDLE_GRACE_MS = 60 * 1000;
+const SCRIPT_TIMEOUT_MS = 4 * 60 * 1000;
+const LOGIN_RETRIES = 5;
+const NOTE_DELAY_MIN_MS = 6000;
+const NOTE_DELAY_MAX_MS = 12000;
+const DUPLICATE_RE = /already exists|already contains|duplicate|unique|index .*contains/i;
+
+function cwdOf(session) {
+  return session && session.header && session.header.cwd ? session.header.cwd : null;
+}
+
+function workspaceOf(session) {
+  return resolveWorkspace({ cwd: cwdOf(session), required: true });
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function abortRequested(invocation) {
+  return Boolean(invocation && invocation.signal && invocation.signal.aborted);
+}
+
+function stampOfNow() {
+  return new Date().toISOString().replace(/[-:T+.]/g, '').slice(0, 14);
+}
+
+function parseToolOutput(result) {
+  try {
+    return JSON.parse(String(result && result.output || ''));
+  } catch {
+    return null;
+  }
+}
+
+async function runTool(toolPath, args, { root, nodePath, timeout = SCRIPT_TIMEOUT_MS }) {
+  const env = { ...process.env };
+  if (nodePath) {
+    env.NODE_PATH = [nodePath, env.NODE_PATH || ''].filter(Boolean).join(path.delimiter);
+  }
+  const result = await spawnScript(process.execPath, [toolPath, ...args], {
+    timeout,
+    cwd: root,
+    env,
+  });
+  return { result, json: parseToolOutput(result) };
+}
+
+function hasDouyinLogin(paths) {
+  const data = readJsonFile(paths.loginState);
+  const cookies = Array.isArray(data && data.cookies) ? data.cookies : [];
+  const authNames = new Set(['sessionid', 'sessionid_ss', 'passport_auth_status', 'sid_tt', 'sid_guard']);
+  return cookies.some((cookie) => cookie && authNames.has(cookie.name));
+}
+
+async function startChrome(root, tools, paths, mode) {
+  const started = await runTool(tools.scripts.start, [
+    '--workspace', root,
+    '--config', paths.chrome,
+    '--profile', paths.profile,
+    '--mode', mode,
+  ], {
+    root,
+    nodePath: tools.nodePath,
+    timeout: 90000,
+  });
+  if (!started.json || started.json.ok !== true) {
+    return {
+      ok: false,
+      error: (started.json && started.json.error)
+        || (started.result && (started.result.stderr || started.result.error))
+        || '未知错误',
+    };
+  }
+  return {
+    ok: true,
+    cdp: started.json.cdp,
+    mode: started.json.mode || mode,
+  };
+}
+
+async function stopChrome(root, tools, paths) {
+  try {
+    await runTool(tools.scripts.start, [
+      '--workspace', root,
+      '--config', paths.chrome,
+      '--stop',
+    ], {
+      root,
+      nodePath: tools.nodePath,
+      timeout: 30000,
+    });
+  } catch { /* best effort */ }
+}
+
+async function runWithLogin(ctx, invocation, session, root, toolPath, args, outDir, paths, label, nodePath, browser, tools) {
+  for (let attempt = 1; attempt <= LOGIN_RETRIES; attempt += 1) {
+    const { result, json } = await runTool(toolPath, args, { root, nodePath });
+    if (json && json.login_required) {
+      if (browser.mode === 'headless') {
+        // Headless cannot show a login UI. Close it, start headed Chrome and
+        // retry the same step so the user can complete the login.
+        const switched = await startChrome(root, tools, paths, 'headed');
+        if (!switched.ok) return { ok: false, error: '切换到有头 Chrome 失败：' + switched.error };
+        browser.mode = 'headed';
+        browser.cdp = switched.cdp;
+        const cdpIndex = args.indexOf('--cdp');
+        if (cdpIndex >= 0) args[cdpIndex + 1] = browser.cdp;
+        continue;
+      }
+      const asked = await askUserQuestion(ctx, invocation, [loginQuestion(paths)]);
+      if (!asked.ok) {
+        return { ok: false, error: asked.failure?.message || '提问失败' };
+      }
+      const selected = asked.answer?.answers?.[0]?.selected?.[0];
+      if (selected !== '已登陆') {
+        return { ok: false, cancelled: true, error: '已取消' };
+      }
+      await sleep(1200);
+      continue;
+    }
+    if (!json) {
+      return {
+        ok: false,
+        error: (result.stderr || result.error || '脚本没有返回 JSON').trim(),
+      };
+    }
+    if (!result.ok || json.ok !== true) {
+      return { ok: false, error: json.error || result.error || (label + '失败') };
+    }
+    return { ok: true, json };
+  }
+  return { ok: false, error: '连续 ' + LOGIN_RETRIES + ' 次检测到登录/验证，已停止' };
+}
+
+function normalizeOtherFile(file) {
+  const text = fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '');
+  const out = [];
+  for (const line of text.split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch (err) {
+      throw new Error('JSON 解析失败：' + (err && err.message ? err.message : String(err)));
+    }
+    if (!record || typeof record !== 'object' || Array.isArray(record)) {
+      throw new Error('JSONL 行不是对象');
+    }
+    record.source = 'other';
+    out.push(JSON.stringify(record));
+  }
+  if (!out.length) throw new Error('LLM 没有写入任何素材');
+  fs.writeFileSync(file, out.join('\n') + '\n', 'utf8');
+  return out.length;
+}
+
+async function validateOtherFile(file, validateScript) {
+  const result = await spawnScript(process.execPath, [validateScript, file], { timeout: 60000 });
+  if (result.code !== 0) {
+    return {
+      ok: false,
+      detail: (result.stderr || result.output || result.error || '格式校验失败').trim(),
+    };
+  }
+  return { ok: true, detail: '' };
+}
+
+function readJsonl(file) {
+  return fs.readFileSync(file, 'utf8')
+    .replace(/^\uFEFF/, '')
+    .split(/\r?\n/)
+    .filter((line) => line.trim())
+    .map((line) => JSON.parse(line));
+}
+
+export async function importOtherMaterials(root, file) {
+  const records = readJsonl(file);
+  const { db } = await connectMaterial(root);
+  let inserted = 0;
+  let skipped = 0;
+  try {
+    for (const record of records) {
+      try {
+        await db.create(new RecordId('material', record.id)).content({
+          source: 'other',
+          kind: record.kind,
+          name: record.name,
+          tags: record.tags,
+          genres: record.genres,
+          note: typeof record.note === 'string' ? record.note : '',
+        });
+        inserted += 1;
+      } catch (err) {
+        if (DUPLICATE_RE.test(err && err.message ? err.message : String(err))) {
+          skipped += 1;
+          continue;
+        }
+        throw err;
+      }
+    }
+  } finally {
+    try { await db.close(); } catch { /* ignore */ }
+  }
+  return { inserted, skipped };
+}
+
+function retryPrompt(target, detail) {
+  return [
+    '上一批素材文件没有通过校验。',
+    '',
+    '目标文件：`' + target + '`',
+    '',
+    '校验错误：',
+    '```text',
+    String(detail || '').trim(),
+    '```',
+    '',
+    '请只修正这些问题，重新写入同一个目标文件。',
+    '保持 source 为 other；不要删除已经正确的素材，也不要输出 Markdown。',
+    '写完后运行 validate.js，只回复：已写入 ' + target,
+  ].join('\n');
+}
+
+async function analyzeNote(ctx, invocation, session, {
+  root, note, screenshots, target, materialTools, paths,
+}) {
+  const prompt = analysisPrompt({
+    accountUrl: note.account_url,
+    note,
+    screenshots,
+    targetFile: target,
+    workspace: root,
+    materialKinds: MATERIAL_KINDS,
+    queryScript: materialTools.queryScript,
+    validateScript: materialTools.validateScript,
+  });
+  const waitForTurn = makeWaitForTurn(invocation, {
+    timeoutMs: ANALYSIS_TIMEOUT_MS,
+    idleGraceMs: ANALYSIS_IDLE_GRACE_MS,
+    signal: invocation.signal,
+    changed: (files, before) => {
+      try {
+        return fs.readFileSync(files[0], 'utf8') !== before;
+      } catch {
+        return true;
+      }
+    },
+  });
+
+  let detail = '';
+  for (let attempt = 1; attempt <= 2; attempt += 1) {
+    let before = null;
+    try { before = fs.readFileSync(target, 'utf8'); } catch { /* new file */ }
+    const sent = await llmFollowup(invocation, attempt === 1 ? prompt : retryPrompt(target, detail), {
+      appendTempRule: false,
+    });
+    if (!sent || sent.ok !== true) {
+      return { ok: false, error: '提示词发送失败：' + ((sent && sent.error) || '未知错误') };
+    }
+    const waited = await waitForTurn([target], before);
+    if (!waited.ok && waited.code === 'ABORTED') {
+      return { ok: false, aborted: true, error: '用户中断' };
+    }
+    if (!waited.ok && waited.code === 'TIMEOUT') {
+      return { ok: false, fatal: true, error: waited.reason || '等待 LLM 超时' };
+    }
+    if (!waited.ok && waited.code !== 'NO_CHANGE') {
+      detail = waited.reason || waited.code || '等待 LLM 写入失败';
+      continue;
+    }
+    if (!waited.ok && waited.code === 'NO_CHANGE') {
+      detail = 'LLM 本轮结束但没有写入目标文件';
+      continue;
+    }
+
+    try {
+      normalizeOtherFile(target);
+    } catch (err) {
+      detail = err && err.message ? err.message : String(err);
+      continue;
+    }
+    const checked = await validateOtherFile(target, materialTools.validateScript);
+    if (!checked.ok) {
+      detail = checked.detail;
+      continue;
+    }
+    try {
+      const result = await importOtherMaterials(root, target);
+      return { ok: true, ...result };
+    } catch (err) {
+      return { ok: false, error: '导入数据库失败：' + (err && err.message ? err.message : String(err)) };
+    }
+  }
+  return { ok: false, error: '素材没有通过校验：' + detail };
+}
+
+async function runCrawl(ctx, invocation, session) {
+  const root = workspaceOf(session);
+  const paths = ensureDouyinDir(root);
+  const accounts = readAccounts(root);
+  if (!accounts.length) {
+    return notify(session, 'error', '还没有记录抖音账号。先运行：/fancy-douyin add <主页地址>');
+  }
+
+  const tools = deployDouyinTools(root);
+  if (!tools.ok) {
+    return notify(session, 'error', '❌ 部署抖音脚本失败：\n' + (tools.problems.join('\n') || '未知错误'));
+  }
+  await ensureMaterialDatabase(root);
+  const materialTools = deployMaterialTools(root);
+  if (!materialTools.ok) {
+    return notify(session, 'error', '❌ 部署素材校验脚本失败：\n' + (materialTools.problems.join('\n') || '未知错误'));
+  }
+
+  const mode = hasDouyinLogin(paths) ? 'headless' : 'headed';
+  const started = await startChrome(root, tools, paths, mode);
+  if (!started.ok) {
+    return notify(session, 'error', '❌ 启动 Chrome 失败：' + started.error);
+  }
+  const browser = { cdp: started.cdp, mode: started.mode };
+
+  try {
+  const runDir = path.join(paths.output, stampOfNow());
+  fs.mkdirSync(runDir, { recursive: true });
+  const crawled = readCrawled(root);
+  const failures = [];
+  let scannedAccounts = 0;
+  let foundNotes = 0;
+  let skippedNotes = 0;
+  let processedNotes = 0;
+  let inserted = 0;
+  let skippedMaterials = 0;
+  let aborted = false;
+
+  outer:
+  for (const account of accounts) {
+    if (abortRequested(invocation)) { aborted = true; break; }
+    const accountDir = path.join(runDir, accountSlug(account.url));
+    const scan = await runWithLogin(
+      ctx, invocation, session, root, tools.scripts.scan,
+      ['--workspace', root, '--account', account.url, '--cdp', browser.cdp,
+        '--out', accountDir, '--login-state', paths.loginState],
+      accountDir,
+      paths,
+      '扫描账号',
+      tools.nodePath,
+      browser,
+      tools,
+    );
+    if (!scan.ok) {
+      failures.push({ url: account.url, reason: scan.error });
+      if (scan.cancelled) { aborted = true; break; }
+      continue;
+    }
+
+    scannedAccounts += 1;
+    const notes = Array.isArray(scan.json.notes) ? scan.json.notes : [];
+    foundNotes += notes.length;
+    for (const card of notes) {
+      if (abortRequested(invocation)) { aborted = true; break outer; }
+      const noteId = noteIdFromUrl(card.href);
+      if (!noteId) continue;
+      if (crawled[noteId]) {
+        skippedNotes += 1;
+        continue;
+      }
+      const noteDir = path.join(accountDir, noteId);
+      const noteResult = await runWithLogin(
+        ctx, invocation, session, root, tools.scripts.note,
+        ['--workspace', root, '--note', card.href, '--cdp', browser.cdp,
+          '--out', noteDir, '--login-state', paths.loginState],
+        noteDir,
+        paths,
+        '采集作品',
+        tools.nodePath,
+        browser,
+        tools,
+      );
+      if (!noteResult.ok) {
+        failures.push({ id: noteId, url: card.href, reason: noteResult.error });
+        if (noteResult.cancelled) { aborted = true; break outer; }
+        continue;
+      }
+      const note = noteResult.json.note;
+      if (!note || !Array.isArray(note.screenshots) || !note.screenshots.length) {
+        failures.push({ id: noteId, url: card.href, reason: '没有截图' });
+        continue;
+      }
+      processedNotes += 1;
+      note.account_url = account.url;
+      const target = path.join(noteDir, 'materials.jsonl');
+      const analyzed = await analyzeNote(ctx, invocation, session, {
+        root,
+        note,
+        screenshots: note.screenshots,
+        target,
+        materialTools,
+        paths,
+      });
+      if (!analyzed.ok) {
+        failures.push({ id: noteId, url: card.href, reason: analyzed.error });
+        if (analyzed.aborted) { aborted = true; break outer; }
+        if (analyzed.fatal) break outer;
+        continue;
+      }
+      inserted += analyzed.inserted;
+      skippedMaterials += analyzed.skipped;
+      markCrawled(root, noteId, {
+        account_url: account.url,
+        url: note.url || card.href,
+        title: note.title || card.text || '',
+        screenshots: note.screenshots.map((shot) => shot.file),
+        dir: noteDir,
+        inserted: analyzed.inserted,
+        skipped: analyzed.skipped,
+      });
+      crawled[noteId] = true;
+      if (NOTE_DELAY_MIN_MS < NOTE_DELAY_MAX_MS) {
+        const delay = NOTE_DELAY_MIN_MS + Math.floor(Math.random() * (NOTE_DELAY_MAX_MS - NOTE_DELAY_MIN_MS));
+        await sleep(delay);
+      }
+    }
+  }
+
+  return notify(session, (aborted || failures.length) ? 'error' : 'success', summaryText({
+    accounts: accounts.length,
+    scannedAccounts,
+    foundNotes,
+    skippedNotes,
+    processedNotes,
+    inserted,
+    skippedMaterials,
+    failures,
+    outputDir: runDir,
+    stateFile: paths.crawled,
+    aborted,
+    mode: browser.mode,
+  }));
+  } finally {
+    await stopChrome(root, tools, paths);
+  }
+}
+
+async function runAdd(session, rawUrl) {
+  const root = workspaceOf(session);
+  try {
+    const result = addAccount(root, rawUrl);
+    if (!result.added) {
+      return notify(session, 'success', '账号已存在：' + result.url);
+    }
+    return notify(session, 'success', [
+      '✅ 已添加抖音账号：' + result.url,
+      '运行 /fancy-douyin 开始采集。',
+    ].join('\n'));
+  } catch (err) {
+    return notify(session, 'error', '❌ ' + (err && err.message ? err.message : String(err)));
+  }
+}
+
+async function runList(session) {
+  const root = workspaceOf(session);
+  return notify(session, 'success', accountListText(root));
+}
+
+async function runRemove(session, selector) {
+  const root = workspaceOf(session);
+  try {
+    const result = removeAccount(root, selector);
+    return notify(session, 'success', '✅ 已删除账号：' + result.url);
+  } catch (err) {
+    return notify(session, 'error', '❌ ' + (err && err.message ? err.message : String(err)));
+  }
+}
+
+export async function apply(ctx) {
+  ctx.commands.register({
+    name: 'fancy-douyin',
+    description: '🎬 抖音图文采集 ( usage: /fancy-douyin [add <主页> | list | remove <序号|地址> | help] )',
+    input: { hint: '[add <主页> | list | remove <序号|地址> | help]' },
+    handler: async (invocation) => {
+      const session = invocation.agent.session;
+      const raw = String(invocation.rawInput || '').trim();
+      const tokens = raw.split(/\s+/).filter(Boolean);
+      const command = (tokens[0] || '').toLowerCase();
+
+      if (command === 'help' || command === '?') {
+        return notify(session, 'success', helpText());
+      }
+      try {
+        if (command === 'add') {
+          if (tokens.length < 2) return notify(session, 'error', '用法：/fancy-douyin add <抖音账号主页地址>');
+          return await runAdd(session, tokens.slice(1).join(' '));
+        }
+        if (command === 'list') {
+          if (tokens.length > 1) return notify(session, 'error', '用法：/fancy-douyin list（不接受额外参数）');
+          return await runList(session);
+        }
+        if (command === 'remove') {
+          if (tokens.length < 2) return notify(session, 'error', '用法：/fancy-douyin remove <序号|主页地址>');
+          return await runRemove(session, tokens.slice(1).join(' '));
+        }
+        if (command) {
+          return notify(session, 'error', '未知子命令：' + command + '\n运行 /fancy-douyin help 查看用法。');
+        }
+        return await runCrawl(ctx, invocation, session);
+      } catch (err) {
+        return notify(session, 'error', '❌ 抖音命令失败：' + (err && err.message ? err.message : String(err)));
+      }
+    },
+  });
+}

package/plugins/fancy-douyin/prompts/01-analysis-prompt.md

@@ -0,0 +1,84 @@
+你正在分析一篇抖音图文作品，把图片里的内容拆成可以直接进入网文素材库的小积木。
+
+## 作品信息
+
+- 账号主页：{{ACCOUNT_URL}}
+- 作品地址：{{NOTE_URL}}
+- 作品标题：{{NOTE_TITLE}}
+- 作品描述：{{NOTE_DESCRIPTION}}
+- 页面正文摘录：{{NOTE_BODY}}
+- 截图目录：`{{NOTE_DIR}}`
+
+## 截图
+
+{{SCREENSHOT_LIST}}
+
+## 任务
+
+读取上面的截图，提取其中可复用的网文创作素材：
+
+- 设定、等级体系、世界观规则；
+- 能力、代价、限制、反制；
+- 物品、材料、功能、视觉特征；
+- 地名、势力、组织、身份；
+- 普通怪物、神话生物、神祇、上古凶兽；
+- 人物外形、气质、性格、习惯、说话风格、出身；
+- 场景、天气、感官、描写词；
+- 脑洞、冲突种子、反转、套路。
+
+不要复述整篇文案，不要把原图里的整段文字原样抄进来。把内容抽象成一条条可以独立取用的素材。
+
+## 输出文件
+
+把 JSONL 写到：
+
+```text
+{{TARGET_FILE}}
+```
+
+一行一个 JSON 对象，不要 Markdown 代码围栏、标题、解释、前后缀。
+
+字段只能是：
+
+```text
+id / kind / name / tags / genres / note / source
+```
+
+- `source` 固定写 `other`。
+- `kind` 不受内置列表限制；如果内容能归入下面的内置 kind，优先使用内置 kind。
+- `tags` 3–8 个短标签。
+- `genres` 1–3 个题材；拿不准就写 `["通用"]`。
+- `note` 可写一句补充，没有就写空字符串。
+- `id` 用小写字母、数字、下划线，格式建议：`<kind>_dy_<作品ID>_<序号>`，全局唯一。
+
+优先使用的内置 kind：
+
+```text
+{{MATERIAL_KINDS}}
+```
+
+## 质量要求
+
+- 具体优先：一条素材必须能在 30 秒内变成一个具体画面、动作、规则或人物关系。
+- 拒绝空泛词：不要“火红、耀眼、神秘、强大、古老、传说、绝世”。
+- 命名必须有具体锚点：数字、地名、材质、动作、职业、年代、禁忌、功能或反差。
+- 拒绝“复原眼、章眼、读心手、旧仓单、神秘剑、黑色长刀”这类废名。
+- 一条只写一个细节；机制、代价、限制、反制分开成条。
+- 不要为了凑数复制、换皮或降低质量；宁可不写，也不要写废名。
+- 通常一篇作品提取 5–40 条，按截图实际信息量决定。
+
+生成前可以查询已有素材，避免同名：
+
+```bash
+node '{{QUERY_SCRIPT}}' --workspace '{{WORKSPACE}}' --kind <kind> --name '<候选名>' --json
+```
+
+写完运行格式校验：
+
+```bash
+node '{{VALIDATE_SCRIPT}}' '{{TARGET_FILE}}'
+```
+
+校验失败就按错误修正，直到通过。
+
+写完后只回复：`已写入 {{TARGET_FILE}}`。