aitable-workflow-core 0.1.13-beta.1

package/wolai-crawler/lib/crawler.js

@@ -0,0 +1,699 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+
+const {
+  ensureDir,
+  resolveRepoPath,
+  sleep,
+  sanitizeFilename,
+  writeJson,
+} = require('./runtime');
+const { launchBrowserContext, saveStorageState } = require('./browser');
+
+async function navigateWithRetry(page, url, { timeout = 120000, retries = 2 } = {}) {
+  let lastError;
+  for (let i = 0; i <= retries; i++) {
+    try {
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
+      return;
+    } catch (err) {
+      lastError = err;
+      if (i < retries) {
+        console.log(`  ⚠ 导航失败，重试 ${i + 1}/${retries}: ${url}`);
+        await sleep(2000);
+      }
+    }
+  }
+  throw lastError;
+}
+
+/**
+ * 从当前页面提取所有 wolai 内部页面链接
+ * 多策略：先尝试 DOM 目录树，再尝试所有内部链接
+ */
+async function collectPageUrls(page, startUrl, options = {}) {
+  const { pageWaitMs = 3000, timeout = 120000 } = options;
+
+  console.log(`→ 打开目录页: ${startUrl}`);
+  await navigateWithRetry(page, startUrl, { timeout });
+  await sleep(pageWaitMs);
+
+  const result = await page.evaluate(() => {
+    const results = [];
+    const seen = new Set();
+
+    function add(url, title = '') {
+      if (!url) return;
+      let fullUrl = url;
+      if (url.startsWith('/')) {
+        fullUrl = `https://wolai.dingtalk.com${url}`;
+      }
+      if (!fullUrl.startsWith('https://wolai.dingtalk.com/')) return;
+      if (fullUrl === window.location.href) return;
+      if (seen.has(fullUrl)) return;
+      seen.add(fullUrl);
+      results.push({ url: fullUrl, title: title.trim().slice(0, 200) });
+    }
+
+    // 策略0：尝试从全局状态提取页面树
+    try {
+      const globals = [
+        window.__INITIAL_STATE__,
+        window.__DATA__,
+        window.__PRELOADED_STATE__,
+        window.__APP_STATE__,
+      ];
+      for (const state of globals) {
+        if (!state) continue;
+        const text = JSON.stringify(state);
+        const matches = text.match(/"[a-zA-Z0-9]{10,30}":/g) || [];
+        // 这里只记录可能包含页面 ID 的全局变量名，实际解析需要看到结构
+      }
+    } catch { /* ignore */ }
+
+    // 策略1：常见目录树选择器
+    const treeSelectors = [
+      '.sidebar a',
+      '.page-tree a',
+      '.catalog a',
+      '.outline a',
+      'nav a',
+      '[class*="tree"] a',
+      '[class*="catalog"] a',
+      '[class*="sidebar"] a',
+      '[class*="outline"] a',
+      '[class*="menu"] a',
+      '[class*="toc"] a',
+      '[class*="index"] a',
+      '[class*="directory"] a',
+    ];
+    for (const selector of treeSelectors) {
+      for (const el of document.querySelectorAll(selector)) {
+        const href = el.getAttribute('href');
+        const title = el.textContent || el.getAttribute('title') || '';
+        add(href, title);
+      }
+    }
+
+    // 策略2：页面中所有内部 a 标签（兜底）
+    if (results.length === 0) {
+      for (const el of document.querySelectorAll('a[href]')) {
+        const href = el.getAttribute('href');
+        const title = el.textContent || el.getAttribute('title') || '';
+        add(href, title);
+      }
+    }
+
+    return {
+      urls: results,
+      globalKeys: Object.keys(window).filter((k) => /STATE|DATA|APP|wolai/i.test(k)).slice(0, 20),
+    };
+  });
+
+  const { urls, globalKeys } = result;
+
+  if (urls.length === 0) {
+    console.log('  ⚠ 未采集到子页面链接，保存目录页快照...');
+    if (globalKeys.length > 0) {
+      console.log(`  检测到可疑全局变量: ${globalKeys.join(', ')}`);
+    }
+    await savePageSnapshot(page, startUrl, options);
+  }
+
+  console.log(`  ✓ 采集到 ${urls.length} 个子页面链接`);
+  return urls;
+}
+
+async function savePageSnapshot(page, url, options = {}) {
+  const fs = require('node:fs');
+  const { ensureDir } = require('./runtime');
+  const snapshotDir = ensureDir('.sandbox/kb-validate/.wolai-raw/debug');
+  const timestamp = Date.now();
+
+  // 截图
+  const screenshotPath = `${snapshotDir}/snapshot-${timestamp}.png`;
+  try {
+    await page.screenshot({ path: screenshotPath, fullPage: true });
+    console.log(`  截图: ${screenshotPath}`);
+  } catch (err) {
+    console.log(`  截图失败: ${err.message}`);
+  }
+
+  // 页面 HTML
+  const htmlPath = `${snapshotDir}/snapshot-${timestamp}.html`;
+  try {
+    const html = await page.content();
+    fs.writeFileSync(htmlPath, html, 'utf-8');
+    console.log(`  HTML: ${htmlPath}`);
+  } catch (err) {
+    console.log(`  HTML 保存失败: ${err.message}`);
+  }
+
+  // 元素快照
+  const elementsPath = `${snapshotDir}/snapshot-${timestamp}-elements.json`;
+  try {
+    const elements = await page.evaluate(() => {
+      return Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
+        .map((el) => ({
+          tag: el.tagName,
+          href: el.getAttribute('href'),
+          text: el.textContent?.trim().slice(0, 80),
+          ariaLabel: el.getAttribute('aria-label'),
+          title: el.getAttribute('title'),
+          className: typeof el.className === 'string' ? el.className.slice(0, 150) : '',
+        }))
+        .filter((item) => item.href || item.text || item.ariaLabel || item.title)
+        .slice(0, 300);
+    });
+    fs.writeFileSync(elementsPath, JSON.stringify(elements, null, 2), 'utf-8');
+    console.log(`  元素: ${elementsPath}`);
+  } catch (err) {
+    console.log(`  元素快照失败: ${err.message}`);
+  }
+}
+
+/**
+ * 点击右上角「⋯ → 导出页面...」导出当前页面
+ * 由于 wolai DOM 可能变化，这里使用多 selector 尝试，
+ * 失败时可在 headed 模式下由用户接管。
+ */
+async function exportPage(page, { url, title }, rawDir, options = {}) {
+  const {
+    pageWaitMs = 10000,
+    downloadWaitMs = 30000,
+    timeout = 120000,
+    headed = false,
+  } = options;
+
+  // 有头模式下给用户足够时间手动干预
+  const effectiveDownloadWaitMs = headed ? Math.max(downloadWaitMs, 1800000) : downloadWaitMs;
+
+  console.log(`→ 导出: ${title || url}`);
+  await navigateWithRetry(page, url, { timeout });
+
+  // 校验登录态：若仍处于演示模式，说明 storageState 未生效
+  const loginCheck = await detectLoginState(page);
+  if (!loginCheck.isLoggedIn) {
+    throw new Error(
+      `当前页面仍为未登录/演示模式（演示模式=${loginCheck.presentationMode}, 登录按钮=${loginCheck.loginButton}）。\n` +
+      '请确认：\n' +
+      '1. 登录态文件存在且未过期；\n' +
+      '2. 你已完成 wolai 登录/扫码；\n' +
+      '3. 你对目标页面具有导出权限。\n' +
+      '如需重新登录，请删除登录态文件后重试。',
+    );
+  }
+
+  // 等待网络空闲和页面稳定
+  console.log('  等待页面渲染...');
+  try {
+    await page.waitForLoadState('networkidle', { timeout: 20000 });
+  } catch { /* ignore */ }
+  await sleep(pageWaitMs);
+
+  // 等待页面主要内容渲染
+  try {
+    await page.waitForFunction(() => document.readyState === 'complete', { timeout: 10000 });
+  } catch { /* ignore */ }
+
+  // 检测并关闭演示模式新手引导弹窗
+  const hasPreviewGuide = await page.evaluate(() => {
+    const bodyText = document.body.innerText;
+    return bodyText.includes('了解如何使用演示模式') ||
+      bodyText.includes('将文档轻松转换成幻灯片') ||
+      (bodyText.includes('立即使用') && bodyText.includes('演示模式'));
+  }).catch(() => false);
+
+  if (hasPreviewGuide) {
+    console.log('  检测到演示模式新手引导，尝试关闭...');
+    try {
+      // 点击「立即使用」关闭引导
+      const guideCloseSelectors = [
+        'text=立即使用',
+        'button:has-text("立即使用")',
+        '.wolaiModal button:has-text("立即使用")',
+        '[class*="wolaiModal"] button:has-text("立即使用")',
+      ];
+      for (const selector of guideCloseSelectors) {
+        const btn = page.locator(selector).first();
+        if (await btn.isVisible({ timeout: 3000 }).catch(() => false)) {
+          await btn.click({ timeout: 5000 });
+          console.log(`  ✓ 关闭演示模式引导: ${selector}`);
+          await sleep(3000);
+          break;
+        }
+      }
+
+      // 有头模式下如果还没关掉，暂停让用户手动处理
+      const stillHasGuide = await page.evaluate(() => {
+        const bodyText = document.body.innerText;
+        return bodyText.includes('了解如何使用演示模式') || bodyText.includes('将文档轻松转换成幻灯片');
+      }).catch(() => false);
+
+      if (stillHasGuide && headed) {
+        console.log('  自动关闭引导失败，请在浏览器中手动关闭演示模式引导弹窗');
+        console.log('  关闭后按回车键继续...');
+        await new Promise((resolve) => {
+          process.stdin.once('data', resolve);
+        });
+      }
+    } catch (err) {
+      console.log(`  关闭引导失败: ${err.message}`);
+    }
+  }
+
+  // 每个页面独立的下载目录
+  const sessionDir = path.join(rawDir, 'downloads', `${Date.now()}_${sanitizeFilename(title || 'untitled')}`);
+  ensureDir(sessionDir);
+
+  // 监听下载（context 级别，兼容 Wolai 在新标签页/新窗口触发下载的情况）
+  const context = page.context();
+
+  const downloadedFiles = [];
+  const collectDownload = async (download) => {
+    const filePath = path.join(sessionDir, download.suggestedFilename());
+    try {
+      await download.saveAs(filePath);
+      downloadedFiles.push(filePath);
+      console.log(`  [download] 收到下载: ${path.basename(filePath)}`);
+    } catch (err) {
+      console.log(`  [download] 保存失败: ${err.message}`);
+    }
+  };
+  context.on('page', (newPage) => {
+    newPage.on('download', collectDownload);
+  });
+  page.on('download', collectDownload);
+
+  const downloadPromise = new Promise((resolve, reject) => {
+    let resolved = false;
+    const timer = setTimeout(() => {
+      if (!resolved) reject(new Error(`下载超时 (${effectiveDownloadWaitMs}ms)，未收到 download 事件`));
+    }, effectiveDownloadWaitMs);
+    const done = (filePath) => {
+      if (resolved) return;
+      resolved = true;
+      clearTimeout(timer);
+      clearInterval(interval);
+      resolve({ filePath });
+    };
+    const checkDownloaded = () => {
+      if (downloadedFiles.length > 0) done(downloadedFiles[0]);
+    };
+    const interval = setInterval(() => {
+      checkDownloaded();
+      // 兜底：扫描下载目录（即使未触发 download 事件，Playwright 也会因 setDefaultDownloadsPath 自动保存）
+      try {
+        const files = fs.readdirSync(sessionDir).filter((f) => !f.startsWith('.'));
+        if (files.length > 0 && downloadedFiles.length === 0) {
+          done(path.join(sessionDir, files[0]));
+        }
+      } catch { /* ignore */ }
+    }, 500);
+    checkDownloaded();
+  });
+
+  // 尝试点击右上角「更多」菜单
+  const moreButtonSelectors = [
+    '#tour-tool-bar > div:last-child',
+    '#tour-tool-bar svg path[d*="M8 6.8a1.2 1.2"]',
+    '[class*="liqjz"]',
+    'button[aria-label="更多"]',
+    'button[title="更多"]',
+    '.page-header .anticon-more',
+    '.page-header [class*="more"]',
+    'header [class*="more"] button',
+    '[class*="page-option"]',
+    '[class*="header-right"] button:last-child',
+  ];
+
+  // 调试：打印页面中 svg 数量和候选菜单按钮信息
+  try {
+    const debugInfo = await page.evaluate(() => {
+      const svgs = Array.from(document.querySelectorAll('svg'));
+      const candidates = svgs
+        .map((svg) => {
+          const path = svg.querySelector('path');
+          return {
+            d: path?.getAttribute('d')?.slice(0, 80) || '',
+            className: svg.closest('[class]')?.className?.toString().slice(0, 120) || '',
+          };
+        })
+        .filter((item) => item.d.includes('1.2') || item.d.includes('M8') || item.d.includes('8 6'));
+      return { svgCount: svgs.length, candidates };
+    });
+    console.log(`  [debug] 页面 svg 数量: ${debugInfo.svgCount}, 候选菜单图标: ${debugInfo.candidates.length}`);
+    debugInfo.candidates.slice(0, 10).forEach((c, i) => console.log(`    ${i}: d="${c.d}..." class="${c.className}"`));
+  } catch (err) {
+    console.log(`  [debug] 获取 svg 信息失败: ${err.message}`);
+  }
+
+  // 重试查找并点击菜单按钮，点击后验证是否出现导出菜单
+  let moreClicked = false;
+  const maxAttempts = headed ? 6 : 3;
+  for (let attempt = 0; attempt < maxAttempts && !moreClicked; attempt++) {
+    if (attempt > 0) {
+      console.log(`  第 ${attempt + 1} 次尝试定位菜单按钮...`);
+      await sleep(2000);
+    }
+
+    for (const selector of moreButtonSelectors) {
+      try {
+        const loc = page.locator(selector).first();
+        const visible = await loc.isVisible({ timeout: 3000 }).catch(() => false);
+        if (visible) {
+          await loc.click({ timeout: 5000 });
+          await sleep(1000);
+          // 验证是否出现导出菜单
+          const hasExportMenu = await page.locator('text=导出页面').first().isVisible({ timeout: 2000 }).catch(() => false);
+          if (hasExportMenu) {
+            moreClicked = true;
+            console.log(`  ✓ 点击菜单按钮: ${selector}`);
+            break;
+          }
+          console.log(`  ✗ ${selector} 点错，未出现导出菜单`);
+        }
+      } catch { /* ignore */ }
+    }
+
+    // 兜底：在 tour-tool-bar 内通过 SVG path 找三个点的更多按钮
+    if (!moreClicked) {
+      try {
+        const clicked = await page.evaluate(() => {
+          const toolbar = document.querySelector('#tour-tool-bar');
+          const svgs = toolbar ? toolbar.querySelectorAll('svg') : document.querySelectorAll('svg');
+          for (const svg of svgs) {
+            const path = svg.querySelector('path');
+            const d = path?.getAttribute('d') || '';
+            if (d.includes('M8 6.8a1.2 1.2') || d.includes('a1.2 1.2 0 1 1 0 2.4') || /M\d+\.?\d* \d+\.?\d*a1\.2 1\.2/.test(d)) {
+              // 向上找可点击父元素，优先找 d-flex 容器
+              let el = svg.closest('[class*="d-flex"]') || svg.parentElement;
+              while (el && el.tagName !== 'BUTTON' && !el.getAttribute('role') && el.tagName !== 'A') {
+                el = el.parentElement;
+              }
+              (el || svg).click();
+              return { ok: true, d: d.slice(0, 40), tag: (el || svg).tagName, className: (el || svg).className?.toString().slice(0, 80) };
+            }
+          }
+          return { ok: false };
+        });
+        if (clicked.ok) {
+          await sleep(1000);
+          const hasExportMenu = await page.locator('text=导出页面').first().isVisible({ timeout: 2000 }).catch(() => false);
+          if (hasExportMenu) {
+            moreClicked = true;
+            console.log(`  ✓ 点击菜单按钮(SVG兜底): d="${clicked.d}..." tag=${clicked.tag} class=${clicked.className}`);
+          }
+        }
+      } catch { /* ignore */ }
+    }
+  }
+
+  if (!moreClicked) {
+    await savePageSnapshot(page, url, options);
+    if (headed) {
+      console.log('  ⚠ 无法自动定位菜单按钮，请在浏览器中手动点击右上角「⋯」→「导出页面...」，脚本会继续等待下载');
+    } else {
+      throw new Error('无法定位右上角「更多」菜单按钮，建议用 --headed 查看页面结构');
+    }
+  }
+
+  // 尝试点击「导出页面...」
+  const exportMenuSelectors = [
+    'text=导出页面...',
+    'text=导出页面',
+    'text=导出当前页面',
+    '.ant-dropdown-menu-item:has-text("导出")',
+    '[class*="dropdown"] >> text=导出',
+    '[role="menuitem"] >> text=导出',
+    '[class*="_21m0P"]',
+    '[class*="_3gFMB"]',
+  ];
+
+  let exportClicked = false;
+  for (let attempt = 0; attempt < (headed ? 6 : 3) && !exportClicked; attempt++) {
+    if (attempt > 0) {
+      console.log(`  第 ${attempt + 1} 次尝试定位导出菜单...`);
+      await sleep(1500);
+    }
+
+    for (const selector of exportMenuSelectors) {
+      try {
+        const loc = page.locator(selector).first();
+        if (await loc.isVisible({ timeout: 2000 }).catch(() => false)) {
+          await loc.click({ timeout: 5000 });
+          exportClicked = true;
+          console.log(`  ✓ 点击导出菜单: ${selector}`);
+          break;
+        }
+      } catch { /* ignore */ }
+    }
+
+    // 兜底：文本匹配「导出页面...」
+    if (!exportClicked) {
+      try {
+        const clicked = await page.evaluate(() => {
+          const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+          let node;
+          while ((node = walker.nextNode())) {
+            if (node.textContent.includes('导出页面')) {
+              let el = node.parentElement;
+              while (el && el.tagName !== 'BUTTON' && el.tagName !== 'A' && el.getAttribute('role') !== 'button' && el.getAttribute('role') !== 'menuitem') {
+                el = el.parentElement;
+              }
+              if (el) {
+                el.click();
+                return true;
+              }
+            }
+          }
+          return false;
+        });
+        if (clicked) {
+          exportClicked = true;
+          console.log('  ✓ 点击导出菜单(文本兜底)');
+        }
+      } catch { /* ignore */ }
+    }
+  }
+
+  if (!exportClicked) {
+    if (headed) {
+      console.log('  ⚠ 无法自动定位导出菜单，请在浏览器中手动点击「导出页面...」，脚本会继续等待下载');
+    } else {
+      throw new Error('无法定位「导出页面...」菜单项，建议用 --headed 查看页面结构');
+    }
+  }
+
+  // 等待导出面板，选择 Markdown 格式，然后点击「导出」
+  try {
+    await page.locator('.wolaiModal, [class*="wolaiModal"]').waitFor({ state: 'visible', timeout: 8000 });
+
+    // 检查当前格式，如果不是 Markdown 则切换
+    const currentFormat = await page.locator('.wolaiModal [class*="_2TgYE"]').first().textContent().catch(() => '');
+    console.log(`  当前导出格式: ${currentFormat}`);
+
+    if (!currentFormat.includes('Markdown')) {
+      // 点击格式下拉
+      const formatDropdown = page.locator('.wolaiModal button:has([class*="_2TgYE"]), .wolaiModal [class*="gsQNQ"] button').first();
+      await formatDropdown.click({ timeout: 5000 });
+      await sleep(800);
+
+      // 选择 Markdown
+      const markdownOption = page.locator('.wolaiModal [class*="MuiMenuItem-root"], [role="listbox"] [role="option"]').filter({ hasText: /Markdown/i }).first();
+      await markdownOption.click({ timeout: 5000 });
+      console.log('  ✓ 切换导出格式为 Markdown');
+      await sleep(800);
+    }
+
+    const exportBtn = page.locator('.wolaiModal button:has-text("导出"), .wolaiModal [class*="_3sMqb"]').first();
+    await exportBtn.click({ timeout: 5000 });
+    console.log('  ✓ 点击导出面板确认按钮');
+  } catch (err) {
+    if (!headed) {
+      throw new Error(`导出面板确认失败: ${err.message}`);
+    }
+    console.log('  ⚠ 未自动点击导出面板确认按钮，请在浏览器中手动完成');
+  }
+
+  // 等待下载完成
+  const { filePath } = await downloadPromise;
+  console.log(`  ✓ 下载完成: ${path.basename(filePath)}`);
+
+  // 如果是 zip，解压到 sessionDir 并删除 zip
+  if (filePath.toLowerCase().endsWith('.zip')) {
+    const extractDir = path.join(sessionDir, 'extracted');
+    ensureDir(extractDir);
+    try {
+      // 使用 execFileSync 数组传参，避免文件名（服务端可控）被 shell 解析导致命令注入
+      execFileSync('unzip', ['-o', filePath, '-d', extractDir], { stdio: 'ignore' });
+      console.log(`  ✓ 解压完成: ${extractDir}`);
+      fs.unlinkSync(filePath);
+      console.log(`  ✓ 删除 zip: ${path.basename(filePath)}`);
+      return { filePath: extractDir, sessionDir, isZip: true };
+    } catch (err) {
+      console.log(`  ✗ 解压失败: ${err.message}`);
+      return { filePath, sessionDir, isZip: false };
+    }
+  }
+
+  return { filePath, sessionDir, isZip: false };
+}
+
+async function detectLoginState(page) {
+  // 演示模式按钮是未登录/访客态的标志
+  const presentationMode = await page.locator('text=演示模式').count().catch(() => 0);
+  const loginButton = await page.locator('text=登录').count().catch(() => 0);
+  return { isLoggedIn: presentationMode === 0 && loginButton === 0, presentationMode, loginButton };
+}
+
+async function loginAndSaveStorageState(config, { loginWaitMs = null } = {}) {
+  const { storageState, timeoutMs = 120000, loginUrl = 'https://wolai.dingtalk.com/' } = config;
+  // 登录必须开有头模式，否则用户看不到浏览器窗口
+  const { browser, context, page } = await launchBrowserContext({ storageState: null, headed: true });
+
+  try {
+    console.log('→ 打开 wolai 登录页...');
+    await navigateWithRetry(page, loginUrl, { timeout: timeoutMs });
+    console.log('请在本机弹出的浏览器窗口中完成登录/扫码。');
+
+    if (process.stdin.isTTY) {
+      console.log('登录成功后，请回到本终端按回车键保存登录态。');
+      if (loginWaitMs) {
+        console.log(`（若 ${loginWaitMs}ms 内未按回车，将自动保存。）`);
+      }
+      const waiters = [
+        new Promise((resolve) => {
+          process.stdin.once('data', resolve);
+        }),
+      ];
+      if (loginWaitMs) waiters.push(sleep(loginWaitMs));
+      await Promise.race(waiters);
+    } else {
+      const waitTime = loginWaitMs ?? 30000;
+      console.log(`将在 ${waitTime}ms 后自动保存登录态（可用 --login-wait-ms 调整）...`);
+      await sleep(waitTime);
+    }
+
+    // 验证是否已真正登录（而非演示模式）
+    let loginCheck = await detectLoginState(page);
+    if (!loginCheck.isLoggedIn) {
+      console.log(`⚠ 当前页面仍处于未登录状态（演示模式=${loginCheck.presentationMode}, 登录按钮=${loginCheck.loginButton}）。`);
+      if (process.stdin.isTTY) {
+        console.log('请继续完成登录，直到页面右上角不再显示"演示模式"，然后按回车保存登录态。');
+        await new Promise((resolve) => {
+          process.stdin.once('data', resolve);
+        });
+        loginCheck = await detectLoginState(page);
+      }
+      if (!loginCheck.isLoggedIn) {
+        throw new Error('登录态验证失败：页面仍处于演示模式。请确认已完成登录/扫码，且对目标页面具有导出权限。');
+      }
+    }
+
+    const savedPath = await saveStorageState(context, storageState);
+    console.log(`✓ 登录态已保存: ${savedPath}`);
+  } finally {
+    await context.close();
+    await browser.close();
+  }
+}
+
+async function runCrawl(config, { force = false, headed = false } = {}) {
+  const rawDir = ensureDir(config.rawDir);
+  const storageState = config.storageState;
+  const skipExisting = config.skipExisting && !force;
+
+  const { browser, context, page } = await launchBrowserContext({
+    storageState,
+    headed: headed || config.headed,
+  });
+
+  const results = {
+    startedAt: new Date().toISOString(),
+    pages: [],
+    errors: [],
+  };
+
+  try {
+    // 1. 采集目录
+    const pageUrls = await collectPageUrls(page, config.startUrl, config);
+    writeJson(config.urlMappingFile, pageUrls);
+
+    // 2. 逐个导出
+    for (let i = 0; i < pageUrls.length; i++) {
+      const item = pageUrls[i];
+      const existing = findExistingRaw(item, rawDir);
+      if (skipExisting && existing) {
+        console.log(`⏭ 跳过（已存在）: ${item.title || item.url}`);
+        results.pages.push({ ...item, skipped: true, rawPath: existing });
+        continue;
+      }
+
+      try {
+        const { filePath } = await exportPage(page, item, rawDir, {
+          ...config,
+          headed: headed || config.headed,
+        });
+        results.pages.push({ ...item, filePath });
+      } catch (err) {
+        console.error(`  ✗ 导出失败: ${item.url} - ${err.message}`);
+        results.errors.push({ url: item.url, error: err.message });
+      }
+
+      if (config.cooldownMs) {
+        await sleep(config.cooldownMs);
+      }
+    }
+  } finally {
+    await context.close();
+    await browser.close();
+  }
+
+  results.finishedAt = new Date().toISOString();
+  const reportPath = path.join(rawDir, `crawl-report-${Date.now()}.json`);
+  writeJson(reportPath, results);
+  console.log(`\n📄 爬取报告: ${reportPath}`);
+  console.log(`   成功: ${results.pages.filter((p) => p.filePath).length}`);
+  console.log(`   跳过: ${results.pages.filter((p) => p.skipped).length}`);
+  console.log(`   失败: ${results.errors.length}`);
+
+  return results;
+}
+
+function findExistingRaw(item, rawDir) {
+  const downloadsDir = path.join(rawDir, 'downloads');
+  if (!fs.existsSync(downloadsDir)) return null;
+
+  const candidates = [];
+  for (const session of fs.readdirSync(downloadsDir)) {
+    const sessionPath = path.join(downloadsDir, session);
+    if (!fs.statSync(sessionPath).isDirectory()) continue;
+
+    // 优先找解压后的目录
+    const extractedPath = path.join(sessionPath, 'extracted');
+    if (fs.existsSync(extractedPath) && fs.statSync(extractedPath).isDirectory()) {
+      candidates.push(extractedPath);
+      continue;
+    }
+
+    for (const file of fs.readdirSync(sessionPath)) {
+      if (file.endsWith('.md') || file.endsWith('.zip')) {
+        candidates.push(path.join(sessionPath, file));
+      }
+    }
+  }
+
+  // 按修改时间取最新
+  if (candidates.length === 0) return null;
+  candidates.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs);
+  return candidates[0];
+}
+
+module.exports = {
+  collectPageUrls,
+  exportPage,
+  loginAndSaveStorageState,
+  runCrawl,
+};