aitable-workflow-core 0.1.13-beta.1

package/wolai-crawler/lib/debug-page.js

@@ -0,0 +1,71 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { launchBrowserContext } = require('./browser');
+const { ensureDir, sleep, resolveRepoPath } = require('./runtime');
+
+async function debugPage(url, options = {}) {
+  const { headed = false, timeout = 120000 } = options;
+  const { browser, context, page } = await launchBrowserContext({ headed });
+
+  try {
+    console.log(`→ 打开: ${url}`);
+    await page.goto(url, { waitUntil: 'networkidle', timeout });
+    await sleep(3000);
+
+    const title = await page.title();
+    console.log(`  标题: ${title}`);
+    console.log(`  URL: ${page.url()}`);
+
+    // 截图
+    const screenshotDir = ensureDir('.sandbox/kb-validate/.wolai-raw/debug');
+    const screenshotPath = path.join(screenshotDir, `debug-${Date.now()}.png`);
+    await page.screenshot({ path: screenshotPath, fullPage: true });
+    console.log(`  截图: ${screenshotPath}`);
+
+    // 提取所有按钮/菜单相关元素
+    const elements = await page.evaluate(() => {
+      const items = [];
+      for (const el of document.querySelectorAll('button, [role="button"], a')) {
+        const text = el.textContent?.trim();
+        const ariaLabel = el.getAttribute('aria-label');
+        const title = el.getAttribute('title');
+        const className = el.className;
+        if ((text && text.length < 50) || ariaLabel || title) {
+          items.push({
+            tag: el.tagName,
+            text: text?.slice(0, 50),
+            ariaLabel,
+            title,
+            className: typeof className === 'string' ? className.slice(0, 100) : '',
+          });
+        }
+      }
+      return items.slice(0, 200);
+    });
+
+    const reportPath = path.join(screenshotDir, `debug-elements-${Date.now()}.json`);
+    fs.writeFileSync(reportPath, JSON.stringify(elements, null, 2), 'utf-8');
+    console.log(`  元素快照: ${reportPath}`);
+
+    // 提取所有链接
+    const links = await page.evaluate(() => {
+      return Array.from(document.querySelectorAll('a[href]'))
+        .map((a) => ({
+          href: a.getAttribute('href'),
+          text: a.textContent?.trim().slice(0, 50),
+        }))
+        .filter((item) => item.href && (item.href.includes('wolai') || item.href.startsWith('/')))
+        .slice(0, 100);
+    });
+
+    const linksPath = path.join(screenshotDir, `debug-links-${Date.now()}.json`);
+    fs.writeFileSync(linksPath, JSON.stringify(links, null, 2), 'utf-8');
+    console.log(`  链接快照: ${linksPath}`);
+  } finally {
+    await context.close();
+    await browser.close();
+  }
+}
+
+module.exports = { debugPage };

package/wolai-crawler/lib/md-processor.js

@@ -0,0 +1,223 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { ensureDir, resolveRepoPath, sanitizeFilename, writeJson } = require('./runtime');
+
+/**
+ * 从 wolai 导出包中整理出标准知识库源文件
+ *
+ * 输入：rawDir/downloads/<session>/*.md + image/
+ * 输出：outputDir/<category>/<title>.md，图片放在同级 image/ 下
+ */
+async function processRawDownloads(config) {
+  const rawDir = resolveRepoPath(config.rawDir);
+  const outputDir = resolveRepoPath(config.outputDir);
+  const urlMapping = loadUrlMapping(config.urlMappingFile);
+
+  ensureDir(outputDir);
+
+  const downloadsDir = path.join(rawDir, 'downloads');
+  if (!fs.existsSync(downloadsDir)) {
+    console.log(`⚠ 下载目录不存在，跳过后处理: ${downloadsDir}`);
+    return { processed: 0, skipped: 0, images: 0, errors: [] };
+  }
+
+  const stats = {
+    processed: 0,
+    skipped: 0,
+    images: 0,
+    errors: [],
+  };
+
+  // 遍历所有下载会话
+  const sessions = fs.readdirSync(downloadsDir).filter((name) => {
+    return fs.statSync(path.join(downloadsDir, name)).isDirectory();
+  });
+
+  for (const session of sessions) {
+    const sessionPath = path.join(downloadsDir, session);
+
+    // wolai 导出的是 zip，已解压到 extracted/ 目录
+    const extractedPath = path.join(sessionPath, 'extracted');
+    const sourcePath = fs.existsSync(extractedPath) && fs.statSync(extractedPath).isDirectory() ? extractedPath : sessionPath;
+
+    const mdFiles = findMarkdownFiles(sourcePath);
+    for (const mdPath of mdFiles) {
+      try {
+        const result = await processSingleMarkdown(mdPath, {
+          outputDir,
+          urlMapping,
+          rawImageDir: findImageDir(sourcePath),
+        });
+        if (result.updated) stats.processed++;
+        else stats.skipped++;
+        stats.images += result.imageCount || 0;
+      } catch (err) {
+        stats.errors.push({ file: mdPath, error: err.message });
+      }
+    }
+  }
+
+  const reportPath = path.join(rawDir, `process-report-${Date.now()}.json`);
+  writeJson(reportPath, { ...stats, finishedAt: new Date().toISOString() });
+  console.log(`\n📄 整理报告: ${reportPath}`);
+  console.log(`   处理: ${stats.processed}`);
+  console.log(`   跳过: ${stats.skipped}`);
+  console.log(`   图片: ${stats.images}`);
+  console.log(`   错误: ${stats.errors.length}`);
+
+  return stats;
+}
+
+async function processSingleMarkdown(mdPath, options) {
+  const { outputDir, urlMapping, rawImageDir } = options;
+  let content = fs.readFileSync(mdPath, 'utf-8');
+
+  // 1. 提取标题
+  const titleMatch = content.match(/^#\s*(.+)$/m);
+  const title = titleMatch ? titleMatch[1].trim() : path.basename(mdPath, '.md');
+
+  // 2. 找到对应 URL
+  const url = findUrlForTitle(title, urlMapping) || findUrlForFile(mdPath, urlMapping);
+
+  // 3. 确定输出分类和路径
+  const category = inferCategory(title, content);
+  const categoryDir = ensureDir(path.join(outputDir, category));
+  const safeTitle = sanitizeFilename(title);
+  const outMdPath = path.join(categoryDir, `${safeTitle}.md`);
+  const outImageDir = ensureDir(path.join(categoryDir, 'image'));
+
+  // 4. 检查是否已存在且内容相同
+  if (fs.existsSync(outMdPath)) {
+    const existing = fs.readFileSync(outMdPath, 'utf-8');
+    // 简单比较：去除来源行后的正文
+    if (normalizeForCompare(existing) === normalizeForCompare(content)) {
+      return { updated: false, imageCount: 0, outMdPath };
+    }
+  }
+
+  // 5. 迁移图片
+  let imageCount = 0;
+  if (fs.existsSync(rawImageDir)) {
+    const images = fs.readdirSync(rawImageDir);
+    for (const img of images) {
+      const srcPath = path.join(rawImageDir, img);
+      const destPath = path.join(outImageDir, img);
+      if (!fs.existsSync(destPath) || fs.statSync(srcPath).size !== fs.statSync(destPath).size) {
+        fs.copyFileSync(srcPath, destPath);
+      }
+      imageCount++;
+    }
+
+    // 修正 markdown 中图片路径
+    content = content.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, src) => {
+      if (src.startsWith('http')) return match;
+      const basename = path.basename(src);
+      // 使用相对于 md 文件的路径
+      return `![${alt}](image/${basename})`;
+    });
+  }
+
+  // 6. 添加/更新来源标注
+  if (url && !content.includes('来源：')) {
+    content = content.replace(/^#\s*.+\n/m, (match) => `${match}\n> 来源：${url}\n`);
+  }
+
+  // 7. 写入文件
+  fs.writeFileSync(outMdPath, content, 'utf-8');
+  return { updated: true, imageCount, outMdPath };
+}
+
+function findMarkdownFiles(dir) {
+  const results = [];
+  if (!fs.existsSync(dir)) return results;
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findMarkdownFiles(fullPath));
+    } else if (entry.name.endsWith('.md')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+function findImageDir(dir) {
+  if (!fs.existsSync(dir)) return null;
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === 'image') return fullPath;
+      const nested = findImageDir(fullPath);
+      if (nested) return nested;
+    }
+  }
+  return null;
+}
+
+function loadUrlMapping(urlMappingFile) {
+  const absolutePath = resolveRepoPath(urlMappingFile);
+  if (!fs.existsSync(absolutePath)) return [];
+  try {
+    return JSON.parse(fs.readFileSync(absolutePath, 'utf-8'));
+  } catch {
+    return [];
+  }
+}
+
+function findUrlForTitle(title, urlMapping) {
+  if (!urlMapping || urlMapping.length === 0) return null;
+  const exact = urlMapping.find((item) => item.title === title);
+  if (exact) return exact.url;
+  // 模糊匹配
+  const fuzzy = urlMapping.find((item) => title.includes(item.title) || item.title.includes(title));
+  return fuzzy ? fuzzy.url : null;
+}
+
+function findUrlForFile(mdPath, urlMapping) {
+  const basename = path.basename(mdPath, '.md');
+  return findUrlForTitle(basename, urlMapping);
+}
+
+function inferCategory(title, content) {
+  // 根据标题关键词推断分类，匹配现有目录结构
+  const rules = [
+    { pattern: /付费|权益|搭建服务|席位/, category: '16-付费权益' },
+    { pattern: /插件|开发者|API|脚本|字段装饰器/, category: '18-开发者指南' },
+    { pattern: /高级权限|行列权限/, category: '12-高级权限' },
+    { pattern: /应用模式|组件|列表|按钮|标签页|过滤器|视图组件|图片|透视表组件/, category: '09-应用模式' },
+    { pattern: /仪表盘|图表|TOP_N|智能总结/, category: '08-使用仪表盘' },
+    { pattern: /视图|看板|甘特|日历|画册|表格视图|查询页面|数据表/, category: '07-使用视图' },
+    { pattern: /数据连接|同步|数据源|听记|OA|待办|日程|考勤|通讯录|表格数据/, category: '06-数据连接中心' },
+    { pattern: /公式|函数/, category: '05-公式函数' },
+    { pattern: /表单/, category: '04-使用表单' },
+    { pattern: /字段|AI字段|字段类型/, category: '03-使用字段' },
+    { pattern: /自动化|工作流|webhook|Webhook|机器人|Agent|群发|HTTP|邮件|待办|日程|审批|外呼/, category: '10-自动化工作流' },
+    { pattern: /快捷键|导入|导出|模板|常见问题|快速上手|入门|行数|上限/, category: '01-快速入门' },
+    { pattern: /分组|排序|筛选|历史记录/, category: '02-基础操作' },
+  ];
+
+  for (const rule of rules) {
+    if (rule.pattern.test(title) || rule.pattern.test(content.slice(0, 500))) {
+      return rule.category;
+    }
+  }
+
+  return '99-未分类';
+}
+
+function normalizeForCompare(content) {
+  return content
+    .replace(/^---[\s\S]*?---\n?/m, '')
+    .replace(/^>\s*来源：.*\n?/m, '')
+    .replace(/\s+/g, '')
+    .trim();
+}
+
+module.exports = {
+  processRawDownloads,
+  processSingleMarkdown,
+  inferCategory,
+};

package/wolai-crawler/lib/runtime.js

@@ -0,0 +1,167 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+// 允许调用方通过 --repo-root 指定项目根目录（发布包场景下无法从 __dirname 推断）
+const explicitRepoRoot = (() => {
+  const idx = process.argv.indexOf('--repo-root');
+  if (idx !== -1 && process.argv[idx + 1]) {
+    return path.resolve(process.argv[idx + 1]);
+  }
+  const env = process.env.WOLAI_REPO_ROOT;
+  if (env) return path.resolve(env);
+  return null;
+})();
+
+function findRepoRoot() {
+  // 从 wolai-crawler 所在目录向上查找，直到遇到 monorepo 根目录的 package.json
+  let dir = __dirname;
+  for (let i = 0; i < 12; i++) {
+    const pkgPath = path.join(dir, 'package.json');
+    if (fs.existsSync(pkgPath)) {
+      try {
+        const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
+        if (pkg.name === 'aitable-workflow-monorepo' || pkg.workspaces) {
+          return dir;
+        }
+      } catch { /* ignore */ }
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  // 兜底：兼容旧位置 scripts/wolai-crawler/
+  return path.resolve(__dirname, '../../..');
+}
+
+const repoRoot = explicitRepoRoot || findRepoRoot();
+
+function resolveRepoPath(filePath) {
+  if (!filePath) return filePath;
+  if (path.isAbsolute(filePath)) return filePath;
+  const normalized = path.normalize(filePath);
+  if (normalized.startsWith('..')) {
+    throw new Error(`配置路径不能跳出仓库根目录: ${filePath}。请使用相对于项目根目录 ${repoRoot} 的路径。`);
+  }
+  return path.resolve(repoRoot, normalized);
+}
+
+function ensureDir(dirPath) {
+  const absolutePath = resolveRepoPath(dirPath);
+  if (!fs.existsSync(absolutePath)) {
+    fs.mkdirSync(absolutePath, { recursive: true });
+  }
+  return absolutePath;
+}
+
+function readJson(filePath) {
+  const absolutePath = resolveRepoPath(filePath);
+  return JSON.parse(fs.readFileSync(absolutePath, 'utf-8'));
+}
+
+function writeJson(filePath, data) {
+  const absolutePath = resolveRepoPath(filePath);
+  ensureDir(path.dirname(absolutePath));
+  fs.writeFileSync(absolutePath, JSON.stringify(data, null, 2) + '\n', 'utf-8');
+  return absolutePath;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function sanitizeFilename(name) {
+  return name
+    .replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 120);
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    config: null,
+    login: false,
+    headed: false,
+    processOnly: false,
+    force: false,
+    loginWaitMs: null,
+    url: null,
+    output: null,
+    repoRoot: null,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    switch (arg) {
+      case '--config':
+      case '-c':
+        args.config = argv[++i];
+        break;
+      case '--login':
+      case '-l':
+        args.login = true;
+        break;
+      case '--login-wait-ms':
+        args.loginWaitMs = Number(argv[++i]) || null;
+        break;
+      case '--url':
+      case '-u':
+        args.url = argv[++i];
+        break;
+      case '--output':
+      case '-o':
+        args.output = argv[++i];
+        break;
+      case '--repo-root':
+        args.repoRoot = argv[++i];
+        break;
+      case '--headed':
+      case '-H':
+        args.headed = true;
+        break;
+      case '--process-only':
+      case '-p':
+        args.processOnly = true;
+        break;
+      case '--force':
+      case '-f':
+        args.force = true;
+        break;
+      default:
+        if (arg.startsWith('--config=')) {
+          args.config = arg.slice('--config='.length);
+        } else if (arg.startsWith('--login-wait-ms=')) {
+          args.loginWaitMs = Number(arg.slice('--login-wait-ms='.length)) || null;
+        } else if (arg.startsWith('--url=')) {
+          args.url = arg.slice('--url='.length);
+        } else if (arg.startsWith('--output=')) {
+          args.output = arg.slice('--output='.length);
+        } else if (arg.startsWith('--repo-root=')) {
+          args.repoRoot = arg.slice('--repo-root='.length);
+        }
+        break;
+    }
+  }
+
+  return args;
+}
+
+function requireDependency(name) {
+  try {
+    return require(require.resolve(name, { paths: [path.resolve(__dirname, '..'), repoRoot] }));
+  } catch (err) {
+    throw new Error(`依赖 ${name} 未找到。请先运行: pnpm --filter wolai-crawler install 或 npm install`);
+  }
+}
+
+module.exports = {
+  repoRoot,
+  resolveRepoPath,
+  ensureDir,
+  readJson,
+  writeJson,
+  sleep,
+  sanitizeFilename,
+  parseArgs,
+  requireDependency,
+};

package/wolai-crawler/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "wolai-crawler",
+  "version": "1.0.0",
+  "private": true,
+  "description": "从 wolai 帮助中心自动导出 markdown 并整理为知识库源文件",
+  "type": "commonjs",
+  "scripts": {
+    "login:wolai": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json --login",
+    "login:wolai:auto": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json --login --login-wait-ms 60000",
+    "crawl": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json",
+    "crawl:headed": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json --headed",
+    "export:page": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json --url",
+    "process": "node run.js --config .sandbox/kb-validate/wolai-crawler-config.json --process-only",
+    "debug": "node debug.js",
+    "debug:headed": "node debug.js --headed"
+  },
+  "dependencies": {
+    "playwright": "^1.52.0"
+  }
+}

package/wolai-crawler/run.js

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+
+/**
+ * wolai 帮助中心爬虫
+ *
+ * 用法：
+ *   1. 登录并保存登录态：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --login
+ *   2. 全量爬取（无头）：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json
+ *   3. 有头模式调试验证：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --headed
+ *   4. 仅处理后处理：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --process-only
+ *   5. 强制重新爬取（不跳过已存在）：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --force
+ *   6. 单页导出测试：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --url https://wolai.dingtalk.com/xxx
+ *   7. 指定输出目录（覆盖配置）：
+ *      node packages/core/src/kb/wolai-crawler/run.js --config .sandbox/kb-validate/wolai-crawler-config.json --url https://wolai.dingtalk.com/xxx --output ./wiki
+ */
+
+const { parseArgs, readJson, resolveRepoPath, ensureDir } = require('./lib/runtime');
+const { launchBrowserContext } = require('./lib/browser');
+const { loginAndSaveStorageState, runCrawl } = require('./lib/crawler');
+const { processRawDownloads } = require('./lib/md-processor');
+
+function loadConfig(configPath) {
+  if (!configPath) {
+    console.error('Usage: node packages/core/src/kb/wolai-crawler/run.js --config <path> [--login] [--headed] [--process-only] [--force] [--output <dir>]');
+    process.exit(1);
+  }
+  return readJson(configPath);
+}
+
+function copyDirectoryContents(srcDir, destDir, { flatten = false } = {}) {
+  const fs = require('node:fs');
+  const path = require('node:path');
+  ensureDir(destDir);
+  let sourceDir = srcDir;
+  // 如果只有一个子目录且没有文件，将其视为容器目录并铺平
+  if (flatten) {
+    const entries = fs.readdirSync(sourceDir, { withFileTypes: true });
+    const dirs = entries.filter((e) => e.isDirectory());
+    const files = entries.filter((e) => e.isFile());
+    if (dirs.length === 1 && files.length === 0) {
+      sourceDir = path.join(sourceDir, dirs[0].name);
+    }
+  }
+  const entries = fs.readdirSync(sourceDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const srcPath = path.join(sourceDir, entry.name);
+    const destPath = path.join(destDir, entry.name);
+    if (entry.isDirectory()) {
+      copyDirectoryContents(srcPath, destPath);
+    } else {
+      fs.copyFileSync(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const args = parseArgs();
+  const config = loadConfig(args.config);
+
+  // CLI --output 覆盖配置里的 outputDir
+  if (args.output) {
+    config.outputDir = args.output;
+  }
+
+  if (args.login) {
+    await loginAndSaveStorageState(config, { loginWaitMs: args.loginWaitMs });
+    return;
+  }
+
+  if (args.processOnly) {
+    await processRawDownloads(config);
+    return;
+  }
+
+  // 单页测试模式
+  if (args.url) {
+    const { exportPage } = require('./lib/crawler');
+    const { browser, context, page } = await launchBrowserContext({
+      storageState: config.storageState,
+      headed: true,
+    });
+    try {
+      const result = await exportPage(page, { url: args.url, title: 'single-test' }, ensureDir(config.rawDir), { ...config, headed: true });
+      console.log(`\n✓ 单页测试成功: ${result.filePath}`);
+
+      // 如果指定了 --output，把最终内容（md 或解压目录）复制到输出目录
+      if (args.output) {
+        const fs = require('node:fs');
+        const path = require('node:path');
+        const outputDir = ensureDir(config.outputDir);
+        if (result.isZip) {
+          copyDirectoryContents(result.filePath, outputDir, { flatten: true });
+        } else {
+          const destMd = path.join(outputDir, path.basename(result.filePath));
+          fs.copyFileSync(result.filePath, destMd);
+          // 同时复制同目录下的 image/ 文件夹（如果存在）
+          const sourceImageDir = path.join(path.dirname(result.filePath), 'image');
+          if (fs.existsSync(sourceImageDir)) {
+            copyDirectoryContents(sourceImageDir, path.join(outputDir, 'image'));
+          }
+        }
+
+        console.log(`✓ 已复制到输出目录: ${outputDir}`);
+      }
+    } finally {
+      await context.close();
+      await browser.close();
+    }
+    return;
+  }
+
+  const results = await runCrawl(config, {
+    force: args.force,
+    headed: args.headed,
+  });
+
+  // 默认爬取成功后自动后处理
+  if (results.pages.some((p) => p.filePath)) {
+    console.log('\n→ 开始整理下载文件...');
+    await processRawDownloads(config);
+  } else {
+    console.log('\n⚠ 没有成功下载的页面，跳过后处理');
+  }
+}
+
+main()
+  .then(() => {
+    // 显式退出，避免 Playwright 残留 timer/worker 导致子进程挂起
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error('Fatal error:', err);
+    process.exit(1);
+  });