@claudeink/mcp-server 0.0.2 → 0.0.4

package/workflow/tools/crawler/config.json

@@ -0,0 +1,173 @@
+{
+  "sources": [
+    {
+      "id": "beomniscient",
+      "name": "Omniscient Digital Blog",
+      "blogUrl": "https://beomniscient.com/blog/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://beomniscient.com/blog/page/{page}/",
+        "startPage": 1,
+        "maxPages": 100
+      },
+      "articleSelector": "a[href*='/blog/']",
+      "articleUrlPattern": "^https://beomniscient\\.com/blog/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/blog/page/",
+        "/blog/category/",
+        "/blog/author/",
+        "/blog/tag/"
+      ],
+      "enabled": true
+    },
+    {
+      "id": "9to5mac",
+      "name": "9to5Mac",
+      "blogUrl": "https://9to5mac.com/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://9to5mac.com/page/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='9to5mac.com/2']",
+      "articleUrlPattern": "^https://9to5mac\\.com/\\d{4}/\\d{2}/\\d{2}/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "extended-comments",
+        "#comments",
+        "#more-",
+        "/page/"
+      ],
+      "enabled": true
+    },
+    {
+      "id": "macrumors",
+      "name": "MacRumors",
+      "blogUrl": "https://www.macrumors.com/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://www.macrumors.com/page/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='macrumors.com/2']",
+      "articleUrlPattern": "^https://www\\.macrumors\\.com/\\d{4}/\\d{2}/\\d{2}/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/page/",
+        "/roundup/",
+        "/guide/",
+        "/deals/"
+      ],
+      "enabled": true
+    },
+    {
+      "id": "appleinsider",
+      "name": "AppleInsider",
+      "blogUrl": "https://appleinsider.com/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://appleinsider.com/articles/page/{page}",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='/articles/']",
+      "articleUrlPattern": "^https://appleinsider\\.com/articles/\\d{2}/\\d{2}/\\d{2}/[a-z0-9-]+",
+      "excludePatterns": [
+        "/articles/page/"
+      ],
+      "enabled": true
+    },
+    {
+      "id": "simonwillison",
+      "name": "Simon Willison's Weblog",
+      "blogUrl": "https://simonwillison.net/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://simonwillison.net/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='/202']",
+      "articleUrlPattern": "^https://simonwillison\\.net/\\d{4}/[A-Za-z]+/\\d+/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/tags/",
+        "/search/"
+      ],
+      "maxArticles": 15,
+      "enabled": true
+    },
+    {
+      "id": "qbitai",
+      "name": "量子位",
+      "blogUrl": "https://www.qbitai.com/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://www.qbitai.com/page/{page}",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='qbitai.com/2']",
+      "articleUrlPattern": "^https://www\\.qbitai\\.com/\\d{4}/\\d{2}/\\d+\\.html$",
+      "excludePatterns": [
+        "/page/",
+        "/category/",
+        "/tag/",
+        "/author/"
+      ],
+      "maxArticles": 15,
+      "enabled": true
+    },
+    {
+      "id": "importai",
+      "name": "Import AI (Jack Clark)",
+      "blogUrl": "https://jack-clark.net/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://jack-clark.net/page/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='jack-clark.net/202']",
+      "articleUrlPattern": "^https://jack-clark\\.net/\\d{4}/\\d{2}/\\d{2}/import-ai-[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/page/",
+        "/category/",
+        "/tag/"
+      ],
+      "maxArticles": 15,
+      "enabled": true
+    },
+    {
+      "id": "techcrunch-robotics",
+      "name": "TechCrunch Robotics",
+      "blogUrl": "https://techcrunch.com/category/robotics/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://techcrunch.com/category/robotics/page/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='techcrunch.com/202']",
+      "articleUrlPattern": "^https://techcrunch\\.com/\\d{4}/\\d{2}/\\d{2}/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/page/",
+        "/category/",
+        "/tag/",
+        "/author/"
+      ],
+      "maxArticles": 15,
+      "enabled": true
+    }
+  ],
+  "output": {
+    "baseDir": "../../sources/articles",
+    "filenameFormat": "{date}-{slug}",
+    "includeMetadata": true
+  },
+  "settings": {
+    "requestDelay": 1500,
+    "maxConcurrent": 3,
+    "userAgent": "AustonBot/1.0 (Content Research)",
+    "maxRetries": 2
+  }
+}

package/workflow/tools/crawler/config.test.json

@@ -0,0 +1,35 @@
+{
+  "sources": [
+    {
+      "id": "beomniscient",
+      "name": "Omniscient Digital Blog",
+      "blogUrl": "https://beomniscient.com/blog/",
+      "type": "paginated",
+      "pagination": {
+        "pattern": "https://beomniscient.com/blog/page/{page}/",
+        "startPage": 1,
+        "maxPages": 1
+      },
+      "articleSelector": "a[href*='/blog/']",
+      "articleUrlPattern": "^https://beomniscient\\.com/blog/[a-z0-9-]+/?$",
+      "excludePatterns": [
+        "/blog/page/",
+        "/blog/category/",
+        "/blog/author/",
+        "/blog/tag/"
+      ],
+      "enabled": true
+    }
+  ],
+  "output": {
+    "baseDir": "../../sources/articles",
+    "filenameFormat": "{date}-{slug}",
+    "includeMetadata": true
+  },
+  "settings": {
+    "requestDelay": 1500,
+    "maxConcurrent": 3,
+    "userAgent": "AustonBot/1.0 (Content Research)",
+    "maxRetries": 2
+  }
+}

package/workflow/tools/crawler/crawl.mjs

@@ -0,0 +1,398 @@
+#!/usr/bin/env node
+
+/**
+ * Auston Blog Crawler
+ * 
+ * 订阅式 Blog 爬虫，基于 Defuddle 提取正文
+ * 
+ * 用法:
+ *   node crawl.mjs                     # 增量抓取所有已启用的源（默认模式）
+ *   node crawl.mjs --full              # 全量抓取所有源（首次使用）
+ *   node crawl.mjs --full --source beomniscient  # 全量抓取指定源
+ *   node crawl.mjs --source beomniscient          # 增量抓取指定源
+ *   node crawl.mjs --url https://...              # 抓取单篇文章
+ */
+
+import { JSDOM } from 'jsdom';
+import { Defuddle } from 'defuddle/node';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const CONFIG_PATH = path.join(__dirname, 'config.json');
+const STATE_PATH = path.join(__dirname, '..', '..', '.crawler-state.json');
+
+// ============================================================
+// Utility Functions
+// ============================================================
+
+function loadConfig() {
+  return JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf-8'));
+}
+
+function loadState() {
+  if (fs.existsSync(STATE_PATH)) {
+    return JSON.parse(fs.readFileSync(STATE_PATH, 'utf-8'));
+  }
+  return { sources: {} };
+}
+
+function saveState(state) {
+  fs.writeFileSync(STATE_PATH, JSON.stringify(state, null, 2), 'utf-8');
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function slugify(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\u4e00-\u9fff]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 60);
+}
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function log(level, msg) {
+  const ts = new Date().toISOString().slice(0, 19).replace('T', ' ');
+  const prefix = { info: 'ℹ️', ok: '✅', warn: '⚠️', error: '❌', skip: '⏭️' }[level] || '•';
+  console.log(`[${ts}] ${prefix} ${msg}`);
+}
+
+// ============================================================
+// Core: Discover Article URLs from a Blog
+// ============================================================
+
+async function discoverArticles(source, config, state, fullMode) {
+  const sourceState = state.sources[source.id] || { fetchedUrls: [], lastFetch: null };
+  const fetchedSet = new Set(sourceState.fetchedUrls || []);
+  const newUrls = [];
+  const excludePatterns = source.excludePatterns || [];
+  const urlRegex = source.articleUrlPattern ? new RegExp(source.articleUrlPattern) : null;
+
+  log('info', `开始发现文章: ${source.name} (${fullMode ? '全量' : '增量'}模式)`);
+
+  const maxArticles = source.maxArticles || 0; // 0 = 不限制
+  let page = source.pagination.startPage;
+  let maxPages = fullMode ? source.pagination.maxPages : 10; // 增量模式最多翻10页
+  let consecutiveKnown = 0;
+
+  while (page <= maxPages) {
+    const pageUrl = page === 1 
+      ? source.blogUrl 
+      : source.pagination.pattern.replace('{page}', page);
+    
+    log('info', `抓取列表页: ${pageUrl}`);
+
+    try {
+      const dom = await JSDOM.fromURL(pageUrl, {
+        pretendToBeVisual: true,
+        userAgent: config.settings.userAgent,
+      });
+
+      const doc = dom.window.document;
+      const links = doc.querySelectorAll(source.articleSelector);
+      let pageNewCount = 0;
+
+      for (const link of links) {
+        let href = link.href;
+        if (!href) continue;
+
+        // 确保是完整 URL
+        if (href.startsWith('/')) {
+          const base = new URL(source.blogUrl);
+          href = `${base.protocol}//${base.host}${href}`;
+        }
+
+        // 过滤
+        const shouldExclude = excludePatterns.some(p => href.includes(p));
+        if (shouldExclude) continue;
+        if (urlRegex && !urlRegex.test(href)) continue;
+
+        // 去重
+        if (fetchedSet.has(href) || newUrls.includes(href)) {
+          consecutiveKnown++;
+          continue;
+        }
+
+        newUrls.push(href);
+        pageNewCount++;
+        consecutiveKnown = 0;
+      }
+
+      log('info', `第 ${page} 页发现 ${pageNewCount} 篇新文章`);
+
+      // maxArticles 限制：够了就停
+      if (maxArticles > 0 && newUrls.length >= maxArticles) {
+        log('info', `已达到 maxArticles 上限 (${maxArticles})，停止翻页`);
+        break;
+      }
+
+      // 增量模式：如果连续遇到太多已知文章，说明已经到了上次抓取的位置
+      if (!fullMode && consecutiveKnown >= 15) {
+        log('info', `连续 ${consecutiveKnown} 篇已知文章，停止翻页`);
+        break;
+      }
+
+      // 检查是否还有下一页（简单检测）
+      const hasNext = doc.querySelector('a[href*="page/"]') || 
+                       doc.querySelector('.next') ||
+                       doc.querySelector('a.next');
+      
+      if (!hasNext && page > 1) {
+        log('info', '没有更多分页了');
+        break;
+      }
+
+      page++;
+      await sleep(config.settings.requestDelay);
+    } catch (err) {
+      if (err.message && err.message.includes('404')) {
+        log('info', `第 ${page} 页不存在，停止翻页`);
+        break;
+      }
+      log('error', `抓取第 ${page} 页失败: ${err.message}`);
+      page++;
+    }
+  }
+
+  // 去重并截断
+  let uniqueUrls = [...new Set(newUrls)];
+  if (maxArticles > 0 && uniqueUrls.length > maxArticles) {
+    uniqueUrls = uniqueUrls.slice(0, maxArticles);
+  }
+  log('ok', `${source.name}: 共发现 ${uniqueUrls.length} 篇新文章${maxArticles > 0 ? ` (上限 ${maxArticles})` : ''}`);
+  return uniqueUrls;
+}
+
+// ============================================================
+// Core: Fetch & Extract a Single Article
+// ============================================================
+
+async function fetchArticle(url, sourceId, config) {
+  try {
+    const dom = await JSDOM.fromURL(url, {
+      pretendToBeVisual: true,
+      userAgent: config.settings.userAgent,
+    });
+
+    const result = await Defuddle(dom, url, { markdown: true });
+
+    if (!result || !result.content || result.content.trim().length < 100) {
+      log('warn', `内容过短或为空: ${url}`);
+      return null;
+    }
+
+    let date = new Date().toISOString().slice(0, 10);
+    if (result.published) {
+      try {
+        const d = new Date(result.published);
+        if (!isNaN(d.getTime())) {
+          date = d.toISOString().slice(0, 10);
+        }
+      } catch (_) { /* use today's date as fallback */ }
+    }
+    
+    const slug = slugify(result.title || path.basename(new URL(url).pathname));
+    const filename = `${date}-${slug}.md`;
+
+    const frontMatter = [
+      '---',
+      `title: "${(result.title || '').replace(/"/g, '\\"')}"`,
+      `author: "${(result.author || '').replace(/"/g, '\\"')}"`,
+      `source: "${url}"`,
+      `domain: "${result.domain || ''}"`,
+      `site: "${result.site || ''}"`,
+      `published: "${result.published || ''}"`,
+      `fetched: "${new Date().toISOString().slice(0, 10)}"`,
+      `wordCount: ${result.wordCount || 0}`,
+      `description: "${(result.description || '').replace(/"/g, '\\"')}"`,
+      '---',
+    ].join('\n');
+
+    const output = `${frontMatter}\n\n${result.content}\n`;
+
+    // 保存
+    const outputDir = path.resolve(__dirname, config.output.baseDir, sourceId || 'manual');
+    ensureDir(outputDir);
+    const filepath = path.join(outputDir, filename);
+    fs.writeFileSync(filepath, output, 'utf-8');
+
+    return {
+      url,
+      title: result.title,
+      author: result.author,
+      published: result.published,
+      wordCount: result.wordCount,
+      filename,
+      filepath,
+    };
+  } catch (err) {
+    log('error', `抓取失败 ${url}: ${err.message}`);
+    return null;
+  }
+}
+
+// ============================================================
+// Core: Batch Fetch with Concurrency Control
+// ============================================================
+
+async function batchFetch(urls, sourceId, config) {
+  const results = [];
+  const maxConcurrent = config.settings.maxConcurrent || 3;
+  
+  for (let i = 0; i < urls.length; i += maxConcurrent) {
+    const batch = urls.slice(i, i + maxConcurrent);
+    const batchNum = Math.floor(i / maxConcurrent) + 1;
+    const totalBatches = Math.ceil(urls.length / maxConcurrent);
+    
+    log('info', `批次 ${batchNum}/${totalBatches}: 抓取 ${batch.length} 篇`);
+    
+    const batchResults = await Promise.all(
+      batch.map(url => fetchArticle(url, sourceId, config))
+    );
+    
+    results.push(...batchResults.filter(Boolean));
+    
+    if (i + maxConcurrent < urls.length) {
+      await sleep(config.settings.requestDelay);
+    }
+  }
+  
+  return results;
+}
+
+// ============================================================
+// Report Generation
+// ============================================================
+
+function generateReport(allResults, startTime) {
+  const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+  const date = new Date().toISOString().slice(0, 10);
+  const time = new Date().toISOString().slice(11, 19);
+  
+  const lines = [
+    `# 抓取报告`,
+    ``,
+    `- 时间: ${date} ${time}`,
+    `- 耗时: ${duration}s`,
+    `- 总计抓取: ${allResults.length} 篇`,
+    ``,
+  ];
+
+  // 按源分组
+  const bySource = {};
+  for (const r of allResults) {
+    const source = path.basename(path.dirname(r.filepath));
+    if (!bySource[source]) bySource[source] = [];
+    bySource[source].push(r);
+  }
+
+  for (const [source, results] of Object.entries(bySource)) {
+    lines.push(`## ${source} (${results.length} 篇)`);
+    lines.push('');
+    for (const r of results) {
+      lines.push(`- **${r.title}** — ${r.author || '未知作者'} — ${r.published || '日期未知'} — ${r.wordCount} 词`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// ============================================================
+// Main
+// ============================================================
+
+async function main() {
+  const args = process.argv.slice(2);
+  const fullMode = args.includes('--full');
+  const sourceFilter = args.includes('--source') 
+    ? args[args.indexOf('--source') + 1] 
+    : null;
+  const singleUrl = args.includes('--url')
+    ? args[args.indexOf('--url') + 1]
+    : null;
+
+  const config = loadConfig();
+  const state = loadState();
+  const startTime = Date.now();
+  const allResults = [];
+
+  // 模式 1: 单篇 URL
+  if (singleUrl) {
+    log('info', `单篇抓取: ${singleUrl}`);
+    const result = await fetchArticle(singleUrl, 'manual', config);
+    if (result) {
+      log('ok', `已保存: ${result.filename}`);
+      allResults.push(result);
+    }
+  } 
+  // 模式 2: 订阅源抓取
+  else {
+    const sources = config.sources
+      .filter(s => s.enabled)
+      .filter(s => !sourceFilter || s.id === sourceFilter);
+
+    if (sources.length === 0) {
+      log('warn', sourceFilter 
+        ? `未找到源: ${sourceFilter}` 
+        : '没有已启用的订阅源');
+      process.exit(1);
+    }
+
+    for (const source of sources) {
+      // Step 1: 发现新文章 URL
+      const newUrls = await discoverArticles(source, config, state, fullMode);
+      
+      if (newUrls.length === 0) {
+        log('skip', `${source.name}: 没有新文章`);
+        continue;
+      }
+
+      // Step 2: 批量抓取
+      const results = await batchFetch(newUrls, source.id, config);
+      allResults.push(...results);
+
+      // Step 3: 更新状态
+      if (!state.sources[source.id]) {
+        state.sources[source.id] = { fetchedUrls: [], lastFetch: null };
+      }
+      const fetchedUrls = results.map(r => r.url);
+      state.sources[source.id].fetchedUrls = [
+        ...new Set([
+          ...fetchedUrls,
+          ...(state.sources[source.id].fetchedUrls || []),
+        ])
+      ];
+      state.sources[source.id].lastFetch = new Date().toISOString();
+      saveState(state);
+
+      log('ok', `${source.name}: 成功抓取 ${results.length}/${newUrls.length} 篇`);
+    }
+  }
+
+  // 生成报告
+  if (allResults.length > 0) {
+    const report = generateReport(allResults, startTime);
+    const reportDir = path.resolve(__dirname, '..', '..');
+    const reportPath = path.join(reportDir, 'crawl-report.md');
+    fs.writeFileSync(reportPath, report, 'utf-8');
+    log('ok', `报告已生成: crawl-report.md`);
+  }
+
+  log('ok', `完成! 共抓取 ${allResults.length} 篇文章`);
+}
+
+main().catch(err => {
+  log('error', `致命错误: ${err.message}`);
+  console.error(err);
+  process.exit(1);
+});