@jackwener/opencli 1.3.0 → 1.3.2

package/src/clis/twitter/download.ts

@@ -6,19 +6,9 @@
  *   opencli twitter download --tweet-url https://x.com/xxx/status/123 --output ./twitter
  */
 
-import * as fs from 'node:fs';
-import * as path from 'node:path';
 import { cli, Strategy } from '../../registry.js';
-import {
-  httpDownload,
-  ytdlpDownload,
-  checkYtdlp,
-  sanitizeFilename,
-  getTempDir,
-  exportCookiesToNetscape,
-  formatCookieHeader,
-} from '../../download/index.js';
-import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+import { formatCookieHeader } from '../../download/index.js';
+import { downloadMedia } from '../../download/media-download.js';
 
 cli({
   site: 'twitter',
@@ -101,32 +91,11 @@ cli({
     `);
 
     if (!data || data.length === 0) {
-      return [{
-        index: 0,
-        type: '-',
-        status: 'failed',
-        size: 'No media found',
-      }];
+      return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
     }
 
     // Extract cookies
-    const cookies = await page.getCookies({ domain: 'x.com' });
-    const cookieString = formatCookieHeader(cookies);
-
-    // Create output directory
-    const outputDir = tweetUrl
-      ? path.join(output, 'tweets')
-      : path.join(output, username || 'media');
-    fs.mkdirSync(outputDir, { recursive: true });
-
-    // Export cookies for yt-dlp
-    let cookiesFile: string | undefined;
-    if (cookies.length > 0) {
-      const tempDir = getTempDir();
-      fs.mkdirSync(tempDir, { recursive: true });
-      cookiesFile = path.join(tempDir, `twitter_cookies_${Date.now()}.txt`);
-      exportCookiesToNetscape(cookies, cookiesFile);
-    }
+    const browserCookies = await page.getCookies({ domain: 'x.com' });
 
     // Deduplicate media
     const seen = new Set<string>();
@@ -136,81 +105,14 @@ cli({
       return true;
     }).slice(0, limit);
 
-    const tracker = new DownloadProgressTracker(uniqueMedia.length, true);
-    const results: any[] = [];
-
-    for (let i = 0; i < uniqueMedia.length; i++) {
-      const media = uniqueMedia[i];
-      const ext = media.type === 'image' ? 'jpg' : 'mp4';
-      const filename = `${username || 'tweet'}_${i + 1}.${ext}`;
-      const destPath = path.join(outputDir, filename);
-
-      const progressBar = tracker.onFileStart(filename, i);
-
-      try {
-        let result: { success: boolean; size: number; error?: string };
-
-        if (media.type === 'video-tweet' && checkYtdlp()) {
-          // Use yt-dlp for video tweets
-          result = await ytdlpDownload(media.url, destPath, {
-            cookiesFile,
-            extraArgs: ['--merge-output-format', 'mp4'],
-            onProgress: (percent) => {
-              if (progressBar) progressBar.update(percent, 100);
-            },
-          });
-        } else if (media.type === 'image') {
-          // Direct HTTP download for images
-          result = await httpDownload(media.url, destPath, {
-            cookies: cookieString,
-            timeout: 30000,
-            onProgress: (received, total) => {
-              if (progressBar) progressBar.update(received, total);
-            },
-          });
-        } else {
-          // Direct HTTP download for direct video URLs
-          result = await httpDownload(media.url, destPath, {
-            cookies: cookieString,
-            timeout: 60000,
-            onProgress: (received, total) => {
-              if (progressBar) progressBar.update(received, total);
-            },
-          });
-        }
-
-        if (progressBar) {
-          progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
-        }
-
-        tracker.onFileComplete(result.success);
-
-        results.push({
-          index: i + 1,
-          type: media.type === 'video-tweet' ? 'video' : media.type,
-          status: result.success ? 'success' : 'failed',
-          size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
-        });
-      } catch (err: any) {
-        if (progressBar) progressBar.fail(err.message);
-        tracker.onFileComplete(false);
-
-        results.push({
-          index: i + 1,
-          type: media.type,
-          status: 'failed',
-          size: err.message,
-        });
-      }
-    }
-
-    tracker.finish();
-
-    // Cleanup cookies file
-    if (cookiesFile && fs.existsSync(cookiesFile)) {
-      fs.unlinkSync(cookiesFile);
-    }
-
-    return results;
+    const subdir = tweetUrl ? 'tweets' : (username || 'media');
+    return downloadMedia(uniqueMedia, {
+      output,
+      subdir,
+      cookies: formatCookieHeader(browserCookies),
+      browserCookies,
+      filenamePrefix: username || 'tweet',
+      ytdlpExtraArgs: ['--merge-output-format', 'mp4'],
+    });
   },
 });

package/src/clis/twitter/thread.ts

@@ -1,4 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
+import { AuthRequiredError } from '../../errors.js';
 
 // ── Twitter GraphQL constants ──────────────────────────────────────────
 
@@ -139,7 +140,7 @@ cli({
     const ct0 = await page.evaluate(`() => {
       return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
     }`);
-    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+    if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
 
     // Build auth headers in TypeScript
     const headers = JSON.stringify({

package/src/clis/v2ex/member.yaml

@@ -0,0 +1,29 @@
+site: v2ex
+name: member
+description: V2EX 用户资料
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  username:
+    positional: true
+    type: str
+    required: true
+    description: Username
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/members/show.json
+      params:
+        username: ${{ args.username }}
+
+  - map:
+      username: ${{ item.username }}
+      tagline: ${{ item.tagline }}
+      website: ${{ item.website }}
+      github: ${{ item.github }}
+      twitter: ${{ item.twitter }}
+      location: ${{ item.location }}
+
+columns: [username, tagline, website, github, twitter, location]

package/src/clis/v2ex/node.yaml

@@ -0,0 +1,34 @@
+site: v2ex
+name: node
+description: V2EX 节点话题列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  name:
+    positional: true
+    type: str
+    required: true
+    description: Node name (e.g. python, javascript, apple)
+  limit:
+    type: int
+    default: 10
+    description: Number of topics (API returns max 20)
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/topics/show.json
+      params:
+        node_name: ${{ args.name }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      author: ${{ item.member.username }}
+      replies: ${{ item.replies }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, author, replies, url]

package/src/clis/v2ex/nodes.yaml

@@ -0,0 +1,31 @@
+site: v2ex
+name: nodes
+description: V2EX 所有节点列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  limit:
+    type: int
+    default: 30
+    description: Number of nodes
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/nodes/all.json
+
+  - sort:
+      by: topics
+      order: desc
+
+  - map:
+      rank: ${{ index + 1 }}
+      name: ${{ item.name }}
+      title: ${{ item.title }}
+      topics: ${{ item.topics }}
+      stars: ${{ item.stars }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, name, title, topics, stars]

package/src/clis/v2ex/replies.yaml

@@ -0,0 +1,32 @@
+site: v2ex
+name: replies
+description: V2EX 主题回复列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  id:
+    positional: true
+    type: str
+    required: true
+    description: Topic ID
+  limit:
+    type: int
+    default: 20
+    description: Number of replies
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/replies/show.json
+      params:
+        topic_id: ${{ args.id }}
+
+  - map:
+      floor: ${{ index + 1 }}
+      author: ${{ item.member.username }}
+      content: ${{ item.content }}
+
+  - limit: ${{ args.limit }}
+
+columns: [floor, author, content]

package/src/clis/v2ex/user.yaml

@@ -0,0 +1,34 @@
+site: v2ex
+name: user
+description: V2EX 用户发帖列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  username:
+    positional: true
+    type: str
+    required: true
+    description: Username
+  limit:
+    type: int
+    default: 10
+    description: Number of topics (API returns max 20)
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/topics/show.json
+      params:
+        username: ${{ args.username }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      node: ${{ item.node.title }}
+      replies: ${{ item.replies }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, node, replies, url]

package/src/clis/weibo/search.ts

@@ -0,0 +1,78 @@
+/**
+ * Weibo search — browser DOM extraction from search results.
+ */
+import { cli, Strategy } from '../../registry.js';
+import { CliError } from '../../errors.js';
+
+cli({
+  site: 'weibo',
+  name: 'search',
+  description: '搜索微博',
+  domain: 'weibo.com',
+  browser: true,
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'keyword', required: true, positional: true, help: 'Search keyword' },
+    { name: 'limit', type: 'int', default: 10, help: 'Number of results (max 50)' },
+  ],
+  columns: ['rank', 'title', 'author', 'time', 'url'],
+  func: async (page, kwargs) => {
+    const limit = Math.max(1, Math.min(Number(kwargs.limit) || 10, 50));
+    const keyword = encodeURIComponent(String(kwargs.keyword ?? '').trim());
+
+    await page.goto(`https://s.weibo.com/weibo?q=${keyword}`);
+    await page.wait(2);
+
+    const data = await page.evaluate(`
+      (() => {
+        const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+        const absoluteUrl = (href) => {
+          if (!href) return '';
+          if (href.startsWith('http://') || href.startsWith('https://')) return href;
+          if (href.startsWith('//')) return window.location.protocol + href;
+          if (href.startsWith('/')) return window.location.origin + href;
+          return href;
+        };
+
+        const cards = Array.from(document.querySelectorAll('.card-wrap'));
+        const rows = [];
+
+        for (const card of cards) {
+          const contentEl =
+            card.querySelector('[node-type="feed_list_content_full"]') ||
+            card.querySelector('[node-type="feed_list_content"]') ||
+            card.querySelector('.txt');
+          const authorEl =
+            card.querySelector('.info .name') ||
+            card.querySelector('.name');
+          const timeEl = card.querySelector('.from a');
+          const urlEl =
+            card.querySelector('.from a[href*="/detail/"]') ||
+            card.querySelector('.from a[href*="/status/"]') ||
+            timeEl;
+
+          const title = clean(contentEl && contentEl.textContent);
+          if (!title) continue;
+
+          rows.push({
+            title,
+            author: clean(authorEl && authorEl.textContent),
+            time: clean(timeEl && timeEl.textContent),
+            url: absoluteUrl(urlEl && urlEl.getAttribute('href')),
+          });
+        }
+
+        return rows;
+      })()
+    `);
+
+    if (!Array.isArray(data) || data.length === 0) {
+      throw new CliError('NOT_FOUND', 'No Weibo search results found', 'Try a different keyword or ensure you are logged into weibo.com');
+    }
+
+    return data.slice(0, limit).map((item, index) => ({
+      rank: index + 1,
+      ...item,
+    }));
+  },
+});

package/src/clis/weixin/download.ts

@@ -0,0 +1,199 @@
+/**
+ * WeChat article download — export WeChat Official Account articles to Markdown.
+ *
+ * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
+ *
+ * Usage:
+ *   opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
+ */
+
+import { cli, Strategy } from '../../registry.js';
+import { downloadArticle } from '../../download/article-download.js';
+
+// ============================================================
+// URL Normalization
+// ============================================================
+
+/**
+ * Normalize a pasted WeChat article URL.
+ */
+export function normalizeWechatUrl(raw: string): string {
+  let s = (raw || '').trim();
+  if (!s) return s;
+
+  // Strip wrapping quotes / angle brackets
+  if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
+    s = s.slice(1, -1).trim();
+  }
+  if (s.startsWith('<') && s.endsWith('>')) {
+    s = s.slice(1, -1).trim();
+  }
+
+  // Remove backslash escapes before URL-significant characters
+  s = s.replace(/\\+([:/&?=#%])/g, '$1');
+
+  // Decode HTML entities
+  s = s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
+
+  // Allow bare hostnames
+  if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
+    s = 'https://' + s.replace(/^\/+/, '');
+  }
+
+  // Force https for mp.weixin.qq.com
+  try {
+    const parsed = new URL(s);
+    if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
+      parsed.protocol = 'https:';
+      s = parsed.toString();
+    }
+  } catch {
+    // Ignore parse errors
+  }
+
+  return s;
+}
+
+// ============================================================
+// CLI Registration
+// ============================================================
+
+cli({
+  site: 'weixin',
+  name: 'download',
+  description: '下载微信公众号文章为 Markdown 格式',
+  domain: 'mp.weixin.qq.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
+    { name: 'output', default: './weixin-articles', help: 'Output directory' },
+    { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
+  ],
+  columns: ['title', 'author', 'publish_time', 'status', 'size'],
+  func: async (page, kwargs) => {
+    const rawUrl = kwargs.url;
+    const url = normalizeWechatUrl(rawUrl);
+
+    if (!url.startsWith('https://mp.weixin.qq.com/')) {
+      return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
+    }
+
+    // Navigate and wait for content to load
+    await page.goto(url);
+    await page.wait(5);
+
+    // Extract article data in browser context
+    const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          publishTime: '',
+          contentHtml: '',
+          codeBlocks: [],
+          imageUrls: []
+        };
+
+        // Title: #activity-name
+        const titleEl = document.querySelector('#activity-name');
+        result.title = titleEl ? titleEl.textContent.trim() : '';
+
+        // Author (WeChat Official Account name): #js_name
+        const authorEl = document.querySelector('#js_name');
+        result.author = authorEl ? authorEl.textContent.trim() : '';
+
+        // Publish time: extract create_time from script tags
+        const htmlStr = document.documentElement.innerHTML;
+        let timeMatch = htmlStr.match(/create_time\\s*:\\s*JsDecode\\('([^']+)'\\)/);
+        if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*:\\s*'(\\d+)'/);
+        if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*[:=]\\s*["']?(\\d+)["']?/);
+        if (timeMatch) {
+          const ts = parseInt(timeMatch[1], 10);
+          if (ts > 0) {
+            const d = new Date(ts * 1000);
+            const pad = n => String(n).padStart(2, '0');
+            const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
+            result.publishTime =
+              utc8.getUTCFullYear() + '-' +
+              pad(utc8.getUTCMonth() + 1) + '-' +
+              pad(utc8.getUTCDate()) + ' ' +
+              pad(utc8.getUTCHours()) + ':' +
+              pad(utc8.getUTCMinutes()) + ':' +
+              pad(utc8.getUTCSeconds());
+          }
+        }
+
+        // Content processing
+        const contentEl = document.querySelector('#js_content');
+        if (!contentEl) return result;
+
+        // Fix lazy-loaded images: data-src -> src
+        contentEl.querySelectorAll('img').forEach(img => {
+          const dataSrc = img.getAttribute('data-src');
+          if (dataSrc) img.setAttribute('src', dataSrc);
+        });
+
+        // Extract code blocks with placeholder replacement
+        const codeBlocks = [];
+        contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
+          el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
+          const pre = el.querySelector('pre[data-lang]');
+          const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
+          const lines = [];
+          el.querySelectorAll('code').forEach(codeTag => {
+            const text = codeTag.textContent;
+            if (/^[ce]?ounter\\(line/.test(text)) return;
+            lines.push(text);
+          });
+          if (lines.length === 0) lines.push(el.textContent);
+          const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
+          codeBlocks.push({ lang, code: lines.join('\\n') });
+          const p = document.createElement('p');
+          p.textContent = placeholder;
+          el.replaceWith(p);
+        });
+        result.codeBlocks = codeBlocks;
+
+        // Remove noise elements
+        ['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
+          contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
+        });
+
+        // Collect image URLs (deduplicated)
+        const seen = new Set();
+        contentEl.querySelectorAll('img[src]').forEach(img => {
+          const src = img.getAttribute('src');
+          if (src && !seen.has(src)) {
+            seen.add(src);
+            result.imageUrls.push(src);
+          }
+        });
+
+        result.contentHtml = contentEl.innerHTML;
+        return result;
+      })()
+    `);
+
+    return downloadArticle(
+      {
+        title: data?.title || '',
+        author: data?.author,
+        publishTime: data?.publishTime,
+        sourceUrl: url,
+        contentHtml: data?.contentHtml || '',
+        codeBlocks: data?.codeBlocks,
+        imageUrls: data?.imageUrls,
+      },
+      {
+        output: kwargs.output,
+        downloadImages: kwargs['download-images'],
+        imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
+        frontmatterLabels: { author: '公众号' },
+        detectImageExt: (url) => {
+          const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
+          return m ? m[1] : 'png';
+        },
+      },
+    );
+  },
+});