@jackwener/opencli 1.3.0 → 1.3.2

package/dist/clis/twitter/download.js

@@ -5,11 +5,9 @@
  *   opencli twitter download elonmusk --limit 10 --output ./twitter
  *   opencli twitter download --tweet-url https://x.com/xxx/status/123 --output ./twitter
  */
-import * as fs from 'node:fs';
-import * as path from 'node:path';
 import { cli, Strategy } from '../../registry.js';
-import { httpDownload, ytdlpDownload, checkYtdlp, getTempDir, exportCookiesToNetscape, formatCookieHeader, } from '../../download/index.js';
-import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+import { formatCookieHeader } from '../../download/index.js';
+import { downloadMedia } from '../../download/media-download.js';
 cli({
     site: 'twitter',
     name: 'download',
@@ -87,29 +85,10 @@ cli({
       })()
     `);
         if (!data || data.length === 0) {
-            return [{
-                    index: 0,
-                    type: '-',
-                    status: 'failed',
-                    size: 'No media found',
-                }];
+            return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
         }
         // Extract cookies
-        const cookies = await page.getCookies({ domain: 'x.com' });
-        const cookieString = formatCookieHeader(cookies);
-        // Create output directory
-        const outputDir = tweetUrl
-            ? path.join(output, 'tweets')
-            : path.join(output, username || 'media');
-        fs.mkdirSync(outputDir, { recursive: true });
-        // Export cookies for yt-dlp
-        let cookiesFile;
-        if (cookies.length > 0) {
-            const tempDir = getTempDir();
-            fs.mkdirSync(tempDir, { recursive: true });
-            cookiesFile = path.join(tempDir, `twitter_cookies_${Date.now()}.txt`);
-            exportCookiesToNetscape(cookies, cookiesFile);
-        }
+        const browserCookies = await page.getCookies({ domain: 'x.com' });
         // Deduplicate media
         const seen = new Set();
         const uniqueMedia = data.filter((m) => {
@@ -118,77 +97,14 @@ cli({
             seen.add(m.url);
             return true;
         }).slice(0, limit);
-        const tracker = new DownloadProgressTracker(uniqueMedia.length, true);
-        const results = [];
-        for (let i = 0; i < uniqueMedia.length; i++) {
-            const media = uniqueMedia[i];
-            const ext = media.type === 'image' ? 'jpg' : 'mp4';
-            const filename = `${username || 'tweet'}_${i + 1}.${ext}`;
-            const destPath = path.join(outputDir, filename);
-            const progressBar = tracker.onFileStart(filename, i);
-            try {
-                let result;
-                if (media.type === 'video-tweet' && checkYtdlp()) {
-                    // Use yt-dlp for video tweets
-                    result = await ytdlpDownload(media.url, destPath, {
-                        cookiesFile,
-                        extraArgs: ['--merge-output-format', 'mp4'],
-                        onProgress: (percent) => {
-                            if (progressBar)
-                                progressBar.update(percent, 100);
-                        },
-                    });
-                }
-                else if (media.type === 'image') {
-                    // Direct HTTP download for images
-                    result = await httpDownload(media.url, destPath, {
-                        cookies: cookieString,
-                        timeout: 30000,
-                        onProgress: (received, total) => {
-                            if (progressBar)
-                                progressBar.update(received, total);
-                        },
-                    });
-                }
-                else {
-                    // Direct HTTP download for direct video URLs
-                    result = await httpDownload(media.url, destPath, {
-                        cookies: cookieString,
-                        timeout: 60000,
-                        onProgress: (received, total) => {
-                            if (progressBar)
-                                progressBar.update(received, total);
-                        },
-                    });
-                }
-                if (progressBar) {
-                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
-                }
-                tracker.onFileComplete(result.success);
-                results.push({
-                    index: i + 1,
-                    type: media.type === 'video-tweet' ? 'video' : media.type,
-                    status: result.success ? 'success' : 'failed',
-                    size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
-                });
-            }
-            catch (err) {
-                if (progressBar)
-                    progressBar.fail(err.message);
-                tracker.onFileComplete(false);
-                results.push({
-                    index: i + 1,
-                    type: media.type,
-                    status: 'failed',
-                    size: err.message,
-                });
-            }
-        }
-        tracker.finish();
-        // Cleanup cookies file
-        if (cookiesFile && fs.existsSync(cookiesFile)) {
-            fs.unlinkSync(cookiesFile);
-        }
-        return results;
+        const subdir = tweetUrl ? 'tweets' : (username || 'media');
+        return downloadMedia(uniqueMedia, {
+            output,
+            subdir,
+            cookies: formatCookieHeader(browserCookies),
+            browserCookies,
+            filenamePrefix: username || 'tweet',
+            ytdlpExtraArgs: ['--merge-output-format', 'mp4'],
+        });
     },
 });

package/dist/clis/twitter/thread.js

@@ -1,4 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
+import { AuthRequiredError } from '../../errors.js';
 // ── Twitter GraphQL constants ──────────────────────────────────────────
 const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
 const TWEET_DETAIL_QUERY_ID = 'nBS-WpgA6ZG0CyNHD517JQ';
@@ -114,7 +115,7 @@ cli({
       return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
     }`);
         if (!ct0)
-            throw new Error('Not logged into x.com (no ct0 cookie)');
+            throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
         // Build auth headers in TypeScript
         const headers = JSON.stringify({
             'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,

package/dist/clis/v2ex/member.yaml

@@ -0,0 +1,29 @@
+site: v2ex
+name: member
+description: V2EX 用户资料
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  username:
+    positional: true
+    type: str
+    required: true
+    description: Username
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/members/show.json
+      params:
+        username: ${{ args.username }}
+
+  - map:
+      username: ${{ item.username }}
+      tagline: ${{ item.tagline }}
+      website: ${{ item.website }}
+      github: ${{ item.github }}
+      twitter: ${{ item.twitter }}
+      location: ${{ item.location }}
+
+columns: [username, tagline, website, github, twitter, location]

package/dist/clis/v2ex/node.yaml

@@ -0,0 +1,34 @@
+site: v2ex
+name: node
+description: V2EX 节点话题列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  name:
+    positional: true
+    type: str
+    required: true
+    description: Node name (e.g. python, javascript, apple)
+  limit:
+    type: int
+    default: 10
+    description: Number of topics (API returns max 20)
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/topics/show.json
+      params:
+        node_name: ${{ args.name }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      author: ${{ item.member.username }}
+      replies: ${{ item.replies }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, author, replies, url]

package/dist/clis/v2ex/nodes.yaml

@@ -0,0 +1,31 @@
+site: v2ex
+name: nodes
+description: V2EX 所有节点列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  limit:
+    type: int
+    default: 30
+    description: Number of nodes
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/nodes/all.json
+
+  - sort:
+      by: topics
+      order: desc
+
+  - map:
+      rank: ${{ index + 1 }}
+      name: ${{ item.name }}
+      title: ${{ item.title }}
+      topics: ${{ item.topics }}
+      stars: ${{ item.stars }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, name, title, topics, stars]

package/dist/clis/v2ex/replies.yaml

@@ -0,0 +1,32 @@
+site: v2ex
+name: replies
+description: V2EX 主题回复列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  id:
+    positional: true
+    type: str
+    required: true
+    description: Topic ID
+  limit:
+    type: int
+    default: 20
+    description: Number of replies
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/replies/show.json
+      params:
+        topic_id: ${{ args.id }}
+
+  - map:
+      floor: ${{ index + 1 }}
+      author: ${{ item.member.username }}
+      content: ${{ item.content }}
+
+  - limit: ${{ args.limit }}
+
+columns: [floor, author, content]

package/dist/clis/v2ex/user.yaml

@@ -0,0 +1,34 @@
+site: v2ex
+name: user
+description: V2EX 用户发帖列表
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  username:
+    positional: true
+    type: str
+    required: true
+    description: Username
+  limit:
+    type: int
+    default: 10
+    description: Number of topics (API returns max 20)
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/topics/show.json
+      params:
+        username: ${{ args.username }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      node: ${{ item.node.title }}
+      replies: ${{ item.replies }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, node, replies, url]

package/dist/clis/weibo/search.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/weibo/search.js

@@ -0,0 +1,73 @@
+/**
+ * Weibo search — browser DOM extraction from search results.
+ */
+import { cli, Strategy } from '../../registry.js';
+import { CliError } from '../../errors.js';
+cli({
+    site: 'weibo',
+    name: 'search',
+    description: '搜索微博',
+    domain: 'weibo.com',
+    browser: true,
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'keyword', required: true, positional: true, help: 'Search keyword' },
+        { name: 'limit', type: 'int', default: 10, help: 'Number of results (max 50)' },
+    ],
+    columns: ['rank', 'title', 'author', 'time', 'url'],
+    func: async (page, kwargs) => {
+        const limit = Math.max(1, Math.min(Number(kwargs.limit) || 10, 50));
+        const keyword = encodeURIComponent(String(kwargs.keyword ?? '').trim());
+        await page.goto(`https://s.weibo.com/weibo?q=${keyword}`);
+        await page.wait(2);
+        const data = await page.evaluate(`
+      (() => {
+        const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+        const absoluteUrl = (href) => {
+          if (!href) return '';
+          if (href.startsWith('http://') || href.startsWith('https://')) return href;
+          if (href.startsWith('//')) return window.location.protocol + href;
+          if (href.startsWith('/')) return window.location.origin + href;
+          return href;
+        };
+
+        const cards = Array.from(document.querySelectorAll('.card-wrap'));
+        const rows = [];
+
+        for (const card of cards) {
+          const contentEl =
+            card.querySelector('[node-type="feed_list_content_full"]') ||
+            card.querySelector('[node-type="feed_list_content"]') ||
+            card.querySelector('.txt');
+          const authorEl =
+            card.querySelector('.info .name') ||
+            card.querySelector('.name');
+          const timeEl = card.querySelector('.from a');
+          const urlEl =
+            card.querySelector('.from a[href*="/detail/"]') ||
+            card.querySelector('.from a[href*="/status/"]') ||
+            timeEl;
+
+          const title = clean(contentEl && contentEl.textContent);
+          if (!title) continue;
+
+          rows.push({
+            title,
+            author: clean(authorEl && authorEl.textContent),
+            time: clean(timeEl && timeEl.textContent),
+            url: absoluteUrl(urlEl && urlEl.getAttribute('href')),
+          });
+        }
+
+        return rows;
+      })()
+    `);
+        if (!Array.isArray(data) || data.length === 0) {
+            throw new CliError('NOT_FOUND', 'No Weibo search results found', 'Try a different keyword or ensure you are logged into weibo.com');
+        }
+        return data.slice(0, limit).map((item, index) => ({
+            rank: index + 1,
+            ...item,
+        }));
+    },
+});

package/dist/clis/weixin/download.d.ts

@@ -0,0 +1,12 @@
+/**
+ * WeChat article download — export WeChat Official Account articles to Markdown.
+ *
+ * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
+ *
+ * Usage:
+ *   opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
+ */
+/**
+ * Normalize a pasted WeChat article URL.
+ */
+export declare function normalizeWechatUrl(raw: string): string;

package/dist/clis/weixin/download.js

@@ -0,0 +1,183 @@
+/**
+ * WeChat article download — export WeChat Official Account articles to Markdown.
+ *
+ * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
+ *
+ * Usage:
+ *   opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
+ */
+import { cli, Strategy } from '../../registry.js';
+import { downloadArticle } from '../../download/article-download.js';
+// ============================================================
+// URL Normalization
+// ============================================================
+/**
+ * Normalize a pasted WeChat article URL.
+ */
+export function normalizeWechatUrl(raw) {
+    let s = (raw || '').trim();
+    if (!s)
+        return s;
+    // Strip wrapping quotes / angle brackets
+    if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
+        s = s.slice(1, -1).trim();
+    }
+    if (s.startsWith('<') && s.endsWith('>')) {
+        s = s.slice(1, -1).trim();
+    }
+    // Remove backslash escapes before URL-significant characters
+    s = s.replace(/\\+([:/&?=#%])/g, '$1');
+    // Decode HTML entities
+    s = s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
+    // Allow bare hostnames
+    if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
+        s = 'https://' + s.replace(/^\/+/, '');
+    }
+    // Force https for mp.weixin.qq.com
+    try {
+        const parsed = new URL(s);
+        if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
+            parsed.protocol = 'https:';
+            s = parsed.toString();
+        }
+    }
+    catch {
+        // Ignore parse errors
+    }
+    return s;
+}
+// ============================================================
+// CLI Registration
+// ============================================================
+cli({
+    site: 'weixin',
+    name: 'download',
+    description: '下载微信公众号文章为 Markdown 格式',
+    domain: 'mp.weixin.qq.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
+        { name: 'output', default: './weixin-articles', help: 'Output directory' },
+        { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
+    ],
+    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    func: async (page, kwargs) => {
+        const rawUrl = kwargs.url;
+        const url = normalizeWechatUrl(rawUrl);
+        if (!url.startsWith('https://mp.weixin.qq.com/')) {
+            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
+        }
+        // Navigate and wait for content to load
+        await page.goto(url);
+        await page.wait(5);
+        // Extract article data in browser context
+        const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          publishTime: '',
+          contentHtml: '',
+          codeBlocks: [],
+          imageUrls: []
+        };
+
+        // Title: #activity-name
+        const titleEl = document.querySelector('#activity-name');
+        result.title = titleEl ? titleEl.textContent.trim() : '';
+
+        // Author (WeChat Official Account name): #js_name
+        const authorEl = document.querySelector('#js_name');
+        result.author = authorEl ? authorEl.textContent.trim() : '';
+
+        // Publish time: extract create_time from script tags
+        const htmlStr = document.documentElement.innerHTML;
+        let timeMatch = htmlStr.match(/create_time\\s*:\\s*JsDecode\\('([^']+)'\\)/);
+        if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*:\\s*'(\\d+)'/);
+        if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*[:=]\\s*["']?(\\d+)["']?/);
+        if (timeMatch) {
+          const ts = parseInt(timeMatch[1], 10);
+          if (ts > 0) {
+            const d = new Date(ts * 1000);
+            const pad = n => String(n).padStart(2, '0');
+            const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
+            result.publishTime =
+              utc8.getUTCFullYear() + '-' +
+              pad(utc8.getUTCMonth() + 1) + '-' +
+              pad(utc8.getUTCDate()) + ' ' +
+              pad(utc8.getUTCHours()) + ':' +
+              pad(utc8.getUTCMinutes()) + ':' +
+              pad(utc8.getUTCSeconds());
+          }
+        }
+
+        // Content processing
+        const contentEl = document.querySelector('#js_content');
+        if (!contentEl) return result;
+
+        // Fix lazy-loaded images: data-src -> src
+        contentEl.querySelectorAll('img').forEach(img => {
+          const dataSrc = img.getAttribute('data-src');
+          if (dataSrc) img.setAttribute('src', dataSrc);
+        });
+
+        // Extract code blocks with placeholder replacement
+        const codeBlocks = [];
+        contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
+          el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
+          const pre = el.querySelector('pre[data-lang]');
+          const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
+          const lines = [];
+          el.querySelectorAll('code').forEach(codeTag => {
+            const text = codeTag.textContent;
+            if (/^[ce]?ounter\\(line/.test(text)) return;
+            lines.push(text);
+          });
+          if (lines.length === 0) lines.push(el.textContent);
+          const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
+          codeBlocks.push({ lang, code: lines.join('\\n') });
+          const p = document.createElement('p');
+          p.textContent = placeholder;
+          el.replaceWith(p);
+        });
+        result.codeBlocks = codeBlocks;
+
+        // Remove noise elements
+        ['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
+          contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
+        });
+
+        // Collect image URLs (deduplicated)
+        const seen = new Set();
+        contentEl.querySelectorAll('img[src]').forEach(img => {
+          const src = img.getAttribute('src');
+          if (src && !seen.has(src)) {
+            seen.add(src);
+            result.imageUrls.push(src);
+          }
+        });
+
+        result.contentHtml = contentEl.innerHTML;
+        return result;
+      })()
+    `);
+        return downloadArticle({
+            title: data?.title || '',
+            author: data?.author,
+            publishTime: data?.publishTime,
+            sourceUrl: url,
+            contentHtml: data?.contentHtml || '',
+            codeBlocks: data?.codeBlocks,
+            imageUrls: data?.imageUrls,
+        }, {
+            output: kwargs.output,
+            downloadImages: kwargs['download-images'],
+            imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
+            frontmatterLabels: { author: '公众号' },
+            detectImageExt: (url) => {
+                const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
+                return m ? m[1] : 'png';
+            },
+        });
+    },
+});