@huyuan-ai/cli 1.0.0 → 1.0.2

package/vendor/opencli/dist/download/article-download.js

@@ -0,0 +1,178 @@
+/**
+ * Article download helper — shared logic for downloading articles as Markdown.
+ *
+ * Used by: zhihu/download, weixin/download, and future article adapters.
+ *
+ * Flow: ArticleData → TurndownService → image download → frontmatter → .md file
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import TurndownService from 'turndown';
+import { httpDownload, sanitizeFilename } from './index.js';
+import { formatBytes } from './progress.js';
+const IMAGE_CONCURRENCY = 5;
+const DEFAULT_LABELS = {
+    author: '作者',
+    publishTime: '发布时间',
+    sourceUrl: '原文链接',
+};
+// ============================================================
+// Markdown Conversion
+// ============================================================
+function createTurndown(configure) {
+    const td = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        bulletListMarker: '-',
+    });
+    td.addRule('linebreak', {
+        filter: 'br',
+        replacement: () => '\n',
+    });
+    if (configure)
+        configure(td);
+    return td;
+}
+function convertToMarkdown(contentHtml, codeBlocks, configure) {
+    const td = createTurndown(configure);
+    let md = td.turndown(contentHtml);
+    // Restore code block placeholders
+    codeBlocks.forEach((block, i) => {
+        const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
+        const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
+        md = md.replace(placeholder, fenced);
+    });
+    // Clean up
+    md = md.replace(/\u00a0/g, ' ');
+    md = md.replace(/\n{4,}/g, '\n\n\n');
+    md = md.replace(/[ \t]+$/gm, '');
+    return md;
+}
+function replaceImageUrls(md, urlMap) {
+    return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
+        const local = urlMap[imgUrl];
+        return local ? `![${alt}](${local})` : match;
+    });
+}
+// ============================================================
+// Image Downloading
+// ============================================================
+function defaultDetectImageExt(url) {
+    const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
+    return extMatch ? extMatch[1] : 'jpg';
+}
+async function downloadImages(imgUrls, imgDir, headers, detectExt) {
+    const urlMap = {};
+    if (imgUrls.length === 0)
+        return urlMap;
+    const detect = detectExt || defaultDetectImageExt;
+    // Deduplicate image URLs
+    const seen = new Set();
+    const uniqueUrls = imgUrls.filter(url => {
+        if (seen.has(url))
+            return false;
+        seen.add(url);
+        return true;
+    });
+    for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
+        const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
+        const results = await Promise.all(batch.map(async (rawUrl, j) => {
+            const index = i + j + 1;
+            let imgUrl = rawUrl;
+            if (imgUrl.startsWith('//'))
+                imgUrl = `https:${imgUrl}`;
+            const ext = detect(imgUrl);
+            const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
+            const filepath = path.join(imgDir, filename);
+            try {
+                const result = await httpDownload(imgUrl, filepath, {
+                    headers,
+                    timeout: 15000,
+                });
+                if (result.success) {
+                    return { remoteUrl: rawUrl, localPath: `images/${filename}` };
+                }
+            }
+            catch {
+                // Skip failed downloads
+            }
+            return null;
+        }));
+        for (const r of results) {
+            if (r)
+                urlMap[r.remoteUrl] = r.localPath;
+        }
+    }
+    return urlMap;
+}
+// ============================================================
+// Main API
+// ============================================================
+/**
+ * Download an article to Markdown with optional image localization.
+ *
+ * Handles the full pipeline:
+ * 1. HTML → Markdown (via TurndownService)
+ * 2. Code block placeholder restoration
+ * 3. Batch image downloading with concurrency + deduplication
+ * 4. Image URL replacement in Markdown
+ * 5. Frontmatter generation (customizable labels)
+ * 6. File write
+ */
+export async function downloadArticle(data, options) {
+    const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, } = options;
+    const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
+    if (!data.title) {
+        return [{
+                title: 'Error',
+                author: '-',
+                publish_time: '-',
+                status: 'failed — no title',
+                size: '-',
+            }];
+    }
+    if (!data.contentHtml) {
+        return [{
+                title: data.title,
+                author: data.author || '-',
+                publish_time: data.publishTime || '-',
+                status: 'failed — no content',
+                size: '-',
+            }];
+    }
+    // Convert HTML to Markdown
+    let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown);
+    // Prepare output directory
+    const safeTitle = sanitizeFilename(data.title, maxTitleLength);
+    const articleDir = path.join(output, safeTitle);
+    fs.mkdirSync(articleDir, { recursive: true });
+    // Download images
+    if (shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
+        const imagesDir = path.join(articleDir, 'images');
+        fs.mkdirSync(imagesDir, { recursive: true });
+        const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
+        markdown = replaceImageUrls(markdown, urlMap);
+    }
+    // Build frontmatter with customizable labels
+    const headerLines = [`# ${data.title}`, ''];
+    if (data.author)
+        headerLines.push(`> ${labels.author}: ${data.author}`);
+    if (data.publishTime)
+        headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
+    if (data.sourceUrl)
+        headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
+    headerLines.push('', '---', '');
+    const fullContent = headerLines.join('\n') + markdown;
+    // Write file
+    const filename = `${safeTitle}.md`;
+    const filePath = path.join(articleDir, filename);
+    fs.writeFileSync(filePath, fullContent, 'utf-8');
+    const size = Buffer.byteLength(fullContent, 'utf-8');
+    return [{
+            title: data.title,
+            author: data.author || '-',
+            publish_time: data.publishTime || '-',
+            status: 'success',
+            size: formatBytes(size),
+        }];
+}

package/vendor/opencli/dist/download/index.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+import type { BrowserCookie } from '../types.js';
+export type { BrowserCookie } from '../types.js';
+export interface DownloadOptions {
+    cookies?: string;
+    headers?: Record<string, string>;
+    timeout?: number;
+    onProgress?: (received: number, total: number) => void;
+    maxRedirects?: number;
+}
+export interface YtdlpOptions {
+    cookies?: string;
+    cookiesFile?: string;
+    format?: string;
+    extraArgs?: string[];
+    onProgress?: (percent: number) => void;
+}
+/** Check if yt-dlp is available in PATH. */
+export declare function checkYtdlp(): boolean;
+/**
+ * Detect content type from URL and optional headers.
+ */
+export declare function detectContentType(url: string, contentType?: string): 'image' | 'video' | 'document' | 'binary';
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export declare function requiresYtdlp(url: string): boolean;
+/**
+ * HTTP download with progress callback.
+ */
+export declare function httpDownload(url: string, destPath: string, options?: DownloadOptions, redirectCount?: number): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+export declare function resolveRedirectUrl(currentUrl: string, location: string): string;
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export declare function exportCookiesToNetscape(cookies: BrowserCookie[], filePath: string): void;
+export declare function formatCookieHeader(cookies: BrowserCookie[]): string;
+/**
+ * Download video using yt-dlp.
+ */
+export declare function ytdlpDownload(url: string, destPath: string, options?: YtdlpOptions): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Save document content to file.
+ */
+export declare function saveDocument(content: string, destPath: string, format?: 'json' | 'markdown' | 'html' | 'text', metadata?: Record<string, any>): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export declare function sanitizeFilename(name: string, maxLength?: number): string;
+/**
+ * Generate filename from URL if not provided.
+ */
+export declare function generateFilename(url: string, index: number, extension?: string): string;
+/**
+ * Get temp directory for cookie files.
+ */
+export declare function getTempDir(): string;

package/vendor/opencli/dist/download/index.js

@@ -0,0 +1,345 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+import { spawn } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { URL } from 'node:url';
+import { isBinaryInstalled } from '../external.js';
+import { getErrorMessage } from '../errors.js';
+import { fetchWithNodeNetwork } from '../node-network.js';
+/** Check if yt-dlp is available in PATH. */
+export function checkYtdlp() {
+    return isBinaryInstalled('yt-dlp');
+}
+/** Domains that host video content and can be downloaded via yt-dlp. */
+const VIDEO_PLATFORM_DOMAINS = [
+    'youtube.com', 'youtu.be', 'bilibili.com', 'twitter.com',
+    'x.com', 'tiktok.com', 'vimeo.com', 'twitch.tv',
+];
+const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif']);
+const VIDEO_EXTENSIONS = new Set(['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts']);
+const DOC_EXTENSIONS = new Set(['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown']);
+/**
+ * Detect content type from URL and optional headers.
+ */
+export function detectContentType(url, contentType) {
+    if (contentType) {
+        if (contentType.startsWith('image/'))
+            return 'image';
+        if (contentType.startsWith('video/'))
+            return 'video';
+        if (contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml'))
+            return 'document';
+    }
+    const urlLower = url.toLowerCase();
+    const ext = path.extname(new URL(url).pathname).toLowerCase();
+    if (IMAGE_EXTENSIONS.has(ext))
+        return 'image';
+    if (VIDEO_EXTENSIONS.has(ext))
+        return 'video';
+    if (VIDEO_PLATFORM_DOMAINS.some(d => urlLower.includes(d)))
+        return 'video';
+    if (DOC_EXTENSIONS.has(ext))
+        return 'document';
+    return 'binary';
+}
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export function requiresYtdlp(url) {
+    const urlLower = url.toLowerCase();
+    return VIDEO_PLATFORM_DOMAINS.some(d => urlLower.includes(d));
+}
+/**
+ * HTTP download with progress callback.
+ */
+export async function httpDownload(url, destPath, options = {}, redirectCount = 0) {
+    const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;
+    return new Promise((resolve) => {
+        const requestHeaders = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
+            ...headers,
+        };
+        if (cookies) {
+            requestHeaders['Cookie'] = cookies;
+        }
+        const tempPath = `${destPath}.tmp`;
+        let settled = false;
+        const finish = (result) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve(result);
+        };
+        const cleanupTempFile = async () => {
+            try {
+                await fs.promises.rm(tempPath, { force: true });
+            }
+            catch {
+                // Ignore cleanup errors so the original failure is preserved.
+            }
+        };
+        void (async () => {
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeout);
+            try {
+                const response = await fetchWithNodeNetwork(url, {
+                    headers: requestHeaders,
+                    signal: controller.signal,
+                    redirect: 'manual',
+                });
+                clearTimeout(timer);
+                // Handle redirects before creating any file handles.
+                if (response.status >= 300 && response.status < 400) {
+                    const location = response.headers.get('location');
+                    if (location) {
+                        if (redirectCount >= maxRedirects) {
+                            finish({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+                            return;
+                        }
+                        const redirectUrl = resolveRedirectUrl(url, location);
+                        const originalHost = new URL(url).hostname;
+                        const redirectHost = new URL(redirectUrl).hostname;
+                        const redirectOptions = originalHost === redirectHost
+                            ? options
+                            : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
+                        finish(await httpDownload(redirectUrl, destPath, redirectOptions, redirectCount + 1));
+                        return;
+                    }
+                }
+                if (response.status !== 200) {
+                    finish({ success: false, size: 0, error: `HTTP ${response.status}` });
+                    return;
+                }
+                if (!response.body) {
+                    finish({ success: false, size: 0, error: 'Empty response body' });
+                    return;
+                }
+                const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
+                let received = 0;
+                const progressStream = new Transform({
+                    transform(chunk, _encoding, callback) {
+                        received += chunk.length;
+                        if (onProgress)
+                            onProgress(received, totalSize);
+                        callback(null, chunk);
+                    },
+                });
+                try {
+                    await fs.promises.mkdir(path.dirname(destPath), { recursive: true });
+                    await pipeline(Readable.fromWeb(response.body), progressStream, fs.createWriteStream(tempPath));
+                    await fs.promises.rename(tempPath, destPath);
+                    finish({ success: true, size: received });
+                }
+                catch (err) {
+                    await cleanupTempFile();
+                    finish({ success: false, size: 0, error: getErrorMessage(err) });
+                }
+            }
+            catch (err) {
+                clearTimeout(timer);
+                await cleanupTempFile();
+                finish({ success: false, size: 0, error: err instanceof Error ? err.message : String(err) });
+            }
+        })();
+    });
+}
+export function resolveRedirectUrl(currentUrl, location) {
+    return new URL(location, currentUrl).toString();
+}
+function stripCookieHeaders(headers) {
+    if (!headers)
+        return headers;
+    return Object.fromEntries(Object.entries(headers).filter(([key]) => key.toLowerCase() !== 'cookie'));
+}
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export function exportCookiesToNetscape(cookies, filePath) {
+    const lines = [
+        '# Netscape HTTP Cookie File',
+        '# https://curl.se/docs/http-cookies.html',
+        '# This is a generated file!  Do not edit.',
+        '',
+    ];
+    for (const cookie of cookies) {
+        const domain = cookie.domain.startsWith('.') ? cookie.domain : `.${cookie.domain}`;
+        const includeSubdomains = 'TRUE';
+        const cookiePath = cookie.path || '/';
+        const secure = cookie.secure ? 'TRUE' : 'FALSE';
+        const expiry = Math.floor(Date.now() / 1000) + 86400 * 365; // 1 year from now
+        const safeName = cookie.name.replace(/[\t\n\r]/g, '');
+        const safeValue = cookie.value.replace(/[\t\n\r]/g, '');
+        lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${safeName}\t${safeValue}`);
+    }
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, lines.join('\n'));
+}
+export function formatCookieHeader(cookies) {
+    return cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ');
+}
+/**
+ * Download video using yt-dlp.
+ */
+export async function ytdlpDownload(url, destPath, options = {}) {
+    const { cookiesFile, format = 'best', extraArgs = [], onProgress } = options;
+    if (!checkYtdlp()) {
+        return { success: false, size: 0, error: 'yt-dlp not installed. Install with: pip install yt-dlp' };
+    }
+    return new Promise((resolve) => {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        // Build yt-dlp arguments
+        const args = [
+            url,
+            '-o', destPath,
+            '-f', format,
+            '--no-playlist',
+            '--progress',
+        ];
+        if (cookiesFile) {
+            if (fs.existsSync(cookiesFile)) {
+                args.push('--cookies', cookiesFile);
+            }
+            else {
+                console.error(`[download] Cookies file not found: ${cookiesFile}, falling back to browser cookies`);
+                args.push('--cookies-from-browser', 'chrome');
+            }
+        }
+        else {
+            // Try to use browser cookies
+            args.push('--cookies-from-browser', 'chrome');
+        }
+        args.push(...extraArgs);
+        const proc = spawn('yt-dlp', args, {
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let lastPercent = 0;
+        let errorOutput = '';
+        proc.stderr.on('data', (data) => {
+            const line = data.toString();
+            errorOutput += line;
+            // Parse progress from yt-dlp output
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.stdout.on('data', (data) => {
+            const line = data.toString();
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.on('close', (code) => {
+            if (code === 0 && fs.existsSync(destPath)) {
+                const stats = fs.statSync(destPath);
+                resolve({ success: true, size: stats.size });
+            }
+            else {
+                // Check for common yt-dlp output patterns
+                const patterns = fs.readdirSync(dir).filter(f => f.startsWith(path.basename(destPath, path.extname(destPath))));
+                if (patterns.length > 0) {
+                    const actualFile = path.join(dir, patterns[0]);
+                    const stats = fs.statSync(actualFile);
+                    resolve({ success: true, size: stats.size });
+                }
+                else {
+                    resolve({ success: false, size: 0, error: errorOutput.slice(0, 200) || `Exit code ${code}` });
+                }
+            }
+        });
+        proc.on('error', (err) => {
+            resolve({ success: false, size: 0, error: err.message });
+        });
+    });
+}
+/**
+ * Save document content to file.
+ */
+export async function saveDocument(content, destPath, format = 'markdown', metadata) {
+    try {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        let output;
+        if (format === 'json') {
+            output = JSON.stringify({ ...metadata, content }, null, 2);
+        }
+        else if (format === 'markdown') {
+            // Add frontmatter if metadata exists
+            const frontmatter = metadata ? `---\n${Object.entries(metadata).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join('\n')}\n---\n\n` : '';
+            output = frontmatter + content;
+        }
+        else {
+            output = content;
+        }
+        fs.writeFileSync(destPath, output, 'utf-8');
+        return { success: true, size: Buffer.byteLength(output, 'utf-8') };
+    }
+    catch (err) {
+        return { success: false, size: 0, error: getErrorMessage(err) };
+    }
+}
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export function sanitizeFilename(name, maxLength = 200) {
+    return name
+        .replace(/[<>:"/\\|?*\x00-\x1f]/g, '_') // Remove invalid chars
+        .replace(/\s+/g, '_') // Replace spaces with underscores
+        .replace(/_+/g, '_') // Collapse multiple underscores
+        .replace(/^_|_$/g, '') // Trim underscores
+        .slice(0, maxLength);
+}
+/**
+ * Generate filename from URL if not provided.
+ */
+export function generateFilename(url, index, extension) {
+    try {
+        const parsedUrl = new URL(url);
+        const pathname = parsedUrl.pathname;
+        const basename = path.basename(pathname);
+        if (basename && basename !== '/' && basename.includes('.')) {
+            return sanitizeFilename(basename);
+        }
+        // Generate from hostname and index
+        const ext = extension || detectExtension(url);
+        const hostname = parsedUrl.hostname.replace(/^www\./, '');
+        return sanitizeFilename(`${hostname}_${index + 1}${ext}`);
+    }
+    catch {
+        const ext = extension || '.bin';
+        return `download_${index + 1}${ext}`;
+    }
+}
+/**
+ * Detect file extension from URL.
+ */
+function detectExtension(url) {
+    const type = detectContentType(url);
+    switch (type) {
+        case 'image': return '.jpg';
+        case 'video': return '.mp4';
+        case 'document': return '.md';
+        default: return '.bin';
+    }
+}
+/**
+ * Get temp directory for cookie files.
+ */
+export function getTempDir() {
+    return path.join(os.tmpdir(), 'opencli-download');
+}

package/vendor/opencli/dist/download/index.test.d.ts

@@ -0,0 +1 @@
+export {};