@jackwener/opencli 0.9.6 → 0.9.8

package/dist/download/index.js

@@ -0,0 +1,325 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+import { spawn, execSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as https from 'node:https';
+import * as http from 'node:http';
+import * as os from 'node:os';
+import { URL } from 'node:url';
+/**
+ * Check if yt-dlp is available in PATH.
+ */
+export function checkYtdlp() {
+    try {
+        execSync('yt-dlp --version', { encoding: 'utf-8', stdio: 'pipe' });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Check if ffmpeg is available in PATH.
+ */
+export function checkFfmpeg() {
+    try {
+        execSync('ffmpeg -version', { encoding: 'utf-8', stdio: 'pipe' });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Detect content type from URL and optional headers.
+ */
+export function detectContentType(url, contentType) {
+    // Check content-type header first
+    if (contentType) {
+        if (contentType.startsWith('image/'))
+            return 'image';
+        if (contentType.startsWith('video/'))
+            return 'video';
+        if (contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml'))
+            return 'document';
+    }
+    // Detect from URL
+    const urlLower = url.toLowerCase();
+    const ext = path.extname(new URL(url).pathname).toLowerCase();
+    // Image extensions
+    if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif'].includes(ext)) {
+        return 'image';
+    }
+    // Video extensions
+    if (['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts'].includes(ext)) {
+        return 'video';
+    }
+    // Video platforms (need yt-dlp)
+    if (urlLower.includes('youtube.com') || urlLower.includes('youtu.be') ||
+        urlLower.includes('bilibili.com') || urlLower.includes('twitter.com') ||
+        urlLower.includes('x.com') || urlLower.includes('tiktok.com') ||
+        urlLower.includes('vimeo.com') || urlLower.includes('twitch.tv')) {
+        return 'video';
+    }
+    // Document extensions
+    if (['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown'].includes(ext)) {
+        return 'document';
+    }
+    return 'binary';
+}
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export function requiresYtdlp(url) {
+    const urlLower = url.toLowerCase();
+    return (urlLower.includes('youtube.com') ||
+        urlLower.includes('youtu.be') ||
+        urlLower.includes('bilibili.com/video') ||
+        urlLower.includes('twitter.com') ||
+        urlLower.includes('x.com') ||
+        urlLower.includes('tiktok.com') ||
+        urlLower.includes('vimeo.com') ||
+        urlLower.includes('twitch.tv'));
+}
+/**
+ * HTTP download with progress callback.
+ */
+export async function httpDownload(url, destPath, options = {}) {
+    const { cookies, headers = {}, timeout = 30000, onProgress } = options;
+    return new Promise((resolve) => {
+        const parsedUrl = new URL(url);
+        const protocol = parsedUrl.protocol === 'https:' ? https : http;
+        const requestHeaders = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            ...headers,
+        };
+        if (cookies) {
+            requestHeaders['Cookie'] = cookies;
+        }
+        // Ensure directory exists
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        const tempPath = `${destPath}.tmp`;
+        const file = fs.createWriteStream(tempPath);
+        const request = protocol.get(url, { headers: requestHeaders, timeout }, (response) => {
+            // Handle redirects
+            if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
+                file.close();
+                fs.unlinkSync(tempPath);
+                httpDownload(response.headers.location, destPath, options).then(resolve);
+                return;
+            }
+            if (response.statusCode !== 200) {
+                file.close();
+                fs.unlinkSync(tempPath);
+                resolve({ success: false, size: 0, error: `HTTP ${response.statusCode}` });
+                return;
+            }
+            const totalSize = parseInt(response.headers['content-length'] || '0', 10);
+            let received = 0;
+            response.on('data', (chunk) => {
+                received += chunk.length;
+                if (onProgress)
+                    onProgress(received, totalSize);
+            });
+            response.pipe(file);
+            file.on('finish', () => {
+                file.close();
+                // Rename temp file to final destination
+                fs.renameSync(tempPath, destPath);
+                resolve({ success: true, size: received });
+            });
+        });
+        request.on('error', (err) => {
+            file.close();
+            if (fs.existsSync(tempPath))
+                fs.unlinkSync(tempPath);
+            resolve({ success: false, size: 0, error: err.message });
+        });
+        request.on('timeout', () => {
+            request.destroy();
+            file.close();
+            if (fs.existsSync(tempPath))
+                fs.unlinkSync(tempPath);
+            resolve({ success: false, size: 0, error: 'Timeout' });
+        });
+    });
+}
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export function exportCookiesToNetscape(cookies, filePath) {
+    const lines = [
+        '# Netscape HTTP Cookie File',
+        '# https://curl.se/docs/http-cookies.html',
+        '# This is a generated file!  Do not edit.',
+        '',
+    ];
+    for (const cookie of cookies) {
+        const domain = cookie.domain.startsWith('.') ? cookie.domain : `.${cookie.domain}`;
+        const includeSubdomains = 'TRUE';
+        const cookiePath = cookie.path || '/';
+        const secure = cookie.secure ? 'TRUE' : 'FALSE';
+        const expiry = Math.floor(Date.now() / 1000) + 86400 * 365; // 1 year from now
+        lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${cookie.name}\t${cookie.value}`);
+    }
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, lines.join('\n'));
+}
+/**
+ * Download video using yt-dlp.
+ */
+export async function ytdlpDownload(url, destPath, options = {}) {
+    const { cookiesFile, format = 'best', extraArgs = [], onProgress } = options;
+    if (!checkYtdlp()) {
+        return { success: false, size: 0, error: 'yt-dlp not installed. Install with: pip install yt-dlp' };
+    }
+    return new Promise((resolve) => {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        // Build yt-dlp arguments
+        const args = [
+            url,
+            '-o', destPath,
+            '-f', format,
+            '--no-playlist',
+            '--progress',
+        ];
+        if (cookiesFile && fs.existsSync(cookiesFile)) {
+            args.push('--cookies', cookiesFile);
+        }
+        else {
+            // Try to use browser cookies
+            args.push('--cookies-from-browser', 'chrome');
+        }
+        args.push(...extraArgs);
+        const proc = spawn('yt-dlp', args, {
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let lastPercent = 0;
+        let errorOutput = '';
+        proc.stderr.on('data', (data) => {
+            const line = data.toString();
+            errorOutput += line;
+            // Parse progress from yt-dlp output
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.stdout.on('data', (data) => {
+            const line = data.toString();
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.on('close', (code) => {
+            if (code === 0 && fs.existsSync(destPath)) {
+                const stats = fs.statSync(destPath);
+                resolve({ success: true, size: stats.size });
+            }
+            else {
+                // Check for common yt-dlp output patterns
+                const patterns = fs.readdirSync(dir).filter(f => f.startsWith(path.basename(destPath, path.extname(destPath))));
+                if (patterns.length > 0) {
+                    const actualFile = path.join(dir, patterns[0]);
+                    const stats = fs.statSync(actualFile);
+                    resolve({ success: true, size: stats.size });
+                }
+                else {
+                    resolve({ success: false, size: 0, error: errorOutput.slice(0, 200) || `Exit code ${code}` });
+                }
+            }
+        });
+        proc.on('error', (err) => {
+            resolve({ success: false, size: 0, error: err.message });
+        });
+    });
+}
+/**
+ * Save document content to file.
+ */
+export async function saveDocument(content, destPath, format = 'markdown', metadata) {
+    try {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        let output;
+        if (format === 'json') {
+            output = JSON.stringify({ ...metadata, content }, null, 2);
+        }
+        else if (format === 'markdown') {
+            // Add frontmatter if metadata exists
+            const frontmatter = metadata ? `---\n${Object.entries(metadata).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join('\n')}\n---\n\n` : '';
+            output = frontmatter + content;
+        }
+        else {
+            output = content;
+        }
+        fs.writeFileSync(destPath, output, 'utf-8');
+        return { success: true, size: Buffer.byteLength(output, 'utf-8') };
+    }
+    catch (err) {
+        return { success: false, size: 0, error: err.message };
+    }
+}
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export function sanitizeFilename(name, maxLength = 200) {
+    return name
+        .replace(/[<>:"/\\|?*\x00-\x1f]/g, '_') // Remove invalid chars
+        .replace(/\s+/g, '_') // Replace spaces with underscores
+        .replace(/_+/g, '_') // Collapse multiple underscores
+        .replace(/^_|_$/g, '') // Trim underscores
+        .slice(0, maxLength);
+}
+/**
+ * Generate filename from URL if not provided.
+ */
+export function generateFilename(url, index, extension) {
+    try {
+        const parsedUrl = new URL(url);
+        const pathname = parsedUrl.pathname;
+        const basename = path.basename(pathname);
+        if (basename && basename !== '/' && basename.includes('.')) {
+            return sanitizeFilename(basename);
+        }
+        // Generate from hostname and index
+        const ext = extension || detectExtension(url);
+        const hostname = parsedUrl.hostname.replace(/^www\./, '');
+        return sanitizeFilename(`${hostname}_${index + 1}${ext}`);
+    }
+    catch {
+        const ext = extension || '.bin';
+        return `download_${index + 1}${ext}`;
+    }
+}
+/**
+ * Detect file extension from URL.
+ */
+function detectExtension(url) {
+    const type = detectContentType(url);
+    switch (type) {
+        case 'image': return '.jpg';
+        case 'video': return '.mp4';
+        case 'document': return '.md';
+        default: return '.bin';
+    }
+}
+/**
+ * Get temp directory for cookie files.
+ */
+export function getTempDir() {
+    return path.join(os.tmpdir(), 'opencli-download');
+}

package/dist/download/progress.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Download progress display: terminal progress bars, status updates.
+ */
+export interface ProgressBar {
+    update(current: number, total: number, label?: string): void;
+    complete(success: boolean, message?: string): void;
+    fail(error: string): void;
+}
+/**
+ * Format bytes as human-readable string (KB, MB, GB).
+ */
+export declare function formatBytes(bytes: number): string;
+/**
+ * Format milliseconds as human-readable duration.
+ */
+export declare function formatDuration(ms: number): string;
+/**
+ * Create a simple progress bar for terminal display.
+ */
+export declare function createProgressBar(filename: string, index: number, total: number): ProgressBar;
+/**
+ * Multi-file download progress tracker.
+ */
+export declare class DownloadProgressTracker {
+    private completed;
+    private failed;
+    private skipped;
+    private total;
+    private startTime;
+    private verbose;
+    constructor(total: number, verbose?: boolean);
+    onFileStart(filename: string, index: number): ProgressBar | null;
+    onFileComplete(success: boolean, skipped?: boolean): void;
+    getSummary(): string;
+    finish(): void;
+}

package/dist/download/progress.js

@@ -0,0 +1,111 @@
+/**
+ * Download progress display: terminal progress bars, status updates.
+ */
+import chalk from 'chalk';
+/**
+ * Format bytes as human-readable string (KB, MB, GB).
+ */
+export function formatBytes(bytes) {
+    if (bytes === 0)
+        return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
+}
+/**
+ * Format milliseconds as human-readable duration.
+ */
+export function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    const seconds = Math.floor(ms / 1000);
+    if (seconds < 60)
+        return `${seconds}s`;
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = seconds % 60;
+    return `${minutes}m ${remainingSeconds}s`;
+}
+/**
+ * Create a simple progress bar for terminal display.
+ */
+export function createProgressBar(filename, index, total) {
+    const prefix = chalk.dim(`[${index + 1}/${total}]`);
+    const truncatedName = filename.length > 40 ? filename.slice(0, 37) + '...' : filename;
+    return {
+        update(current, totalBytes, label) {
+            const percent = totalBytes > 0 ? Math.round((current / totalBytes) * 100) : 0;
+            const bar = createBar(percent);
+            const size = totalBytes > 0 ? formatBytes(totalBytes) : '';
+            const extra = label ? ` ${label}` : '';
+            process.stderr.write(`\r${prefix} ${truncatedName} ${bar} ${percent}% ${size}${extra}`);
+        },
+        complete(success, message) {
+            const icon = success ? chalk.green('✓') : chalk.red('✗');
+            const msg = message ? ` ${chalk.dim(message)}` : '';
+            process.stderr.write(`\r${prefix} ${icon} ${truncatedName}${msg}\n`);
+        },
+        fail(error) {
+            process.stderr.write(`\r${prefix} ${chalk.red('✗')} ${truncatedName} ${chalk.red(error)}\n`);
+        },
+    };
+}
+/**
+ * Create a progress bar string.
+ */
+function createBar(percent, width = 20) {
+    const filled = Math.round((percent / 100) * width);
+    const empty = width - filled;
+    return chalk.cyan('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
+}
+/**
+ * Multi-file download progress tracker.
+ */
+export class DownloadProgressTracker {
+    completed = 0;
+    failed = 0;
+    skipped = 0;
+    total;
+    startTime;
+    verbose;
+    constructor(total, verbose = true) {
+        this.total = total;
+        this.startTime = Date.now();
+        this.verbose = verbose;
+    }
+    onFileStart(filename, index) {
+        if (!this.verbose)
+            return null;
+        return createProgressBar(filename, index, this.total);
+    }
+    onFileComplete(success, skipped = false) {
+        if (skipped) {
+            this.skipped++;
+        }
+        else if (success) {
+            this.completed++;
+        }
+        else {
+            this.failed++;
+        }
+    }
+    getSummary() {
+        const elapsed = formatDuration(Date.now() - this.startTime);
+        const parts = [];
+        if (this.completed > 0) {
+            parts.push(chalk.green(`${this.completed} downloaded`));
+        }
+        if (this.skipped > 0) {
+            parts.push(chalk.yellow(`${this.skipped} skipped`));
+        }
+        if (this.failed > 0) {
+            parts.push(chalk.red(`${this.failed} failed`));
+        }
+        return `${parts.join(', ')} in ${elapsed}`;
+    }
+    finish() {
+        if (this.verbose) {
+            process.stderr.write(`\n${chalk.bold('Download complete:')} ${this.getSummary()}\n`);
+        }
+    }
+}

package/dist/engine.test.js

@@ -11,6 +11,21 @@ describe('discoverClis', () => {
     });
 });
 describe('executeCommand', () => {
+    it('accepts kebab-case option names after Commander camelCases them', async () => {
+        const cmd = cli({
+            site: 'test-engine',
+            name: 'kebab-arg-test',
+            description: 'test command with kebab-case arg',
+            browser: false,
+            strategy: Strategy.PUBLIC,
+            args: [
+                { name: 'note-id', required: true, help: 'Note ID' },
+            ],
+            func: async (_page, kwargs) => [{ noteId: kwargs['note-id'] }],
+        });
+        const result = await executeCommand(cmd, null, { 'note-id': 'abc123' });
+        expect(result).toEqual([{ noteId: 'abc123' }]);
+    });
     it('executes a command with func', async () => {
         const cmd = cli({
             site: 'test-engine',

package/dist/main.js

@@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url';
 import { Command } from 'commander';
 import chalk from 'chalk';
 import { discoverClis, executeCommand } from './engine.js';
-import { fullName, getRegistry, strategyLabel } from './registry.js';
+import { Strategy, fullName, getRegistry, strategyLabel } from './registry.js';
 import { render as renderOutput } from './output.js';
 import { PlaywrightMCP } from './browser/index.js';
 import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
@@ -205,7 +205,8 @@ for (const [, cmd] of registry) {
         for (const arg of cmd.args) {
             if (arg.positional)
                 continue;
-            const v = actionOpts[arg.name];
+            const camelName = arg.name.replace(/-([a-z])/g, (_m, ch) => ch.toUpperCase());
+            const v = actionOpts[arg.name] ?? actionOpts[camelName];
             if (v !== undefined)
                 kwargs[arg.name] = v;
         }
@@ -214,7 +215,19 @@ for (const [, cmd] of registry) {
                 process.env.OPENCLI_VERBOSE = '1';
             let result;
             if (cmd.browser) {
-                result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
+                result = await browserSession(PlaywrightMCP, async (page) => {
+                    // Cookie/header strategies require same-origin context for credentialed fetch.
+                    // In CDP mode the active tab may be on an unrelated domain, causing CORS failures.
+                    // Navigate to the command's domain first (mirrors cascade command behavior).
+                    if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
+                        try {
+                            await page.goto(`https://${cmd.domain}`);
+                            await page.wait(2);
+                        }
+                        catch { }
+                    }
+                    return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
+                });
             }
             else {
                 result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);

package/dist/pipeline/registry.js

@@ -8,6 +8,7 @@ import { stepFetch } from './steps/fetch.js';
 import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
 import { stepIntercept } from './steps/intercept.js';
 import { stepTap } from './steps/tap.js';
+import { stepDownload } from './steps/download.js';
 const _stepRegistry = new Map();
 /**
  * Get a registered step handler by name.
@@ -39,3 +40,4 @@ registerStep('sort', stepSort);
 registerStep('limit', stepLimit);
 registerStep('intercept', stepIntercept);
 registerStep('tap', stepTap);
+registerStep('download', stepDownload);

package/dist/pipeline/steps/download.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import type { IPage } from '../../types.js';
+export interface DownloadResult {
+    status: 'success' | 'skipped' | 'failed';
+    path?: string;
+    size?: number;
+    error?: string;
+    duration?: number;
+}
+/**
+ * Download step handler for YAML pipelines.
+ *
+ * Usage in YAML:
+ * ```yaml
+ * pipeline:
+ *   - download:
+ *       url: ${{ item.imageUrl }}
+ *       dir: ./downloads
+ *       filename: ${{ item.title }}.jpg
+ *       concurrency: 5
+ *       skip_existing: true
+ *       use_ytdlp: false
+ *       type: auto
+ * ```
+ */
+export declare function stepDownload(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any>;