@jackwener/opencli 0.9.6 → 1.0.0

package/dist/download/progress.js

@@ -0,0 +1,111 @@
+/**
+ * Download progress display: terminal progress bars, status updates.
+ */
+import chalk from 'chalk';
+/**
+ * Format bytes as human-readable string (KB, MB, GB).
+ */
+export function formatBytes(bytes) {
+    if (bytes === 0)
+        return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
+}
+/**
+ * Format milliseconds as human-readable duration.
+ */
+export function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    const seconds = Math.floor(ms / 1000);
+    if (seconds < 60)
+        return `${seconds}s`;
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = seconds % 60;
+    return `${minutes}m ${remainingSeconds}s`;
+}
+/**
+ * Create a simple progress bar for terminal display.
+ */
+export function createProgressBar(filename, index, total) {
+    const prefix = chalk.dim(`[${index + 1}/${total}]`);
+    const truncatedName = filename.length > 40 ? filename.slice(0, 37) + '...' : filename;
+    return {
+        update(current, totalBytes, label) {
+            const percent = totalBytes > 0 ? Math.round((current / totalBytes) * 100) : 0;
+            const bar = createBar(percent);
+            const size = totalBytes > 0 ? formatBytes(totalBytes) : '';
+            const extra = label ? ` ${label}` : '';
+            process.stderr.write(`\r${prefix} ${truncatedName} ${bar} ${percent}% ${size}${extra}`);
+        },
+        complete(success, message) {
+            const icon = success ? chalk.green('✓') : chalk.red('✗');
+            const msg = message ? ` ${chalk.dim(message)}` : '';
+            process.stderr.write(`\r${prefix} ${icon} ${truncatedName}${msg}\n`);
+        },
+        fail(error) {
+            process.stderr.write(`\r${prefix} ${chalk.red('✗')} ${truncatedName} ${chalk.red(error)}\n`);
+        },
+    };
+}
+/**
+ * Create a progress bar string.
+ */
+function createBar(percent, width = 20) {
+    const filled = Math.round((percent / 100) * width);
+    const empty = width - filled;
+    return chalk.cyan('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
+}
+/**
+ * Multi-file download progress tracker.
+ */
+export class DownloadProgressTracker {
+    completed = 0;
+    failed = 0;
+    skipped = 0;
+    total;
+    startTime;
+    verbose;
+    constructor(total, verbose = true) {
+        this.total = total;
+        this.startTime = Date.now();
+        this.verbose = verbose;
+    }
+    onFileStart(filename, index) {
+        if (!this.verbose)
+            return null;
+        return createProgressBar(filename, index, this.total);
+    }
+    onFileComplete(success, skipped = false) {
+        if (skipped) {
+            this.skipped++;
+        }
+        else if (success) {
+            this.completed++;
+        }
+        else {
+            this.failed++;
+        }
+    }
+    getSummary() {
+        const elapsed = formatDuration(Date.now() - this.startTime);
+        const parts = [];
+        if (this.completed > 0) {
+            parts.push(chalk.green(`${this.completed} downloaded`));
+        }
+        if (this.skipped > 0) {
+            parts.push(chalk.yellow(`${this.skipped} skipped`));
+        }
+        if (this.failed > 0) {
+            parts.push(chalk.red(`${this.failed} failed`));
+        }
+        return `${parts.join(', ')} in ${elapsed}`;
+    }
+    finish() {
+        if (this.verbose) {
+            process.stderr.write(`\n${chalk.bold('Download complete:')} ${this.getSummary()}\n`);
+        }
+    }
+}

package/dist/engine.test.js

@@ -11,6 +11,21 @@ describe('discoverClis', () => {
     });
 });
 describe('executeCommand', () => {
+    it('accepts kebab-case option names after Commander camelCases them', async () => {
+        const cmd = cli({
+            site: 'test-engine',
+            name: 'kebab-arg-test',
+            description: 'test command with kebab-case arg',
+            browser: false,
+            strategy: Strategy.PUBLIC,
+            args: [
+                { name: 'note-id', required: true, help: 'Note ID' },
+            ],
+            func: async (_page, kwargs) => [{ noteId: kwargs['note-id'] }],
+        });
+        const result = await executeCommand(cmd, null, { 'note-id': 'abc123' });
+        expect(result).toEqual([{ noteId: 'abc123' }]);
+    });
     it('executes a command with func', async () => {
         const cmd = cli({
             site: 'test-engine',

package/dist/main.js

@@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url';
 import { Command } from 'commander';
 import chalk from 'chalk';
 import { discoverClis, executeCommand } from './engine.js';
-import { fullName, getRegistry, strategyLabel } from './registry.js';
+import { Strategy, fullName, getRegistry, strategyLabel } from './registry.js';
 import { render as renderOutput } from './output.js';
 import { PlaywrightMCP } from './browser/index.js';
 import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
@@ -122,37 +122,18 @@ program.command('cascade').description('Strategy cascade: find simplest working
     console.log(renderCascadeResult(result));
 });
 program.command('doctor')
-    .description('Diagnose Playwright MCP Bridge, token consistency, and Chrome remote debugging')
-    .option('--fix', 'Apply suggested fixes to shell rc and detected MCP configs', false)
-    .option('-y, --yes', 'Skip confirmation prompts when applying fixes', false)
-    .option('--token <token>', 'Override token to write instead of auto-detecting')
+    .description('Diagnose opencli browser bridge connectivity')
     .option('--live', 'Test browser connectivity (requires Chrome running)', false)
-    .option('--shell-rc <path>', 'Shell startup file to update')
-    .option('--mcp-config <paths>', 'Comma-separated MCP config paths to scan/update')
     .action(async (opts) => {
-    const { runBrowserDoctor, renderBrowserDoctorReport, applyBrowserDoctorFix } = await import('./doctor.js');
-    const configPaths = opts.mcpConfig ? String(opts.mcpConfig).split(',').map((s) => s.trim()).filter(Boolean) : undefined;
-    const report = await runBrowserDoctor({ token: opts.token, live: opts.live, shellRc: opts.shellRc, configPaths, cliVersion: PKG_VERSION });
+    const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js');
+    const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION });
     console.log(renderBrowserDoctorReport(report));
-    if (opts.fix) {
-        const written = await applyBrowserDoctorFix(report, { fix: true, yes: opts.yes, token: opts.token, shellRc: opts.shellRc, configPaths });
-        console.log();
-        if (written.length > 0) {
-            console.log(chalk.green('Updated files:'));
-            for (const filePath of written)
-                console.log(`- ${filePath}`);
-        }
-        else {
-            console.log(chalk.yellow('No files were changed.'));
-        }
-    }
 });
 program.command('setup')
-    .description('Interactive setup: configure Playwright MCP token across all detected tools')
-    .option('--token <token>', 'Provide token directly instead of auto-detecting')
-    .action(async (opts) => {
+    .description('Interactive setup: verify browser bridge connectivity')
+    .action(async () => {
     const { runSetup } = await import('./setup.js');
-    await runSetup({ cliVersion: PKG_VERSION, token: opts.token });
+    await runSetup({ cliVersion: PKG_VERSION });
 });
 program.command('completion')
     .description('Output shell completion script')
@@ -205,7 +186,8 @@ for (const [, cmd] of registry) {
         for (const arg of cmd.args) {
             if (arg.positional)
                 continue;
-            const v = actionOpts[arg.name];
+            const camelName = arg.name.replace(/-([a-z])/g, (_m, ch) => ch.toUpperCase());
+            const v = actionOpts[arg.name] ?? actionOpts[camelName];
             if (v !== undefined)
                 kwargs[arg.name] = v;
         }
@@ -214,7 +196,19 @@ for (const [, cmd] of registry) {
                 process.env.OPENCLI_VERBOSE = '1';
             let result;
             if (cmd.browser) {
-                result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
+                result = await browserSession(PlaywrightMCP, async (page) => {
+                    // Cookie/header strategies require same-origin context for credentialed fetch.
+                    // In CDP mode the active tab may be on an unrelated domain, causing CORS failures.
+                    // Navigate to the command's domain first (mirrors cascade command behavior).
+                    if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
+                        try {
+                            await page.goto(`https://${cmd.domain}`);
+                            await page.wait(2);
+                        }
+                        catch { }
+                    }
+                    return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
+                });
             }
             else {
                 result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);

package/dist/pipeline/executor.test.js

@@ -23,6 +23,7 @@ function createMockPage(overrides = {}) {
         autoScroll: vi.fn(),
         installInterceptor: vi.fn(),
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
+        screenshot: vi.fn().mockResolvedValue(''),
         ...overrides,
     };
 }

package/dist/pipeline/registry.js

@@ -8,6 +8,7 @@ import { stepFetch } from './steps/fetch.js';
 import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
 import { stepIntercept } from './steps/intercept.js';
 import { stepTap } from './steps/tap.js';
+import { stepDownload } from './steps/download.js';
 const _stepRegistry = new Map();
 /**
  * Get a registered step handler by name.
@@ -39,3 +40,4 @@ registerStep('sort', stepSort);
 registerStep('limit', stepLimit);
 registerStep('intercept', stepIntercept);
 registerStep('tap', stepTap);
+registerStep('download', stepDownload);

package/dist/pipeline/steps/browser.js

@@ -2,7 +2,7 @@
  * Pipeline step: navigate, click, type, wait, press, snapshot.
  * Browser interaction primitives.
  */
-import { render, normalizeEvaluateSource } from '../template.js';
+import { render } from '../template.js';
 export async function stepNavigate(page, params, data, args) {
     const url = render(params, { args, data });
     await page.goto(String(url));
@@ -49,7 +49,7 @@ export async function stepSnapshot(page, params, _data, _args) {
 }
 export async function stepEvaluate(page, params, data, args) {
     const js = String(render(params, { args, data }));
-    let result = await page.evaluate(normalizeEvaluateSource(js));
+    let result = await page.evaluate(js);
     // MCP may return JSON as a string — auto-parse it
     if (typeof result === 'string') {
         const trimmed = result.trim();

package/dist/pipeline/steps/download.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import type { IPage } from '../../types.js';
+export interface DownloadResult {
+    status: 'success' | 'skipped' | 'failed';
+    path?: string;
+    size?: number;
+    error?: string;
+    duration?: number;
+}
+/**
+ * Download step handler for YAML pipelines.
+ *
+ * Usage in YAML:
+ * ```yaml
+ * pipeline:
+ *   - download:
+ *       url: ${{ item.imageUrl }}
+ *       dir: ./downloads
+ *       filename: ${{ item.title }}.jpg
+ *       concurrency: 5
+ *       skip_existing: true
+ *       use_ytdlp: false
+ *       type: auto
+ * ```
+ */
+export declare function stepDownload(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any>;

package/dist/pipeline/steps/download.js

@@ -0,0 +1,251 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { render } from '../template.js';
+import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, } from '../../download/index.js';
+import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+/**
+ * Simple async concurrency limiter for downloads.
+ */
+async function mapConcurrent(items, limit, fn) {
+    const results = new Array(items.length);
+    let index = 0;
+    async function worker() {
+        while (index < items.length) {
+            const i = index++;
+            results[i] = await fn(items[i], i);
+        }
+    }
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+/**
+ * Extract cookies from browser page.
+ */
+async function extractBrowserCookies(page, domain) {
+    try {
+        // Use browser evaluate to get document.cookie
+        const cookieString = await page.evaluate(`(() => document.cookie)()`);
+        return typeof cookieString === 'string' ? cookieString : '';
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Extract cookies as array for yt-dlp Netscape format.
+ */
+async function extractCookiesArray(page, domain) {
+    try {
+        const cookieString = await extractBrowserCookies(page);
+        if (!cookieString)
+            return [];
+        return cookieString.split(';').map((c) => {
+            const [name, ...rest] = c.trim().split('=');
+            return {
+                name: name || '',
+                value: rest.join('=') || '',
+                domain,
+                path: '/',
+                secure: true,
+                httpOnly: false,
+            };
+        }).filter((c) => c.name);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Download step handler for YAML pipelines.
+ *
+ * Usage in YAML:
+ * ```yaml
+ * pipeline:
+ *   - download:
+ *       url: ${{ item.imageUrl }}
+ *       dir: ./downloads
+ *       filename: ${{ item.title }}.jpg
+ *       concurrency: 5
+ *       skip_existing: true
+ *       use_ytdlp: false
+ *       type: auto
+ * ```
+ */
+export async function stepDownload(page, params, data, args) {
+    // Parse parameters with defaults
+    const urlTemplate = typeof params === 'string' ? params : (params?.url ?? '');
+    const dirTemplate = params?.dir ?? './downloads';
+    const filenameTemplate = params?.filename ?? '';
+    const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 3;
+    const skipExisting = params?.skip_existing !== false;
+    const timeout = typeof params?.timeout === 'number' ? params.timeout * 1000 : 30000;
+    const useYtdlp = params?.use_ytdlp ?? false;
+    const ytdlpArgs = Array.isArray(params?.ytdlp_args) ? params.ytdlp_args : [];
+    const contentType = params?.type ?? 'auto';
+    const showProgress = params?.progress !== false;
+    const contentTemplate = params?.content;
+    const metadataTemplate = params?.metadata;
+    // Resolve output directory
+    const dir = String(render(dirTemplate, { args, data }));
+    fs.mkdirSync(dir, { recursive: true });
+    // Normalize data to array
+    const items = Array.isArray(data) ? data : data ? [data] : [];
+    if (items.length === 0) {
+        return [];
+    }
+    // Create progress tracker
+    const tracker = new DownloadProgressTracker(items.length, showProgress);
+    // Extract cookies if browser is available
+    let cookies = '';
+    let cookiesFile;
+    if (page) {
+        cookies = await extractBrowserCookies(page);
+        // For yt-dlp, we need to export cookies to Netscape format
+        if (useYtdlp || items.some((item, index) => {
+            const url = String(render(urlTemplate, { args, data, item, index }));
+            return requiresYtdlp(url);
+        })) {
+            try {
+                // Try to get domain from first URL
+                const firstUrl = String(render(urlTemplate, { args, data, item: items[0], index: 0 }));
+                const domain = new URL(firstUrl).hostname;
+                const cookiesArray = await extractCookiesArray(page, domain);
+                if (cookiesArray.length > 0) {
+                    const tempDir = getTempDir();
+                    fs.mkdirSync(tempDir, { recursive: true });
+                    cookiesFile = path.join(tempDir, `cookies_${Date.now()}.txt`);
+                    exportCookiesToNetscape(cookiesArray, cookiesFile);
+                }
+            }
+            catch {
+                // Ignore cookie extraction errors
+            }
+        }
+    }
+    // Process downloads with concurrency
+    const results = await mapConcurrent(items, concurrency, async (item, index) => {
+        const startTime = Date.now();
+        // Render URL
+        const url = String(render(urlTemplate, { args, data, item, index }));
+        if (!url) {
+            tracker.onFileComplete(false);
+            return {
+                ...item,
+                _download: { status: 'failed', error: 'Empty URL' },
+            };
+        }
+        // Render filename
+        let filename;
+        if (filenameTemplate) {
+            filename = String(render(filenameTemplate, { args, data, item, index }));
+        }
+        else {
+            filename = generateFilename(url, index);
+        }
+        filename = sanitizeFilename(filename);
+        const destPath = path.join(dir, filename);
+        // Check if file exists and skip_existing is true
+        if (skipExisting && fs.existsSync(destPath)) {
+            tracker.onFileComplete(true, true);
+            return {
+                ...item,
+                _download: {
+                    status: 'skipped',
+                    path: destPath,
+                    size: fs.statSync(destPath).size,
+                },
+            };
+        }
+        // Create progress bar for this file
+        const progressBar = tracker.onFileStart(filename, index);
+        // Determine download method
+        const detectedType = contentType === 'auto' ? detectContentType(url) : contentType;
+        const shouldUseYtdlp = useYtdlp || (detectedType === 'video' && requiresYtdlp(url));
+        let result;
+        try {
+            if (detectedType === 'document' && contentTemplate) {
+                // Save extracted content as document
+                const content = String(render(contentTemplate, { args, data, item, index }));
+                const metadata = metadataTemplate
+                    ? Object.fromEntries(Object.entries(metadataTemplate).map(([k, v]) => [k, render(v, { args, data, item, index })]))
+                    : undefined;
+                const ext = path.extname(filename).toLowerCase();
+                const format = ext === '.json' ? 'json' : ext === '.html' ? 'html' : 'markdown';
+                result = await saveDocument(content, destPath, format, metadata);
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else if (shouldUseYtdlp) {
+                // Use yt-dlp for video downloads
+                result = await ytdlpDownload(url, destPath, {
+                    cookiesFile,
+                    extraArgs: ytdlpArgs,
+                    onProgress: (percent) => {
+                        if (progressBar) {
+                            progressBar.update(percent, 100);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else {
+                // Direct HTTP download
+                result = await httpDownload(url, destPath, {
+                    cookies,
+                    timeout,
+                    onProgress: (received, total) => {
+                        if (progressBar) {
+                            progressBar.update(received, total);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+        }
+        catch (err) {
+            result = { success: false, size: 0, error: err.message };
+            if (progressBar) {
+                progressBar.fail(err.message);
+            }
+        }
+        tracker.onFileComplete(result.success);
+        const duration = Date.now() - startTime;
+        return {
+            ...item,
+            _download: {
+                status: result.success ? 'success' : 'failed',
+                path: result.success ? destPath : undefined,
+                size: result.size,
+                error: result.error,
+                duration,
+            },
+        };
+    });
+    // Cleanup temp cookie file
+    if (cookiesFile && fs.existsSync(cookiesFile)) {
+        try {
+            fs.unlinkSync(cookiesFile);
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+    // Show summary
+    tracker.finish();
+    return results;
+}

package/dist/pipeline/steps/intercept.js

@@ -1,7 +1,7 @@
 /**
  * Pipeline step: intercept — declarative XHR interception.
  */
-import { render } from '../template.js';
+import { render, normalizeEvaluateSource } from '../template.js';
 import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
 export async function stepIntercept(page, params, data, args) {
     const cfg = typeof params === 'object' ? params : {};
@@ -20,7 +20,6 @@ export async function stepIntercept(page, params, data, args) {
     }
     else if (trigger.startsWith('evaluate:')) {
         const js = trigger.slice('evaluate:'.length);
-        const { normalizeEvaluateSource } = await import('../template.js');
         await page.evaluate(normalizeEvaluateSource(render(js, { args, data })));
     }
     else if (trigger.startsWith('click:')) {

package/dist/pipeline/template.js

@@ -114,6 +114,34 @@ function applyFilter(filterExpr, value) {
             return Array.isArray(value) ? value[value.length - 1] : value;
         case 'json':
             return JSON.stringify(value ?? null);
+        case 'slugify':
+            // Convert to URL-safe slug
+            return typeof value === 'string'
+                ? value
+                    .toLowerCase()
+                    .replace(/[^\p{L}\p{N}]+/gu, '-')
+                    .replace(/^-|-$/g, '')
+                : value;
+        case 'sanitize':
+            // Remove invalid filename characters
+            return typeof value === 'string'
+                ? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
+                : value;
+        case 'ext': {
+            // Extract file extension from URL or path
+            if (typeof value !== 'string')
+                return value;
+            const lastDot = value.lastIndexOf('.');
+            const lastSlash = Math.max(value.lastIndexOf('/'), value.lastIndexOf('\\'));
+            return lastDot > lastSlash ? value.slice(lastDot) : '';
+        }
+        case 'basename': {
+            // Extract filename from URL or path
+            if (typeof value !== 'string')
+                return value;
+            const parts = value.split(/[/\\]/);
+            return parts[parts.length - 1] || value;
+        }
         default:
             return value;
     }

package/dist/setup.d.ts

@@ -1,3 +1,9 @@
+/**
+ * setup.ts — Interactive browser setup for opencli
+ *
+ * Simplified for daemon-based architecture. No more token management.
+ * Just verifies daemon + extension connectivity.
+ */
 export declare function runSetup(opts?: {
     cliVersion?: string;
     token?: string;