@jackwener/opencli 0.9.5 → 0.9.8

package/dist/pipeline/steps/download.js

@@ -0,0 +1,251 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { render } from '../template.js';
+import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, } from '../../download/index.js';
+import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+/**
+ * Simple async concurrency limiter for downloads.
+ */
+async function mapConcurrent(items, limit, fn) {
+    const results = new Array(items.length);
+    let index = 0;
+    async function worker() {
+        while (index < items.length) {
+            const i = index++;
+            results[i] = await fn(items[i], i);
+        }
+    }
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+/**
+ * Extract cookies from browser page.
+ */
+async function extractBrowserCookies(page, domain) {
+    try {
+        // Use browser evaluate to get document.cookie
+        const cookieString = await page.evaluate(`(() => document.cookie)()`);
+        return typeof cookieString === 'string' ? cookieString : '';
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Extract cookies as array for yt-dlp Netscape format.
+ */
+async function extractCookiesArray(page, domain) {
+    try {
+        const cookieString = await extractBrowserCookies(page);
+        if (!cookieString)
+            return [];
+        return cookieString.split(';').map((c) => {
+            const [name, ...rest] = c.trim().split('=');
+            return {
+                name: name || '',
+                value: rest.join('=') || '',
+                domain,
+                path: '/',
+                secure: true,
+                httpOnly: false,
+            };
+        }).filter((c) => c.name);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Download step handler for YAML pipelines.
+ *
+ * Usage in YAML:
+ * ```yaml
+ * pipeline:
+ *   - download:
+ *       url: ${{ item.imageUrl }}
+ *       dir: ./downloads
+ *       filename: ${{ item.title }}.jpg
+ *       concurrency: 5
+ *       skip_existing: true
+ *       use_ytdlp: false
+ *       type: auto
+ * ```
+ */
+export async function stepDownload(page, params, data, args) {
+    // Parse parameters with defaults
+    const urlTemplate = typeof params === 'string' ? params : (params?.url ?? '');
+    const dirTemplate = params?.dir ?? './downloads';
+    const filenameTemplate = params?.filename ?? '';
+    const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 3;
+    const skipExisting = params?.skip_existing !== false;
+    const timeout = typeof params?.timeout === 'number' ? params.timeout * 1000 : 30000;
+    const useYtdlp = params?.use_ytdlp ?? false;
+    const ytdlpArgs = Array.isArray(params?.ytdlp_args) ? params.ytdlp_args : [];
+    const contentType = params?.type ?? 'auto';
+    const showProgress = params?.progress !== false;
+    const contentTemplate = params?.content;
+    const metadataTemplate = params?.metadata;
+    // Resolve output directory
+    const dir = String(render(dirTemplate, { args, data }));
+    fs.mkdirSync(dir, { recursive: true });
+    // Normalize data to array
+    const items = Array.isArray(data) ? data : data ? [data] : [];
+    if (items.length === 0) {
+        return [];
+    }
+    // Create progress tracker
+    const tracker = new DownloadProgressTracker(items.length, showProgress);
+    // Extract cookies if browser is available
+    let cookies = '';
+    let cookiesFile;
+    if (page) {
+        cookies = await extractBrowserCookies(page);
+        // For yt-dlp, we need to export cookies to Netscape format
+        if (useYtdlp || items.some((item, index) => {
+            const url = String(render(urlTemplate, { args, data, item, index }));
+            return requiresYtdlp(url);
+        })) {
+            try {
+                // Try to get domain from first URL
+                const firstUrl = String(render(urlTemplate, { args, data, item: items[0], index: 0 }));
+                const domain = new URL(firstUrl).hostname;
+                const cookiesArray = await extractCookiesArray(page, domain);
+                if (cookiesArray.length > 0) {
+                    const tempDir = getTempDir();
+                    fs.mkdirSync(tempDir, { recursive: true });
+                    cookiesFile = path.join(tempDir, `cookies_${Date.now()}.txt`);
+                    exportCookiesToNetscape(cookiesArray, cookiesFile);
+                }
+            }
+            catch {
+                // Ignore cookie extraction errors
+            }
+        }
+    }
+    // Process downloads with concurrency
+    const results = await mapConcurrent(items, concurrency, async (item, index) => {
+        const startTime = Date.now();
+        // Render URL
+        const url = String(render(urlTemplate, { args, data, item, index }));
+        if (!url) {
+            tracker.onFileComplete(false);
+            return {
+                ...item,
+                _download: { status: 'failed', error: 'Empty URL' },
+            };
+        }
+        // Render filename
+        let filename;
+        if (filenameTemplate) {
+            filename = String(render(filenameTemplate, { args, data, item, index }));
+        }
+        else {
+            filename = generateFilename(url, index);
+        }
+        filename = sanitizeFilename(filename);
+        const destPath = path.join(dir, filename);
+        // Check if file exists and skip_existing is true
+        if (skipExisting && fs.existsSync(destPath)) {
+            tracker.onFileComplete(true, true);
+            return {
+                ...item,
+                _download: {
+                    status: 'skipped',
+                    path: destPath,
+                    size: fs.statSync(destPath).size,
+                },
+            };
+        }
+        // Create progress bar for this file
+        const progressBar = tracker.onFileStart(filename, index);
+        // Determine download method
+        const detectedType = contentType === 'auto' ? detectContentType(url) : contentType;
+        const shouldUseYtdlp = useYtdlp || (detectedType === 'video' && requiresYtdlp(url));
+        let result;
+        try {
+            if (detectedType === 'document' && contentTemplate) {
+                // Save extracted content as document
+                const content = String(render(contentTemplate, { args, data, item, index }));
+                const metadata = metadataTemplate
+                    ? Object.fromEntries(Object.entries(metadataTemplate).map(([k, v]) => [k, render(v, { args, data, item, index })]))
+                    : undefined;
+                const ext = path.extname(filename).toLowerCase();
+                const format = ext === '.json' ? 'json' : ext === '.html' ? 'html' : 'markdown';
+                result = await saveDocument(content, destPath, format, metadata);
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else if (shouldUseYtdlp) {
+                // Use yt-dlp for video downloads
+                result = await ytdlpDownload(url, destPath, {
+                    cookiesFile,
+                    extraArgs: ytdlpArgs,
+                    onProgress: (percent) => {
+                        if (progressBar) {
+                            progressBar.update(percent, 100);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else {
+                // Direct HTTP download
+                result = await httpDownload(url, destPath, {
+                    cookies,
+                    timeout,
+                    onProgress: (received, total) => {
+                        if (progressBar) {
+                            progressBar.update(received, total);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+        }
+        catch (err) {
+            result = { success: false, size: 0, error: err.message };
+            if (progressBar) {
+                progressBar.fail(err.message);
+            }
+        }
+        tracker.onFileComplete(result.success);
+        const duration = Date.now() - startTime;
+        return {
+            ...item,
+            _download: {
+                status: result.success ? 'success' : 'failed',
+                path: result.success ? destPath : undefined,
+                size: result.size,
+                error: result.error,
+                duration,
+            },
+        };
+    });
+    // Cleanup temp cookie file
+    if (cookiesFile && fs.existsSync(cookiesFile)) {
+        try {
+            fs.unlinkSync(cookiesFile);
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+    // Show summary
+    tracker.finish();
+    return results;
+}

package/dist/pipeline/template.js

@@ -114,6 +114,34 @@ function applyFilter(filterExpr, value) {
             return Array.isArray(value) ? value[value.length - 1] : value;
         case 'json':
             return JSON.stringify(value ?? null);
+        case 'slugify':
+            // Convert to URL-safe slug
+            return typeof value === 'string'
+                ? value
+                    .toLowerCase()
+                    .replace(/[^\p{L}\p{N}]+/gu, '-')
+                    .replace(/^-|-$/g, '')
+                : value;
+        case 'sanitize':
+            // Remove invalid filename characters
+            return typeof value === 'string'
+                ? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
+                : value;
+        case 'ext': {
+            // Extract file extension from URL or path
+            if (typeof value !== 'string')
+                return value;
+            const lastDot = value.lastIndexOf('.');
+            const lastSlash = Math.max(value.lastIndexOf('/'), value.lastIndexOf('\\'));
+            return lastDot > lastSlash ? value.slice(lastDot) : '';
+        }
+        case 'basename': {
+            // Extract filename from URL or path
+            if (typeof value !== 'string')
+                return value;
+            const parts = value.split(/[/\\]/);
+            return parts[parts.length - 1] || value;
+        }
         default:
             return value;
     }

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@jackwener/opencli",
-  "version": "0.9.5",
+  "version": "0.9.8",
   "publishConfig": {
     "access": "public"
   },
   "description": "Make any website your CLI. AI-powered.",
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "type": "module",
   "main": "dist/main.js",
@@ -25,6 +25,7 @@
     "lint": "tsc --noEmit",
     "prepublishOnly": "npm run build",
     "test": "vitest run",
+    "test:site": "node scripts/test-site.mjs",
     "test:watch": "vitest"
   },
   "keywords": [
@@ -43,7 +44,7 @@
   "dependencies": {
     "chalk": "^5.3.0",
     "cli-table3": "^0.6.5",
-    "commander": "^13.1.0",
+    "commander": "^14.0.3",
     "js-yaml": "^4.1.0"
   },
   "devDependencies": {

package/scripts/test-site.mjs

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const site = process.argv[2]?.trim();
+
+if (!site) {
+  console.error('Usage: npm run test:site -- <site>');
+  process.exit(1);
+}
+
+const repoRoot = path.resolve(new URL('..', import.meta.url).pathname);
+const srcDir = path.join(repoRoot, 'src');
+
+function runStep(label, command, args) {
+  console.log(`\n==> ${label}`);
+  const result = spawnSync(command, args, {
+    cwd: repoRoot,
+    stdio: 'inherit',
+    env: process.env,
+  });
+
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1);
+  }
+}
+
+function walk(dir) {
+  const files = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...walk(fullPath));
+    } else {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function toPosix(filePath) {
+  return filePath.split(path.sep).join('/');
+}
+
+function findSiteTests() {
+  return walk(srcDir)
+    .filter(filePath => filePath.endsWith('.test.ts'))
+    .filter(filePath => {
+      const normalized = toPosix(path.relative(repoRoot, filePath));
+      return normalized.includes(`/clis/${site}/`) || normalized.includes(`/${site}.test.ts`);
+    })
+    .sort();
+}
+
+runStep('Typecheck', 'npm', ['run', 'typecheck']);
+runStep('Targeted verify', 'npx', ['tsx', 'src/main.ts', 'verify', site]);
+
+const testFiles = findSiteTests();
+if (testFiles.length === 0) {
+  console.log(`\nNo site-specific vitest files found for "${site}". Skipping full vitest run.`);
+  process.exit(0);
+}
+
+runStep(
+  `Site tests (${site})`,
+  'npx',
+  ['vitest', 'run', ...testFiles.map(filePath => path.relative(repoRoot, filePath))],
+);

package/src/browser/discover.ts

@@ -12,6 +12,19 @@ let _cachedMcpServerPath: string | null | undefined;
 let _existsSync = fs.existsSync;
 let _execSync = execSync;
 
+function isSupportedMcpEntrypoint(candidate: string): boolean {
+  const normalized = candidate.replace(/\\/g, '/').toLowerCase();
+  return normalized.endsWith('/@playwright/mcp/cli.js') ||
+    normalized.endsWith('/mcp-server-playwright') ||
+    normalized.endsWith('/mcp-server-playwright.js');
+}
+
+function resolveSupportedMcpPath(candidate: string | null | undefined): string | null {
+  const trimmed = candidate?.trim();
+  if (!trimmed || !_existsSync(trimmed)) return null;
+  return isSupportedMcpEntrypoint(trimmed) ? trimmed : null;
+}
+
 export function resetMcpServerPathCache(): void {
   _cachedMcpServerPath = undefined;
 }
@@ -80,8 +93,9 @@ export function findMcpServerPath(): string | null {
   // Try npx resolution (legacy package name)
   try {
     const result = _execSync('npx -y --package=@playwright/mcp which mcp-server-playwright 2>/dev/null', { encoding: 'utf-8', timeout: 10000 }).trim();
-    if (result && _existsSync(result)) {
-      _cachedMcpServerPath = result;
+    const resolved = resolveSupportedMcpPath(result);
+    if (resolved) {
+      _cachedMcpServerPath = resolved;
       return _cachedMcpServerPath;
     }
   } catch {}
@@ -89,8 +103,9 @@ export function findMcpServerPath(): string | null {
   // Try which
   try {
     const result = _execSync('which mcp-server-playwright 2>/dev/null', { encoding: 'utf-8', timeout: 5000 }).trim();
-    if (result && _existsSync(result)) {
-      _cachedMcpServerPath = result;
+    const resolved = resolveSupportedMcpPath(result);
+    if (resolved) {
+      _cachedMcpServerPath = resolved;
       return _cachedMcpServerPath;
     }
   } catch {}
@@ -99,9 +114,10 @@ export function findMcpServerPath(): string | null {
   for (const base of candidates) {
     if (!_existsSync(base)) continue;
     try {
-      const found = _execSync(`find "${base}" -name "cli.js" -path "*playwright*mcp*" 2>/dev/null | head -1`, { encoding: 'utf-8', timeout: 5000 }).trim();
-      if (found) {
-        _cachedMcpServerPath = found;
+      const found = _execSync(`find "${base}" -type f -path "*/@playwright/mcp/cli.js" 2>/dev/null | head -1`, { encoding: 'utf-8', timeout: 5000 }).trim();
+      const resolved = resolveSupportedMcpPath(found);
+      if (resolved) {
+        _cachedMcpServerPath = resolved;
         return _cachedMcpServerPath;
       }
     } catch {}

package/src/browser.test.ts

@@ -1,4 +1,6 @@
 import { afterEach, describe, it, expect, vi } from 'vitest';
+import * as os from 'node:os';
+import * as path from 'node:path';
 import { PlaywrightMCP, __test__ } from './browser/index.js';
 
 afterEach(() => {
@@ -248,6 +250,27 @@ describe('browser helpers', () => {
       });
     }
   });
+
+  it('ignores non-server playwright cli paths discovered from fallback scans', () => {
+    const wrongCli = '/root/.npm/_npx/e41f203b7505f1fb/node_modules/playwright/lib/mcp/terminal/cli.js';
+    const npxCacheBase = path.join(os.homedir(), '.npm', '_npx');
+
+    const existsSync = vi.fn((candidate: any) => {
+      const value = String(candidate);
+      return value === npxCacheBase || value === wrongCli;
+    });
+
+    const execSync = vi.fn((command: string) => {
+      if (String(command).includes('npm root -g')) return '/missing/global/node_modules\n' as any;
+      if (String(command).includes('--package=@playwright/mcp which mcp-server-playwright')) return `${wrongCli}\n` as any;
+      if (String(command).includes('which mcp-server-playwright')) return '' as any;
+      if (String(command).includes(`find "${npxCacheBase}"`)) return `${wrongCli}\n` as any;
+      throw new Error(`unexpected command: ${String(command)}`);
+    });
+    __test__.setMcpDiscoveryTestHooks({ existsSync, execSync: execSync as any });
+
+    expect(__test__.findMcpServerPath()).toBeNull();
+  });
 });
 
 describe('PlaywrightMCP state', () => {

package/src/build-manifest.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest';
+import { parseTsArgsBlock } from './build-manifest.js';
+
+describe('parseTsArgsBlock', () => {
+  it('keeps args with nested choices arrays', () => {
+    const args = parseTsArgsBlock(`
+      {
+        name: 'period',
+        type: 'string',
+        default: 'seven',
+        help: 'Stats period: seven or thirty',
+        choices: ['seven', 'thirty'],
+      },
+    `);
+
+    expect(args).toEqual([
+      {
+        name: 'period',
+        type: 'string',
+        default: 'seven',
+        required: false,
+        positional: undefined,
+        help: 'Stats period: seven or thirty',
+        choices: ['seven', 'thirty'],
+      },
+    ]);
+  });
+});