@agentrhq/webcmd 0.1.1 → 0.2.0

package/dist/src/download/index.js

@@ -0,0 +1,363 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+import { spawn } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { URL } from 'node:url';
+import { isBinaryInstalled } from '../external.js';
+import { getErrorMessage } from '../errors.js';
+import { fetchWithNodeNetwork } from '../node-network.js';
+import { log } from '../logger.js';
+/** Check if yt-dlp is available in PATH. */
+export function checkYtdlp() {
+    return isBinaryInstalled('yt-dlp');
+}
+/** Domains that host video content and can be downloaded via yt-dlp. */
+const VIDEO_PLATFORM_DOMAINS = [
+    'youtube.com', 'youtu.be', 'twitter.com',
+    'x.com', 'tiktok.com', 'vimeo.com', 'twitch.tv',
+];
+/**
+ * Whether a URL's host is a known video platform.
+ *
+ * Matches on the parsed hostname with a domain boundary (exact host or a
+ * subdomain) rather than a raw substring of the whole URL. A plain
+ * `url.includes('x.com')` wrongly classifies `netflix.com`, `max.com`, etc.
+ * (and any URL whose path merely contains a token like `youtu.be`) as a
+ * video platform, routing ordinary files to yt-dlp and mislabelling content
+ * types. Mirrors the host-boundary check used in execution.ts.
+ */
+function isVideoPlatformUrl(url) {
+    let host;
+    try {
+        host = new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return false;
+    }
+    return VIDEO_PLATFORM_DOMAINS.some((d) => host === d || host.endsWith(`.${d}`));
+}
+const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif']);
+const VIDEO_EXTENSIONS = new Set(['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts']);
+const DOC_EXTENSIONS = new Set(['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown']);
+/**
+ * Detect content type from URL and optional headers.
+ */
+export function detectContentType(url, contentType) {
+    if (contentType) {
+        if (contentType.startsWith('image/'))
+            return 'image';
+        if (contentType.startsWith('video/'))
+            return 'video';
+        if (contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml'))
+            return 'document';
+    }
+    const ext = path.extname(new URL(url).pathname).toLowerCase();
+    if (IMAGE_EXTENSIONS.has(ext))
+        return 'image';
+    if (VIDEO_EXTENSIONS.has(ext))
+        return 'video';
+    if (isVideoPlatformUrl(url))
+        return 'video';
+    if (DOC_EXTENSIONS.has(ext))
+        return 'document';
+    return 'binary';
+}
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export function requiresYtdlp(url) {
+    return isVideoPlatformUrl(url);
+}
+/**
+ * HTTP download with progress callback.
+ */
+export async function httpDownload(url, destPath, options = {}, redirectCount = 0) {
+    const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;
+    const requestHeaders = {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
+        ...headers,
+    };
+    if (cookies) {
+        requestHeaders['Cookie'] = cookies;
+    }
+    const tempPath = `${destPath}.tmp`;
+    const cleanupTempFile = async () => {
+        try {
+            await fs.promises.rm(tempPath, { force: true });
+        }
+        catch {
+            // Ignore cleanup errors so the original failure is preserved.
+        }
+    };
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+        const response = await fetchWithNodeNetwork(url, {
+            headers: requestHeaders,
+            signal: controller.signal,
+            redirect: 'manual',
+        });
+        clearTimeout(timer);
+        // Handle redirects before creating any file handles.
+        if (response.status >= 300 && response.status < 400) {
+            const location = response.headers.get('location');
+            if (location) {
+                if (redirectCount >= maxRedirects) {
+                    return { success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` };
+                }
+                const redirectUrl = resolveRedirectUrl(url, location);
+                const originalHost = new URL(url).hostname;
+                const redirectHost = new URL(redirectUrl).hostname;
+                const redirectOptions = originalHost === redirectHost
+                    ? options
+                    : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
+                return httpDownload(redirectUrl, destPath, redirectOptions, redirectCount + 1);
+            }
+        }
+        if (response.status !== 200) {
+            return { success: false, size: 0, error: `HTTP ${response.status}` };
+        }
+        if (!response.body) {
+            return { success: false, size: 0, error: 'Empty response body' };
+        }
+        const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
+        let received = 0;
+        const progressStream = new Transform({
+            transform(chunk, _encoding, callback) {
+                received += chunk.length;
+                if (onProgress)
+                    onProgress(received, totalSize);
+                callback(null, chunk);
+            },
+        });
+        try {
+            await fs.promises.mkdir(path.dirname(destPath), { recursive: true });
+            await pipeline(Readable.fromWeb(response.body), progressStream, fs.createWriteStream(tempPath));
+            await fs.promises.rename(tempPath, destPath);
+            return { success: true, size: received };
+        }
+        catch (err) {
+            await cleanupTempFile();
+            return { success: false, size: 0, error: getErrorMessage(err) };
+        }
+    }
+    catch (err) {
+        clearTimeout(timer);
+        await cleanupTempFile();
+        return { success: false, size: 0, error: err instanceof Error ? err.message : String(err) };
+    }
+}
+export function resolveRedirectUrl(currentUrl, location) {
+    return new URL(location, currentUrl).toString();
+}
+function stripCookieHeaders(headers) {
+    if (!headers)
+        return headers;
+    return Object.fromEntries(Object.entries(headers).filter(([key]) => key.toLowerCase() !== 'cookie'));
+}
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export function exportCookiesToNetscape(cookies, filePath) {
+    const lines = [
+        '# Netscape HTTP Cookie File',
+        '# https://curl.se/docs/http-cookies.html',
+        '# This is a generated file!  Do not edit.',
+        '',
+    ];
+    for (const cookie of cookies) {
+        const domain = cookie.domain.startsWith('.') ? cookie.domain : `.${cookie.domain}`;
+        const includeSubdomains = 'TRUE';
+        const cookiePath = cookie.path || '/';
+        const secure = cookie.secure ? 'TRUE' : 'FALSE';
+        const expiry = typeof cookie.expirationDate === 'number' && cookie.expirationDate > 0
+            ? Math.floor(cookie.expirationDate)
+            : Math.floor(Date.now() / 1000) + 86400 * 365; // fallback: 1 year from now
+        const safeName = cookie.name.replace(/[\t\n\r]/g, '');
+        const safeValue = cookie.value.replace(/[\t\n\r]/g, '');
+        lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${safeName}\t${safeValue}`);
+    }
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    // 0o600: file holds live session cookies, must be owner-only. fchmod before
+    // writing also tightens a pre-existing broad file before new secrets land.
+    let fd;
+    try {
+        fd = fs.openSync(filePath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC, 0o600);
+        fs.fchmodSync(fd, 0o600);
+        fs.writeFileSync(fd, lines.join('\n'), 'utf8');
+    }
+    finally {
+        if (fd !== undefined) {
+            fs.closeSync(fd);
+        }
+    }
+}
+export function formatCookieHeader(cookies) {
+    return cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ');
+}
+/**
+ * Download video using yt-dlp.
+ */
+export async function ytdlpDownload(url, destPath, options = {}) {
+    const { cookiesFile, format = 'best', extraArgs = [], onProgress } = options;
+    if (!checkYtdlp()) {
+        return { success: false, size: 0, error: 'yt-dlp not installed. Install with: pip install yt-dlp' };
+    }
+    return new Promise((resolve) => {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        // Build yt-dlp arguments
+        const args = [
+            url,
+            '-o', destPath,
+            '-f', format,
+            '--no-playlist',
+            '--progress',
+        ];
+        if (cookiesFile) {
+            if (fs.existsSync(cookiesFile)) {
+                args.push('--cookies', cookiesFile);
+            }
+            else {
+                log.warn(`[download] Cookies file not found: ${cookiesFile}, falling back to browser cookies`);
+                args.push('--cookies-from-browser', 'chrome');
+            }
+        }
+        else {
+            // Try to use browser cookies
+            args.push('--cookies-from-browser', 'chrome');
+        }
+        args.push(...extraArgs);
+        const proc = spawn('yt-dlp', args, {
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let lastPercent = 0;
+        let errorOutput = '';
+        proc.stderr.on('data', (data) => {
+            const line = data.toString();
+            errorOutput += line;
+            // Parse progress from yt-dlp output
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.stdout.on('data', (data) => {
+            const line = data.toString();
+            const match = line.match(/(\d+\.?\d*)%/);
+            if (match && onProgress) {
+                const percent = parseFloat(match[1]);
+                if (percent > lastPercent) {
+                    lastPercent = percent;
+                    onProgress(percent);
+                }
+            }
+        });
+        proc.on('close', (code) => {
+            if (code === 0 && fs.existsSync(destPath)) {
+                const stats = fs.statSync(destPath);
+                resolve({ success: true, size: stats.size });
+            }
+            else {
+                // Check for common yt-dlp output patterns
+                const patterns = fs.readdirSync(dir).filter(f => f.startsWith(path.basename(destPath, path.extname(destPath))));
+                if (patterns.length > 0) {
+                    const actualFile = path.join(dir, patterns[0]);
+                    const stats = fs.statSync(actualFile);
+                    resolve({ success: true, size: stats.size });
+                }
+                else {
+                    resolve({ success: false, size: 0, error: errorOutput.slice(0, 200) || `Exit code ${code}` });
+                }
+            }
+        });
+        proc.on('error', (err) => {
+            resolve({ success: false, size: 0, error: err.message });
+        });
+    });
+}
+/**
+ * Save document content to file.
+ */
+export async function saveDocument(content, destPath, format = 'markdown', metadata) {
+    try {
+        const dir = path.dirname(destPath);
+        fs.mkdirSync(dir, { recursive: true });
+        let output;
+        if (format === 'json') {
+            output = JSON.stringify({ ...metadata, content }, null, 2);
+        }
+        else if (format === 'markdown') {
+            // Add frontmatter if metadata exists
+            const frontmatter = metadata ? `---\n${Object.entries(metadata).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join('\n')}\n---\n\n` : '';
+            output = frontmatter + content;
+        }
+        else {
+            output = content;
+        }
+        fs.writeFileSync(destPath, output, 'utf-8');
+        return { success: true, size: Buffer.byteLength(output, 'utf-8') };
+    }
+    catch (err) {
+        return { success: false, size: 0, error: getErrorMessage(err) };
+    }
+}
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export function sanitizeFilename(name, maxLength = 200) {
+    return name
+        .replace(/[<>:"/\\|?*\x00-\x1f]/g, '_') // Remove invalid chars
+        .replace(/\s+/g, '_') // Replace spaces with underscores
+        .replace(/_+/g, '_') // Collapse multiple underscores
+        .replace(/^_|_$/g, '') // Trim underscores
+        .slice(0, maxLength);
+}
+/**
+ * Generate filename from URL if not provided.
+ */
+export function generateFilename(url, index, extension) {
+    try {
+        const parsedUrl = new URL(url);
+        const pathname = parsedUrl.pathname;
+        const basename = path.basename(pathname);
+        if (basename && basename !== '/' && basename.includes('.')) {
+            return sanitizeFilename(basename);
+        }
+        // Generate from hostname and index
+        const ext = extension || detectExtension(url);
+        const hostname = parsedUrl.hostname.replace(/^www\./, '');
+        return sanitizeFilename(`${hostname}_${index + 1}${ext}`);
+    }
+    catch {
+        const ext = extension || '.bin';
+        return `download_${index + 1}${ext}`;
+    }
+}
+/**
+ * Detect file extension from URL.
+ */
+function detectExtension(url) {
+    const type = detectContentType(url);
+    switch (type) {
+        case 'image': return '.jpg';
+        case 'video': return '.mp4';
+        case 'document': return '.md';
+        default: return '.bin';
+    }
+}
+/**
+ * Get temp directory for cookie files.
+ */
+export function getTempDir() {
+    return path.join(os.tmpdir(), 'webcmd-download');
+}

package/dist/src/download/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/download/index.test.js

@@ -0,0 +1,173 @@
+import * as fs from 'node:fs';
+import * as http from 'node:http';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { detectContentType, exportCookiesToNetscape, formatCookieHeader, httpDownload, requiresYtdlp, resolveRedirectUrl } from './index.js';
+const servers = [];
+const tempDirs = [];
+afterEach(async () => {
+    vi.unstubAllEnvs();
+    await Promise.all(servers.map((server) => new Promise((resolve, reject) => {
+        server.close((err) => (err ? reject(err) : resolve()));
+    })));
+    servers.length = 0;
+    for (const dir of tempDirs) {
+        try {
+            fs.rmSync(dir, { recursive: true, force: true });
+        }
+        catch { /* ignore */ }
+    }
+    tempDirs.length = 0;
+});
+async function startServer(handler, hostname = '127.0.0.1') {
+    const server = http.createServer(handler);
+    servers.push(server);
+    await new Promise((resolve) => server.listen(0, hostname, resolve));
+    const address = server.address();
+    if (!address || typeof address === 'string') {
+        throw new Error('Failed to start test server');
+    }
+    return `http://${hostname}:${address.port}`;
+}
+// Windows Defender can briefly lock newly-written .tmp files, causing EPERM.
+// Retry once to handle this flakiness.
+describe('download helpers', { retry: process.platform === 'win32' ? 2 : 0 }, () => {
+    it('resolves relative redirects against the original URL', () => {
+        expect(resolveRedirectUrl('https://example.com/a/file', '/cdn/file.bin')).toBe('https://example.com/cdn/file.bin');
+        expect(resolveRedirectUrl('https://example.com/a/file', '../next')).toBe('https://example.com/next');
+    });
+    it('formats browser cookies into a Cookie header', () => {
+        expect(formatCookieHeader([
+            { name: 'sid', value: 'abc', domain: 'example.com' },
+            { name: 'ct0', value: 'def', domain: 'example.com' },
+        ])).toBe('sid=abc; ct0=def');
+    });
+    it('fails after exceeding the redirect limit', async () => {
+        const baseUrl = await startServer((_req, res) => {
+            res.statusCode = 302;
+            res.setHeader('Location', '/loop');
+            res.end();
+        });
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const destPath = path.join(tempDir, 'file.txt');
+        const result = await httpDownload(`${baseUrl}/loop`, destPath, { maxRedirects: 2 });
+        expect(result).toEqual({
+            success: false,
+            size: 0,
+            error: 'Too many redirects (> 2)',
+        });
+        expect(fs.existsSync(destPath)).toBe(false);
+    });
+    it('does not forward cookies across cross-domain redirects', async () => {
+        let forwardedCookie;
+        const targetUrl = await startServer((req, res) => {
+            forwardedCookie = req.headers.cookie;
+            res.statusCode = 200;
+            res.end('ok');
+        }, 'localhost');
+        const redirectUrl = await startServer((_req, res) => {
+            res.statusCode = 302;
+            res.setHeader('Location', targetUrl);
+            res.end();
+        });
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const destPath = path.join(tempDir, 'redirect.txt');
+        const result = await httpDownload(`${redirectUrl}/start`, destPath, { cookies: 'sid=abc' });
+        expect(result).toEqual({ success: true, size: 2 });
+        expect(forwardedCookie).toBeUndefined();
+        expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
+    });
+    it('does not forward cookie headers across cross-domain redirects', async () => {
+        let forwardedCookie;
+        const targetUrl = await startServer((req, res) => {
+            forwardedCookie = req.headers.cookie;
+            res.statusCode = 200;
+            res.end('ok');
+        }, 'localhost');
+        const redirectUrl = await startServer((_req, res) => {
+            res.statusCode = 302;
+            res.setHeader('Location', targetUrl);
+            res.end();
+        });
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const destPath = path.join(tempDir, 'redirect-header.txt');
+        const result = await httpDownload(`${redirectUrl}/start`, destPath, {
+            headers: { Cookie: 'sid=header-cookie' },
+        });
+        expect(result).toEqual({ success: true, size: 2 });
+        expect(forwardedCookie).toBeUndefined();
+        expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
+    });
+    it('bypasses proxy settings for loopback downloads', async () => {
+        vi.stubEnv('HTTP_PROXY', 'http://127.0.0.1:9');
+        const baseUrl = await startServer((_req, res) => {
+            res.statusCode = 200;
+            res.end('ok');
+        });
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const destPath = path.join(tempDir, 'loopback.txt');
+        const result = await httpDownload(`${baseUrl}/ok`, destPath);
+        expect(result).toEqual({ success: true, size: 2 });
+        expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
+    });
+    it.skipIf(process.platform === 'win32')('writes the Netscape cookie file with 0o600 owner-only permissions', async () => {
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const cookiesPath = path.join(tempDir, 'cookies.txt');
+        exportCookiesToNetscape([
+            { name: 'sid', value: 'secret', domain: 'example.com' },
+        ], cookiesPath);
+        const mode = fs.statSync(cookiesPath).mode & 0o777;
+        expect(mode).toBe(0o600);
+    });
+    it.skipIf(process.platform === 'win32')('tightens an existing Netscape cookie file before rewriting it', async () => {
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-dl-'));
+        tempDirs.push(tempDir);
+        const cookiesPath = path.join(tempDir, 'cookies.txt');
+        fs.writeFileSync(cookiesPath, 'stale-cookie', { mode: 0o644 });
+        exportCookiesToNetscape([
+            { name: 'sid', value: 'secret', domain: 'example.com' },
+        ], cookiesPath);
+        const mode = fs.statSync(cookiesPath).mode & 0o777;
+        expect(mode).toBe(0o600);
+        expect(fs.readFileSync(cookiesPath, 'utf8')).toContain('sid\tsecret');
+    });
+});
+describe('video-platform host detection', () => {
+    it('matches real video-platform hosts (host or subdomain)', () => {
+        for (const url of [
+            'https://www.youtube.com/watch?v=abc',
+            'https://youtu.be/abc',
+            'https://vimeo.com/12345',
+            'https://twitter.com/u/status/1',
+            'https://x.com/u/status/1',
+            'https://www.tiktok.com/@u/video/1',
+            'https://www.twitch.tv/u',
+        ]) {
+            expect(requiresYtdlp(url)).toBe(true);
+            expect(detectContentType(url)).toBe('video');
+        }
+    });
+    it('does not match hosts that merely end with a token or carry it in the path', () => {
+        // netflix.com / max.com / phoenix.com all *contain* the substring "x.com";
+        // the path-only token "youtu.be-guide" must not match either.
+        for (const url of [
+            'https://netflix.com/watch/123',
+            'https://www.phoenix.com/a.zip',
+            'https://max.com/movie',
+            'https://cdn.site.com/youtu.be-guide.bin',
+            'https://notx.com/a',
+        ]) {
+            expect(requiresYtdlp(url)).toBe(false);
+            expect(detectContentType(url)).not.toBe('video');
+        }
+    });
+    it('returns false for an unparseable URL instead of throwing', () => {
+        expect(requiresYtdlp('not a url')).toBe(false);
+    });
+});

package/dist/src/download/media-download.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Media download helper — shared logic for batch downloading images/videos.
+ *
+ * Used by: twitter/download, youtube/download,
+ * and future media adapters.
+ *
+ * Flow: MediaItem[] → DownloadProgressTracker → httpDownload/ytdlpDownload → results
+ */
+import type { BrowserCookie } from '../types.js';
+export interface MediaItem {
+    type: 'image' | 'video' | 'video-tweet' | 'video-ytdlp';
+    url: string;
+    /** Optional custom filename (without directory) */
+    filename?: string;
+}
+export interface MediaDownloadOptions {
+    output: string;
+    /** Subdirectory inside output */
+    subdir?: string;
+    /** Cookie string for HTTP downloads */
+    cookies?: string;
+    /** Raw browser cookies — auto-exported to Netscape for yt-dlp, auto-cleaned up */
+    browserCookies?: BrowserCookie[];
+    /** Timeout in ms (default: 30000 for images, 60000 for videos) */
+    timeout?: number;
+    /** File name prefix (default: 'download') */
+    filenamePrefix?: string;
+    /** Extra yt-dlp args */
+    ytdlpExtraArgs?: string[];
+    /** Whether to show progress (default: true) */
+    verbose?: boolean;
+}
+export interface MediaDownloadResult {
+    index: number;
+    type: string;
+    status: string;
+    size: string;
+}
+/**
+ * Batch download media files with progress tracking.
+ *
+ * Handles:
+ * - DownloadProgressTracker for terminal UX
+ * - Automatic httpDownload vs ytdlpDownload routing via MediaItem.type
+ * - Cookie export to Netscape format for yt-dlp (auto-cleanup)
+ * - Directory creation
+ * - Error handling with per-file results
+ */
+export declare function downloadMedia(items: MediaItem[], options: MediaDownloadOptions): Promise<MediaDownloadResult[]>;