@agentrhq/webcmd 0.1.1 → 0.2.0

package/dist/src/download/article-download.js

@@ -0,0 +1,305 @@
+/**
+ * Article download helper — shared logic for downloading articles as Markdown.
+ *
+ * Used by article adapters that export readable web pages as Markdown.
+ *
+ * Flow: ArticleData → TurndownService → image download → frontmatter → .md file
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { httpDownload, sanitizeFilename } from './index.js';
+import { formatBytes } from './progress.js';
+const IMAGE_CONCURRENCY = 5;
+const DEFAULT_LABELS = {
+    author: 'Author',
+    publishTime: 'Published',
+    sourceUrl: 'Source URL',
+};
+// ============================================================
+// Markdown Conversion
+// ============================================================
+// Nodes that never carry article content. Turndown keeps them by default — if an
+// adapter's contentHtml extraction misses one, CSS / scripts / widget markup
+// ends up inline in the .md. Strip them unconditionally at the converter level.
+// `svg` is not in HTMLElementTagNameMap, so we type-narrow manually.
+// `header/footer/nav/aside` cover page chrome that adapters occasionally
+// forget to trim — the article's own title/author/publishTime are supplied
+// as separate fields on ArticleData, so duplicated nodes are redundant.
+// `iframe` is NOT in this set — it's handled by a dedicated rule below that
+// degrades to a link so embedded content (YouTube, Twitter, CodePen …) keeps
+// a reachable URL in the exported markdown.
+const STRIPPED_TAGS = [
+    'script', 'style', 'noscript',
+    'canvas',
+    'form', 'button', 'dialog',
+    'header', 'footer', 'nav', 'aside',
+];
+function createTurndown(configure, cleanSelectors) {
+    const td = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        bulletListMarker: '-',
+    });
+    td.use(gfm);
+    td.remove(STRIPPED_TAGS);
+    // turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which
+    // is not the canonical GFM form. Override it so exported markdown is
+    // portable across common renderers.
+    td.addRule('canonicalStrikethrough', {
+        filter: (node) => ['DEL', 'S', 'STRIKE'].includes(node.nodeName),
+        replacement: (content) => `~~${content}~~`,
+    });
+    // SVG isn't in the static HTML tag map; match by name with a custom filter.
+    td.addRule('stripSvg', {
+        filter: (node) => node.nodeName === 'svg' || node.nodeName === 'SVG',
+        replacement: () => '',
+    });
+    td.addRule('linebreak', {
+        filter: 'br',
+        replacement: () => '\n',
+    });
+    // Inline base64 images would land as huge `![](data:image/...;base64,...)`
+    // strings that the image downloader can't localize. Drop them.
+    td.addRule('ignoreBase64Images', {
+        filter: (node) => {
+            if (node.nodeName !== 'IMG')
+                return false;
+            const src = node.getAttribute?.('src') ?? '';
+            return src.startsWith('data:');
+        },
+        replacement: () => '',
+    });
+    // Markdown has no native video/audio primitive. Emit inline HTML so
+    // renderers that support it (GitHub, VS Code preview …) still play the
+    // media; viewers that don't simply show the tag as text, which is still
+    // more information than dropping the node outright.
+    td.addRule('videoElement', {
+        filter: (node) => node.nodeName === 'VIDEO',
+        replacement: (_content, node) => {
+            const el = node;
+            const src = el.getAttribute('src')
+                || el.querySelector('source')?.getAttribute('src')
+                || '';
+            if (!src)
+                return '';
+            const poster = el.getAttribute('poster') || '';
+            return `\n<video src="${src}" controls${poster ? ` poster="${poster}"` : ''}></video>\n`;
+        },
+    });
+    td.addRule('audioElement', {
+        filter: (node) => node.nodeName === 'AUDIO',
+        replacement: (_content, node) => {
+            const el = node;
+            const src = el.getAttribute('src')
+                || el.querySelector('source')?.getAttribute('src')
+                || '';
+            return src ? `\n<audio src="${src}" controls></audio>\n` : '';
+        },
+    });
+    // Iframes (YouTube, Twitter, CodePen …) degrade to a markdown link so the
+    // embedded resource is still reachable from the exported file.
+    td.addRule('iframeToLink', {
+        filter: (node) => node.nodeName === 'IFRAME',
+        replacement: (_content, node) => {
+            const el = node;
+            const src = el.getAttribute('src') || '';
+            if (!src)
+                return '';
+            const title = el.getAttribute('title') || 'Embedded content';
+            return `\n[${title}](${src})\n`;
+        },
+    });
+    // Per-adapter dirty-node removal. Adapters know their site's specific noise
+    // (social widgets, reward panels, wiki infoboxes, and similar chrome);
+    // we keep the default set empty so the generic converter stays untouched.
+    const selectorRules = (cleanSelectors ?? [])
+        .map(sel => sel.trim())
+        .filter(Boolean);
+    if (selectorRules.length > 0) {
+        td.addRule('cleanSelectors', {
+            filter: (node) => {
+                const match = node.matches;
+                if (typeof match !== 'function')
+                    return false;
+                return selectorRules.some((sel) => {
+                    try {
+                        return match.call(node, sel);
+                    }
+                    catch {
+                        return false;
+                    }
+                });
+            },
+            replacement: () => '',
+        });
+    }
+    if (configure)
+        configure(td);
+    return td;
+}
+function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
+    const td = createTurndown(configure, cleanSelectors);
+    let md = td.turndown(contentHtml);
+    // Restore code block placeholders
+    codeBlocks.forEach((block, i) => {
+        const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
+        const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
+        md = md.replace(placeholder, fenced);
+    });
+    // Clean up
+    md = md.replace(/\u00a0/g, ' ');
+    // Turndown leaves behind lone dashes / middle dots when list bullets or
+    // decorative separators lose their surrounding inline context.
+    md = md.replace(/^[ \t]*[-\u00b7][ \t]*$/gm, '');
+    md = md.replace(/^[ \t]+$/gm, '');
+    md = md.replace(/[ \t]+$/gm, '');
+    md = md.replace(/\n{3,}/g, '\n\n');
+    return md;
+}
+function replaceImageUrls(md, urlMap) {
+    return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
+        const local = urlMap[imgUrl];
+        return local ? `![${alt}](${local})` : match;
+    });
+}
+// ============================================================
+// Image Downloading
+// ============================================================
+function defaultDetectImageExt(url) {
+    const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
+    return extMatch ? extMatch[1] : 'jpg';
+}
+async function downloadImages(imgUrls, imgDir, headers, detectExt) {
+    const urlMap = {};
+    if (imgUrls.length === 0)
+        return urlMap;
+    const detect = detectExt || defaultDetectImageExt;
+    // Deduplicate image URLs
+    const seen = new Set();
+    const uniqueUrls = imgUrls.filter(url => {
+        if (seen.has(url))
+            return false;
+        seen.add(url);
+        return true;
+    });
+    for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
+        const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
+        const results = await Promise.all(batch.map(async (rawUrl, j) => {
+            const index = i + j + 1;
+            let imgUrl = rawUrl;
+            if (imgUrl.startsWith('//'))
+                imgUrl = `https:${imgUrl}`;
+            const ext = detect(imgUrl);
+            const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
+            const filepath = path.join(imgDir, filename);
+            try {
+                const result = await httpDownload(imgUrl, filepath, {
+                    headers,
+                    timeout: 15000,
+                });
+                if (result.success) {
+                    return { remoteUrl: rawUrl, localPath: `images/${filename}` };
+                }
+            }
+            catch {
+                // Skip failed downloads
+            }
+            return null;
+        }));
+        for (const r of results) {
+            if (r)
+                urlMap[r.remoteUrl] = r.localPath;
+        }
+    }
+    return urlMap;
+}
+// ============================================================
+// Main API
+// ============================================================
+/**
+ * Download an article to Markdown with optional image localization.
+ *
+ * Handles the full pipeline:
+ * 1. HTML → Markdown (via TurndownService)
+ * 2. Code block placeholder restoration
+ * 3. Batch image downloading with concurrency + deduplication
+ * 4. Image URL replacement in Markdown
+ * 5. Frontmatter generation (customizable labels)
+ * 6. File write
+ */
+export async function downloadArticle(data, options) {
+    const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, } = options;
+    const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
+    if (!data.title) {
+        return [{
+                title: 'Error',
+                author: '-',
+                publish_time: '-',
+                status: 'failed — no title',
+                size: '-',
+                saved: '-',
+            }];
+    }
+    if (!data.contentHtml) {
+        return [{
+                title: data.title,
+                author: data.author || '-',
+                publish_time: data.publishTime || '-',
+                status: 'failed — no content',
+                size: '-',
+                saved: '-',
+            }];
+    }
+    // Convert HTML to Markdown
+    let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors);
+    const safeTitle = sanitizeFilename(data.title, maxTitleLength);
+    // Download images only when writing to disk. In stdout mode remote URLs
+    // stay intact so the piped output is self-contained.
+    if (!stdout && shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
+        const articleDir = path.join(output, safeTitle);
+        fs.mkdirSync(articleDir, { recursive: true });
+        const imagesDir = path.join(articleDir, 'images');
+        fs.mkdirSync(imagesDir, { recursive: true });
+        const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
+        markdown = replaceImageUrls(markdown, urlMap);
+    }
+    // Build frontmatter with customizable labels.
+    // Shape: `# Title\n[> meta\n...]\n---\n\n<markdown>` — exactly one blank
+    // line separates every section, so we never produce ≥3 consecutive newlines.
+    const headerLines = [`# ${data.title}`];
+    if (data.author)
+        headerLines.push(`> ${labels.author}: ${data.author}`);
+    if (data.publishTime)
+        headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
+    if (data.sourceUrl)
+        headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
+    const frontmatter = headerLines.join('\n') + '\n\n---\n\n';
+    const fullContent = frontmatter + markdown;
+    const size = Buffer.byteLength(fullContent, 'utf-8');
+    if (stdout) {
+        process.stdout.write(fullContent.endsWith('\n') ? fullContent : fullContent + '\n');
+        return [{
+                title: data.title,
+                author: data.author || '-',
+                publish_time: data.publishTime || '-',
+                status: 'success',
+                size: formatBytes(size),
+                saved: '-',
+            }];
+    }
+    const articleDir = path.join(output, safeTitle);
+    fs.mkdirSync(articleDir, { recursive: true });
+    const filename = `${safeTitle}.md`;
+    const filePath = path.join(articleDir, filename);
+    fs.writeFileSync(filePath, fullContent, 'utf-8');
+    return [{
+            title: data.title,
+            author: data.author || '-',
+            publish_time: data.publishTime || '-',
+            status: 'success',
+            size: formatBytes(size),
+            saved: filePath,
+        }];
+}

package/dist/src/download/article-download.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/download/article-download.test.js

@@ -0,0 +1,235 @@
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { downloadArticle } from './article-download.js';
+const tempDirs = [];
+afterEach(() => {
+    for (const dir of tempDirs) {
+        try {
+            fs.rmSync(dir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup errors in tests.
+        }
+    }
+    tempDirs.length = 0;
+});
+async function runAndRead(contentHtml, opts = {}) {
+    const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-article-'));
+    tempDirs.push(tempDir);
+    const result = await downloadArticle({
+        title: 'Test Article',
+        contentHtml,
+    }, {
+        output: tempDir,
+        downloadImages: false,
+        ...(opts.cleanSelectors && { cleanSelectors: opts.cleanSelectors }),
+    });
+    expect(result[0].status).toBe('success');
+    return fs.readFileSync(result[0].saved, 'utf8');
+}
+describe('downloadArticle', () => {
+    it('returns the saved markdown file path on success', async () => {
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-article-'));
+        tempDirs.push(tempDir);
+        const result = await downloadArticle({
+            title: 'Test Article',
+            author: 'Author',
+            publishTime: '2026-04-20 12:00:00',
+            sourceUrl: 'https://example.com/article',
+            contentHtml: '<p>Hello world</p>',
+        }, {
+            output: tempDir,
+            downloadImages: false,
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0].status).toBe('success');
+        expect(result[0].saved).toMatch(new RegExp(`^${tempDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`));
+        expect(path.extname(result[0].saved)).toBe('.md');
+        expect(fs.existsSync(result[0].saved)).toBe(true);
+        expect(fs.readFileSync(result[0].saved, 'utf8')).toContain('Hello world');
+    });
+    describe('markdown pipeline', () => {
+        it('converts GFM tables', async () => {
+            const md = await runAndRead('<table><thead><tr><th>a</th><th>b</th></tr></thead>' +
+                '<tbody><tr><td>1</td><td>2</td></tr></tbody></table>');
+            expect(md).toMatch(/\|\s*a\s*\|\s*b\s*\|/);
+            expect(md).toMatch(/\|\s*---\s*\|\s*---\s*\|/);
+            expect(md).toMatch(/\|\s*1\s*\|\s*2\s*\|/);
+        });
+        it('converts strikethrough and task lists', async () => {
+            const md = await runAndRead('<p><del>gone</del></p>' +
+                '<ul><li><input type="checkbox" checked>done</li><li><input type="checkbox">todo</li></ul>');
+            expect(md).toContain('~~gone~~');
+            expect(md).toContain('[x] done');
+            expect(md).toContain('[ ] todo');
+        });
+        it('strips script / style / noscript / form but keeps iframe as a link', async () => {
+            const md = await runAndRead('<p>keep</p>' +
+                '<script>alert(1)</script>' +
+                '<style>.x{color:red}</style>' +
+                '<noscript>nojs</noscript>' +
+                '<iframe src="https://www.youtube.com/embed/abc" title="Demo video"></iframe>' +
+                '<form><button>click</button></form>');
+            expect(md).toContain('keep');
+            expect(md).not.toContain('alert');
+            expect(md).not.toContain('color:red');
+            expect(md).not.toContain('nojs');
+            expect(md).not.toContain('click');
+            // Iframe degrades to a link preserving the embedded URL.
+            expect(md).toContain('[Demo video](https://www.youtube.com/embed/abc)');
+        });
+        it('strips SVG nodes entirely', async () => {
+            const md = await runAndRead('<p>before</p><svg><circle cx="5" cy="5" r="4"/></svg><p>after</p>');
+            expect(md).toContain('before');
+            expect(md).toContain('after');
+            expect(md).not.toContain('svg');
+            expect(md).not.toContain('circle');
+        });
+        it('drops base64 data URI images but keeps regular images', async () => {
+            const md = await runAndRead('<p><img alt="inline" src="data:image/png;base64,iVBORw0KGgo="></p>' +
+                '<p><img alt="keep" src="https://example.com/a.jpg"></p>');
+            expect(md).not.toContain('data:image');
+            expect(md).toContain('![keep](https://example.com/a.jpg)');
+        });
+        it('collapses 3+ blank lines and strips lone bullet / middle-dot residue', async () => {
+            const md = await runAndRead('<p>top</p>' +
+                '<p>-</p>' +
+                '<p>\u00b7</p>' +
+                '<p>bottom</p>');
+            expect(md).not.toMatch(/\n{3,}/);
+            expect(md).not.toMatch(/^\s*-\s*$/m);
+            expect(md).not.toMatch(/^\s*\u00b7\s*$/m);
+            expect(md).toContain('top');
+            expect(md).toContain('bottom');
+        });
+        it('strips page chrome (header / footer / nav / aside)', async () => {
+            const md = await runAndRead('<header><p>page-header-text</p></header>' +
+                '<nav><a href="/">home-link</a></nav>' +
+                '<p>article-body</p>' +
+                '<aside><p>sidebar-text</p></aside>' +
+                '<footer><p>page-footer-text</p></footer>');
+            expect(md).toContain('article-body');
+            expect(md).not.toContain('page-header-text');
+            expect(md).not.toContain('home-link');
+            expect(md).not.toContain('sidebar-text');
+            expect(md).not.toContain('page-footer-text');
+        });
+        it('cleanSelectors removes matching nodes before conversion', async () => {
+            const md = await runAndRead('<p>keep-me</p>' +
+                '<div class="vote-card">Collapsed card</div>' +
+                '<section class="reward-panel">Reward panel</section>' +
+                '<p>also-keep</p>', { cleanSelectors: ['.vote-card', '.reward-panel'] });
+            expect(md).toContain('keep-me');
+            expect(md).toContain('also-keep');
+            expect(md).not.toContain('Collapsed card');
+            expect(md).not.toContain('Reward panel');
+        });
+        it('cleanSelectors silently ignores invalid selectors', async () => {
+            const md = await runAndRead('<p>survives</p><div class="x">and-this-too</div>', { cleanSelectors: ['!!!not-a-valid-selector', '.missing'] });
+            expect(md).toContain('survives');
+            expect(md).toContain('and-this-too');
+        });
+        it('cleanSelectors keeps valid selectors active when one selector is invalid', async () => {
+            const md = await runAndRead('<p>keep</p><div class="vote-card">strip-me</div><p>also-keep</p>', { cleanSelectors: ['!!!not-a-valid-selector', '.vote-card'] });
+            expect(md).toContain('keep');
+            expect(md).toContain('also-keep');
+            expect(md).not.toContain('strip-me');
+        });
+        it('preserves <video> as inline HTML with src + poster', async () => {
+            const md = await runAndRead('<p>before</p>' +
+                '<video src="https://cdn.example.com/clip.mp4" poster="https://cdn.example.com/poster.jpg"></video>' +
+                '<p>after</p>');
+            expect(md).toContain('<video src="https://cdn.example.com/clip.mp4" controls poster="https://cdn.example.com/poster.jpg"></video>');
+            expect(md).toContain('before');
+            expect(md).toContain('after');
+        });
+        it('falls back to <source> inside <video> when src attribute is absent', async () => {
+            const md = await runAndRead('<video><source src="https://cdn.example.com/clip.mp4" type="video/mp4"></video>');
+            expect(md).toContain('<video src="https://cdn.example.com/clip.mp4" controls></video>');
+        });
+        it('drops <video> with no src and no <source>', async () => {
+            const md = await runAndRead('<p>before</p><video></video><p>after</p>');
+            expect(md).not.toContain('<video');
+            expect(md).toContain('before');
+            expect(md).toContain('after');
+        });
+        it('preserves <audio> as inline HTML', async () => {
+            const md = await runAndRead('<audio src="https://cdn.example.com/podcast.mp3"></audio>');
+            expect(md).toContain('<audio src="https://cdn.example.com/podcast.mp3" controls></audio>');
+        });
+        it('degrades <iframe> to a markdown link with title', async () => {
+            const md = await runAndRead('<iframe src="https://codepen.io/pen/abc" title="Live demo"></iframe>');
+            expect(md).toContain('[Live demo](https://codepen.io/pen/abc)');
+        });
+        it('defaults iframe title to "Embedded content" when missing', async () => {
+            const md = await runAndRead('<iframe src="https://example.com/embed"></iframe>');
+            expect(md).toContain('[Embedded content](https://example.com/embed)');
+        });
+        it('drops <iframe> with no src', async () => {
+            const md = await runAndRead('<p>before</p><iframe></iframe><p>after</p>');
+            expect(md).not.toContain('iframe');
+            expect(md).toContain('before');
+            expect(md).toContain('after');
+        });
+    });
+    describe('stdout mode', () => {
+        it('writes markdown to process.stdout and skips file write', async () => {
+            const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-article-'));
+            tempDirs.push(tempDir);
+            const chunks = [];
+            const originalWrite = process.stdout.write.bind(process.stdout);
+            process.stdout.write = ((chunk) => {
+                chunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
+                return true;
+            });
+            try {
+                const result = await downloadArticle({
+                    title: 'Piped',
+                    contentHtml: '<p>Streaming body</p>',
+                    sourceUrl: 'https://example.com/a',
+                }, {
+                    output: tempDir,
+                    stdout: true,
+                });
+                expect(result[0].status).toBe('success');
+                expect(result[0].saved).toBe('-');
+                expect(fs.readdirSync(tempDir)).toHaveLength(0);
+                const emitted = chunks.join('');
+                expect(emitted).toContain('# Piped');
+                expect(emitted).toContain('Streaming body');
+                expect(emitted.endsWith('\n')).toBe(true);
+            }
+            finally {
+                process.stdout.write = originalWrite;
+            }
+        });
+        it('keeps remote image URLs intact in stdout mode (no download)', async () => {
+            const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-article-'));
+            tempDirs.push(tempDir);
+            const chunks = [];
+            const originalWrite = process.stdout.write.bind(process.stdout);
+            process.stdout.write = ((chunk) => {
+                chunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
+                return true;
+            });
+            try {
+                await downloadArticle({
+                    title: 'WithImage',
+                    contentHtml: '<p><img src="https://example.com/a.jpg"></p>',
+                    imageUrls: ['https://example.com/a.jpg'],
+                }, {
+                    output: tempDir,
+                    downloadImages: true,
+                    stdout: true,
+                });
+                expect(fs.readdirSync(tempDir)).toHaveLength(0);
+                expect(chunks.join('')).toContain('https://example.com/a.jpg');
+            }
+            finally {
+                process.stdout.write = originalWrite;
+            }
+        });
+    });
+});

package/dist/src/download/index.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+import type { BrowserCookie } from '../types.js';
+export type { BrowserCookie } from '../types.js';
+export interface DownloadOptions {
+    cookies?: string;
+    headers?: Record<string, string>;
+    timeout?: number;
+    onProgress?: (received: number, total: number) => void;
+    maxRedirects?: number;
+}
+export interface YtdlpOptions {
+    cookies?: string;
+    cookiesFile?: string;
+    format?: string;
+    extraArgs?: string[];
+    onProgress?: (percent: number) => void;
+}
+/** Check if yt-dlp is available in PATH. */
+export declare function checkYtdlp(): boolean;
+/**
+ * Detect content type from URL and optional headers.
+ */
+export declare function detectContentType(url: string, contentType?: string): 'image' | 'video' | 'document' | 'binary';
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export declare function requiresYtdlp(url: string): boolean;
+/**
+ * HTTP download with progress callback.
+ */
+export declare function httpDownload(url: string, destPath: string, options?: DownloadOptions, redirectCount?: number): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+export declare function resolveRedirectUrl(currentUrl: string, location: string): string;
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export declare function exportCookiesToNetscape(cookies: BrowserCookie[], filePath: string): void;
+export declare function formatCookieHeader(cookies: BrowserCookie[]): string;
+/**
+ * Download video using yt-dlp.
+ */
+export declare function ytdlpDownload(url: string, destPath: string, options?: YtdlpOptions): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Save document content to file.
+ */
+export declare function saveDocument(content: string, destPath: string, format?: 'json' | 'markdown' | 'html' | 'text', metadata?: Record<string, unknown>): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export declare function sanitizeFilename(name: string, maxLength?: number): string;
+/**
+ * Generate filename from URL if not provided.
+ */
+export declare function generateFilename(url: string, index: number, extension?: string): string;
+/**
+ * Get temp directory for cookie files.
+ */
+export declare function getTempDir(): string;