@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/cli/utils.js

@@ -0,0 +1,806 @@
+/**
+ * Shared CLI utilities — config, API client, output formatting, helpers.
+ * Imported by all command modules.
+ */
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
+// ─── CLI version ────────────────────────────────────────────────────────────
+let _cliVersion = '0.0.0';
+try {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    // utils.ts compiles to dist/cli/utils.js; package.json is at dist/../../package.json
+    const pkgPath = resolve(__dirname, '..', '..', 'package.json');
+    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+    _cliVersion = pkg.version;
+}
+catch { /* fallback to 0.0.0 */ }
+export const cliVersion = _cliVersion;
+// ─── Verb aliases ────────────────────────────────────────────────────────────
+// Intercept verb-first syntax before Commander parses
+// "webpeel fetch <url>" → "webpeel <url>"
+// Note: 'read' is intentionally excluded — it's a registered subcommand.
+export const VERB_ALIASES = new Set(['fetch', 'get', 'scrape', 'peel']);
+// ─── Update check ────────────────────────────────────────────────────────────
+export async function checkForUpdates() {
+    try {
+        const res = await fetch('https://registry.npmjs.org/webpeel/latest', {
+            signal: AbortSignal.timeout(2000),
+        });
+        if (!res.ok)
+            return;
+        const data = await res.json();
+        const latest = data.version;
+        if (latest && latest !== cliVersion && cliVersion !== '0.0.0') {
+            // Skip update notice in silent mode
+            if (process.env.WEBPEEL_LOG_LEVEL !== 'silent') {
+                const msg = `Update available: ${cliVersion} → ${latest}`;
+                const cmd = 'npm i -g webpeel@latest';
+                const width = Math.max(msg.length, cmd.length) + 4;
+                const line = '─'.repeat(width);
+                console.error(`\n\x1b[33m╭${line}╮\x1b[0m`);
+                console.error(`\x1b[33m│\x1b[0m  ${msg.padEnd(width - 2)}  \x1b[33m│\x1b[0m`);
+                console.error(`\x1b[33m│\x1b[0m  Run: \x1b[36m${cmd}\x1b[0m${' '.repeat(width - 6 - cmd.length)}  \x1b[33m│\x1b[0m`);
+                console.error(`\x1b[33m╰${line}╯\x1b[0m\n`);
+            }
+        }
+    }
+    catch { /* silently ignore — don't slow down the user */ }
+}
+// ─── ANSI helpers ────────────────────────────────────────────────────────────
+export const NO_COLOR = process.env.NO_COLOR !== undefined || !process.stdout.isTTY;
+export const bold = (s) => NO_COLOR ? s : `\x1b[1m${s}\x1b[0m`;
+export const dim = (s) => NO_COLOR ? s : `\x1b[2m${s}\x1b[0m`;
+export const cyan = (s) => NO_COLOR ? s : `\x1b[36m${s}\x1b[0m`;
+// ─── Parse page actions ──────────────────────────────────────────────────────
+/**
+ * Parse action strings into PageAction array
+ * Formats:
+ *   click:.selector         — click an element
+ *   type:.selector=text     — type text into an input
+ *   fill:.selector=text     — fill an input (replaces existing value)
+ *   scroll:down:500         — scroll direction + amount
+ *   scroll:bottom           — scroll to bottom (legacy)
+ *   scroll:top              — scroll to top (legacy)
+ *   wait:2000               — wait N ms
+ *   press:Enter             — press a keyboard key
+ *   hover:.selector         — hover over an element
+ *   waitFor:.selector       — wait for a selector to appear
+ *   select:.selector=value  — select dropdown option
+ *   screenshot              — take a screenshot
+ */
+export function parseActions(actionStrings) {
+    return actionStrings.map(str => {
+        const [type, ...rest] = str.split(':');
+        const value = rest.join(':');
+        switch (type) {
+            case 'wait':
+                return { type: 'wait', ms: parseInt(value) || 1000 };
+            case 'click':
+                return { type: 'click', selector: value };
+            case 'scroll': {
+                // scroll:down:500  or  scroll:bottom  or  scroll:500  or  scroll:0,1500
+                const parts = value.split(':');
+                const dir = parts[0];
+                // Handle scroll:x,y format (e.g., scroll:0,1500)
+                if (dir && dir.includes(',')) {
+                    const [x, y] = dir.split(',').map(Number);
+                    if (!isNaN(x) && !isNaN(y)) {
+                        return { type: 'scroll', to: { x, y } };
+                    }
+                }
+                if (dir === 'top' || dir === 'bottom') {
+                    return { type: 'scroll', to: dir };
+                }
+                if (dir === 'down' || dir === 'up' || dir === 'left' || dir === 'right') {
+                    const amount = parseInt(parts[1] || '500', 10);
+                    return { type: 'scroll', direction: dir, amount };
+                }
+                // Bare number: absolute position
+                const num = parseInt(dir, 10);
+                if (!isNaN(num)) {
+                    return { type: 'scroll', to: num };
+                }
+                // Default: scroll to bottom
+                return { type: 'scroll', to: 'bottom' };
+            }
+            case 'type': {
+                const [sel, ...text] = value.split('=');
+                return { type: 'type', selector: sel, value: text.join('=') };
+            }
+            case 'fill': {
+                const [sel, ...text] = value.split('=');
+                return { type: 'fill', selector: sel, value: text.join('=') };
+            }
+            case 'select': {
+                const [sel, ...vals] = value.split('=');
+                return { type: 'select', selector: sel, value: vals.join('=') };
+            }
+            case 'press':
+                return { type: 'press', key: value };
+            case 'hover':
+                return { type: 'hover', selector: value };
+            case 'waitFor':
+                return { type: 'waitForSelector', selector: value };
+            case 'wait-for':
+                return { type: 'waitForSelector', selector: value, timeout: 10000 };
+            case 'screenshot':
+                return { type: 'screenshot' };
+            default:
+                throw new Error(`Unknown action type: ${type}`);
+        }
+    });
+}
+// ─── Format error ────────────────────────────────────────────────────────────
+/**
+ * Format an error with actionable suggestions based on error type
+ */
+export function formatError(error, _url, options) {
+    const msg = error.message || String(error);
+    const errorType = error.errorType || '';
+    const lines = [`\x1b[31m✖ ${msg}\x1b[0m`];
+    // Check structured errorType from API first (takes precedence over message heuristics)
+    if (errorType === 'timeout' || msg.includes('took too long') || msg.includes('timeout') || msg.includes('Timeout') || msg.includes('Navigation timeout')) {
+        lines.push('\x1b[33m💡 Try increasing timeout: --timeout 60000\x1b[0m');
+        if (!options.render) {
+            lines.push('\x1b[33m💡 Site may need browser rendering: --render\x1b[0m');
+        }
+    }
+    else if (errorType === 'blocked' || msg.includes('blocking automated') || msg.includes('bot protection') || msg.includes('blocked') || msg.includes('403') || msg.includes('Access Denied') || msg.includes('challenge')) {
+        if (!options.stealth) {
+            lines.push('\x1b[33m💡 Try stealth mode to bypass bot detection: --stealth\x1b[0m');
+        }
+        lines.push('\x1b[33m💡 Try a different user agent: --ua "Mozilla/5.0..."\x1b[0m');
+    }
+    else if (errorType === 'not_found' || msg.includes('domain may not exist') || msg.includes('not found') || msg.includes('ENOTFOUND') || msg.includes('net::ERR_') || msg.includes('ECONNREFUSED')) {
+        lines.push('\x1b[33m💡 Check the URL is correct and the site is accessible.\x1b[0m');
+    }
+    else if (errorType === 'network' || msg.includes('Could not reach') || msg.includes('could not connect') || msg.includes('ECONNREFUSED') || msg.includes('ENOTFOUND')) {
+        lines.push('\x1b[33m💡 Check the URL is correct and the site is accessible.\x1b[0m');
+    }
+    else if (errorType === 'server_error' || msg.includes('server error')) {
+        lines.push('\x1b[33m💡 The target site returned a server error. Try again in a moment.\x1b[0m');
+    }
+    else if (msg.includes('empty') || msg.includes('no content') || msg.includes('0 tokens')) {
+        if (!options.render) {
+            lines.push('\x1b[33m💡 Page may be JavaScript-rendered. Try: --render\x1b[0m');
+        }
+        else if (!options.stealth) {
+            lines.push('\x1b[33m💡 Content may be behind bot detection. Try: --stealth\x1b[0m');
+        }
+        lines.push('\x1b[33m💡 Try waiting longer for content: --wait 5000\x1b[0m');
+    }
+    else if (msg.includes('captcha') || msg.includes('CAPTCHA') || msg.includes('Captcha')) {
+        lines.push('\x1b[33m💡 This site requires CAPTCHA solving. Try a browser profile: --profile mysite --headed\x1b[0m');
+    }
+    else if (msg.includes('rate limit') || msg.includes('429')) {
+        lines.push('\x1b[33m💡 Rate limited. Wait a moment and try again, or use --proxy.\x1b[0m');
+    }
+    else if (msg.toLowerCase().includes('enotfound') || msg.toLowerCase().includes('getaddrinfo')) {
+        lines.push('\x1b[33m💡 Could not resolve hostname. Check the URL is correct.\x1b[0m');
+    }
+    else if (msg.toLowerCase().includes('certificate') || msg.toLowerCase().includes('ssl') || msg.toLowerCase().includes('tls')) {
+        lines.push('\x1b[33m💡 SSL/TLS error. The site may have an invalid certificate.\x1b[0m');
+    }
+    else if (msg.toLowerCase().includes('usage') || msg.toLowerCase().includes('quota') || msg.toLowerCase().includes('limit')) {
+        lines.push('\x1b[33m💡 Run `webpeel usage` to check your quota, or `webpeel login` to authenticate.\x1b[0m');
+    }
+    return lines.join('\n');
+}
+// ─── API-based fetch ─────────────────────────────────────────────────────────
+/**
+ * Routes ALL fetch requests through the WebPeel API.
+ * CLI is a pure API client — no local Playwright.
+ */
+export async function fetchViaApi(url, options, apiKey, apiUrl) {
+    // --format is a CLI output flag; API format is always the content extraction format
+    const apiFormat = (['text', 'html', 'markdown', 'md'].includes((options.format || '').toLowerCase()))
+        ? (options.format.toLowerCase() === 'md' ? 'markdown' : options.format.toLowerCase())
+        : (options.html ? 'html' : options.text ? 'text' : 'markdown');
+    const params = new URLSearchParams({ url, format: apiFormat });
+    if (options.render)
+        params.set('render', 'true');
+    if (options.stealth)
+        params.set('stealth', 'true');
+    if (options.wait)
+        params.set('wait', String(options.wait));
+    if (options.selector)
+        params.set('selector', options.selector);
+    if (options.readable)
+        params.set('readable', 'true');
+    if (options.summary)
+        params.set('summary', 'true');
+    if (options.budget)
+        params.set('budget', String(options.budget));
+    if (options.question)
+        params.set('question', options.question);
+    if (options.noDomainApi)
+        params.set('noDomainApi', 'true');
+    if (options.highlightQuery)
+        params.set('highlightQuery', options.highlightQuery);
+    if (options.highlightMaxChars)
+        params.set('highlightMaxChars', String(options.highlightMaxChars));
+    const res = await fetch(`${apiUrl}/v1/fetch?${params}`, {
+        headers: { Authorization: `Bearer ${apiKey}` },
+        signal: AbortSignal.timeout(60000),
+    });
+    if (res.status === 401) {
+        throw Object.assign(new Error('API key invalid or expired. Run: webpeel auth <new-key>'), { code: 'AUTH_FAILED' });
+    }
+    if (res.status === 429) {
+        throw Object.assign(new Error('Rate limit exceeded. Check your plan at https://app.webpeel.dev/billing'), { code: 'RATE_LIMITED' });
+    }
+    if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        // Sanitize error message — don't expose raw HTML (e.g. Cloudflare 502 pages)
+        const isHtml = body.trimStart().startsWith('<') || body.includes('<!DOCTYPE') || body.includes('<html');
+        let errorMsg;
+        let errorType;
+        if (res.status === 502 || res.status === 503 || res.status === 504) {
+            errorMsg = `Could not reach this website. The site may be blocking our server or timing out.`;
+            errorType = res.status === 504 ? 'timeout' : 'network';
+        }
+        else if (isHtml) {
+            errorMsg = `Server returned an error page (${res.status})`;
+        }
+        else {
+            // Try to parse a structured JSON error response
+            try {
+                const json = JSON.parse(body);
+                const errObj = json?.error;
+                if (errObj && typeof errObj === 'object') {
+                    errorMsg = typeof errObj.message === 'string' ? errObj.message : (body.slice(0, 200) || 'Unknown error');
+                    if (typeof errObj.type === 'string')
+                        errorType = errObj.type;
+                }
+                else {
+                    errorMsg = body.slice(0, 200) || 'Unknown error';
+                }
+            }
+            catch {
+                errorMsg = body.slice(0, 200) || 'Unknown error';
+            }
+        }
+        const err = new Error(`${errorMsg}`);
+        if (errorType)
+            err.errorType = errorType;
+        err.statusCode = res.status;
+        throw err;
+    }
+    let data = await res.json();
+    // Handle async job queue mode — API returns { jobId, pollUrl } and we need to poll
+    if (data.jobId && data.pollUrl && !data.content) {
+        const pollEndpoint = `${apiUrl}${data.pollUrl}`;
+        const maxPollMs = 90_000; // 90s max
+        const pollInterval = 1_000; // 1s intervals
+        const start = Date.now();
+        while (Date.now() - start < maxPollMs) {
+            await new Promise(r => setTimeout(r, pollInterval));
+            const pollRes = await fetch(pollEndpoint, {
+                headers: { Authorization: `Bearer ${apiKey}` },
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (!pollRes.ok) {
+                throw new Error(`Job poll failed: HTTP ${pollRes.status}`);
+            }
+            const pollData = await pollRes.json();
+            if (pollData.status === 'completed' || pollData.content) {
+                data = pollData.result || pollData;
+                break;
+            }
+            if (pollData.status === 'failed' || pollData.status === 'error') {
+                throw new Error(pollData.error?.message || pollData.error || 'Job failed on server');
+            }
+            // Still processing — keep polling
+        }
+        // If we exited the loop without data, warn
+        if (!data.content && data.jobId) {
+            throw new Error('Job timed out waiting for server response. Try again or use local mode (unset WEBPEEL_API_KEY).');
+        }
+    }
+    // Map API response to PeelResult shape that the CLI already handles
+    // Spread all API fields first, then override with normalized names
+    return {
+        ...data,
+        url: data.url || url,
+        title: data.metadata?.title || data.title || '',
+        content: data.content || '',
+        method: data.method || 'simple',
+        tokens: data.tokenCount || data.tokens || 0,
+        elapsed: data.fetchTimeMs || data.elapsed || 0,
+        tokenSavingsPercent: data.tokenSavingsPercent,
+        rawTokenEstimate: data.rawTokenEstimate,
+        metadata: data.metadata || {},
+        links: data.links || [],
+        answer: data.answer,
+        summary: data.summary,
+        format: options.format || 'markdown',
+    };
+}
+// ─── Help formatting ─────────────────────────────────────────────────────────
+/**
+ * Reconstruct the standard Commander help layout for --help-all and subcommands.
+ * This mirrors Commander's own default formatHelp() so subcommand help keeps working.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function buildCommanderHelp(cmd, helper) {
+    const termWidth = helper.padWidth(cmd, helper);
+    const helpWidth = helper.helpWidth ?? 80;
+    const pad = '  ';
+    const formatItem = (term, description) => {
+        if (description) {
+            const full = `${term.padEnd(termWidth + 2)}${description}`;
+            return helper.wrap(full, helpWidth - pad.length, termWidth + 2);
+        }
+        return term;
+    };
+    const formatList = (items) => items.join('\n').replace(/^/gm, pad);
+    let out = [`Usage: ${helper.commandUsage(cmd)}`, ''];
+    const desc = helper.commandDescription(cmd);
+    if (desc.length > 0) {
+        out = out.concat([helper.wrap(desc, helpWidth, 0), '']);
+    }
+    // Arguments
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const args = helper.visibleArguments(cmd).map(a => formatItem(helper.argumentTerm(a), helper.argumentDescription(a)));
+    if (args.length > 0)
+        out = out.concat(['Arguments:', formatList(args), '']);
+    // Options
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const opts = helper.visibleOptions(cmd).map(o => formatItem(helper.optionTerm(o), helper.optionDescription(o)));
+    if (opts.length > 0)
+        out = out.concat(['Options:', formatList(opts), '']);
+    // Subcommands
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const cmds = helper.visibleCommands(cmd).map(c => formatItem(helper.subcommandTerm(c), helper.subcommandDescription(c)));
+    if (cmds.length > 0)
+        out = out.concat(['Commands:', formatList(cmds), '']);
+    // Append grouped option sections only on root command (--help-all)
+    if (cmd.parent === null) {
+        out = out.concat([`
+Output Formats:
+  --json                  JSON output with full metadata
+  --html                  Raw HTML output
+  --text                  Plain text output
+  --csv / --table         Tabular output for extractions
+  -s, --silent            No spinner or progress output
+
+Content Control:
+  --readable              Reader mode — clean article content only
+  --budget <n>            Smart token budget (no LLM key needed)
+  --focus <query>         BM25 query-focused filtering
+  --selector <css>        Extract specific CSS selector
+  --only-main-content     Just main/article content
+  --full-content          Disable content pruning
+  -q, --question <q>      Ask a question about the content
+
+Rendering:
+  -r, --render            Browser rendering for JS-heavy sites
+  --stealth               Stealth mode for bot-protected sites
+  --profile <path>        Persistent browser profile
+  --headed                Visible browser (for debugging)
+  --action <actions>      Browser automation (click, type, scroll...)
+
+Extraction:
+  --extract <json>        CSS selector extraction
+  --extract-all           Auto-detect listing items
+  --schema <name>         Named extraction schema
+  --llm-extract [inst]    LLM-powered extraction (BYOK)
+
+Examples:
+  $ webpeel "https://example.com"                              Basic fetch
+  $ webpeel "https://youtube.com/watch?v=..." --json           YouTube transcript
+  $ webpeel "https://openai.com/pricing" -q "GPT-4 cost?"     Quick answer
+  $ webpeel "https://nytimes.com/article" --readable           Reader mode
+  $ webpeel search "best restaurants in NYC"                   Web search
+  $ webpeel hotels "Manhattan" --checkin tomorrow              Hotel search
+
+Agent Integration:
+  $ webpeel mcp                                                Start MCP server
+  $ cat urls.txt | webpeel batch                               Batch from stdin
+  $ webpeel pipe "https://example.com" | jq .content           Pipe-friendly JSON
+  $ webpeel "https://site.com" --json --silent                 Same as pipe
+  $ curl https://webpeel.dev/llms.txt                          AI-readable docs
+`]);
+    }
+    return out.join('\n');
+}
+/**
+ * Condensed, Anthropic-style help for the root command (default --help).
+ */
+export function buildCondensedHelp() {
+    const v = cliVersion;
+    return [
+        '',
+        `  ${bold('◆ WebPeel')} ${dim(`v${v}`)}`,
+        `  ${dim('The web data platform for AI agents')}`,
+        '',
+        `  ${bold('Usage:')}  webpeel [url] [options]`,
+        `          webpeel <command> [options]`,
+        '',
+        `  ${bold('Examples:')}`,
+        `    webpeel https://example.com            ${dim('Clean content (reader mode)')}`,
+        `    webpeel read https://example.com       ${dim('Explicit reader mode')}`,
+        `    webpeel screenshot https://example.com ${dim('Screenshot any page')}`,
+        `    webpeel ask https://news.com "summary" ${dim('Ask about any page')}`,
+        `    webpeel search "webpeel vs jina"       ${dim('Web search')}`,
+        `    echo "url" | webpeel                   ${dim('Pipe mode (auto JSON)')}`,
+        '',
+        `  ${bold('Commands:')}`,
+        `    fetch (default)       Fetch a URL as clean markdown`,
+        `    read <url>            Reader mode (article content only)`,
+        `    screenshot <url>      Take a screenshot`,
+        `    ask <url> <question>  Ask about any page`,
+        `    search <query>        Search the web (DuckDuckGo + sources)`,
+        `    crawl <url>           Crawl a website`,
+        `    mcp                   Start MCP server for AI tools`,
+        `    ${dim('... (use --help-all for all 25+ commands)')}`,
+        '',
+        `  ${bold('Common Options:')}`,
+        `    -r, --render          Browser rendering (JS-heavy sites)`,
+        `    --stealth             Stealth mode (anti-bot bypass)`,
+        `    --raw                 Full page (disable auto reader mode)`,
+        `    --full                Full page, no budget limit`,
+        `    --json                JSON output with metadata`,
+        `    --budget <n>          Token budget (default: 4000 in pipe mode)`,
+        `    -q, --question <q>    Ask about the content`,
+        `    -s, --silent          No spinner output`,
+        '',
+        `  Use ${cyan("'webpeel <command> --help'")} for command-specific options.`,
+        `  Use ${cyan("'webpeel --help-all'")} for the full option reference.`,
+        '',
+        `  Docs: ${cyan('https://webpeel.dev/docs')}`,
+        '',
+    ].join('\n');
+}
+// ─── Time formatting ─────────────────────────────────────────────────────────
+/**
+ * Format a past Date relative to now (e.g. "2h ago", "5m ago").
+ */
+export function formatRelativeTime(past) {
+    const diffMs = Date.now() - past.getTime();
+    const diffSec = Math.round(diffMs / 1000);
+    if (diffSec < 60)
+        return `${diffSec}s ago`;
+    const diffMin = Math.round(diffSec / 60);
+    if (diffMin < 60)
+        return `${diffMin}m ago`;
+    const diffHr = Math.round(diffMin / 60);
+    if (diffHr < 24)
+        return `${diffHr}h ago`;
+    const diffDay = Math.round(diffHr / 24);
+    return `${diffDay}d ago`;
+}
+// ─── Error classification ─────────────────────────────────────────────────────
+export function classifyErrorCode(error) {
+    if (!(error instanceof Error))
+        return 'FETCH_FAILED';
+    // Check for our custom _code first (set in pre-fetch validation)
+    if (error._code)
+        return error._code;
+    // Check for structured errorType from API responses (set by fetchViaApi)
+    const errorType = error.errorType;
+    if (errorType) {
+        const typeMap = {
+            timeout: 'TIMEOUT',
+            blocked: 'BLOCKED',
+            not_found: 'NOT_FOUND',
+            server_error: 'SERVER_ERROR',
+            network: 'NETWORK',
+            unknown: 'FETCH_FAILED',
+        };
+        if (typeMap[errorType])
+            return typeMap[errorType];
+    }
+    const msg = error.message.toLowerCase();
+    const name = error.name || '';
+    if (name === 'TimeoutError' || msg.includes('timeout') || msg.includes('timed out') || msg.includes('took too long')) {
+        return 'TIMEOUT';
+    }
+    if (name === 'BlockedError' || msg.includes('blocked') || msg.includes('403') || msg.includes('cloudflare') || msg.includes('bot protection')) {
+        return 'BLOCKED';
+    }
+    if (msg.includes('domain may not exist') || msg.includes('enotfound') || msg.includes('getaddrinfo') || msg.includes('dns resolution failed')) {
+        return 'NOT_FOUND';
+    }
+    if (msg.includes('http 404') || msg.includes('page was not found')) {
+        return 'NOT_FOUND';
+    }
+    if (msg.includes('invalid url') || msg.includes('invalid hostname') || msg.includes('only http')) {
+        return 'INVALID_URL';
+    }
+    if (msg.includes('could not reach') || msg.includes('could not connect') || msg.includes('econnrefused')) {
+        return 'NETWORK';
+    }
+    return 'FETCH_FAILED';
+}
+/**
+ * Build a unified PeelEnvelope from a PeelResult.
+ *
+ * All existing PeelResult fields are spread first (backward compatibility),
+ * then canonical envelope fields override/extend them.
+ */
+export function buildEnvelope(result, extra) {
+    const envelope = {
+        // Spread all PeelResult fields for backward compatibility
+        ...result,
+        // Required envelope fields (override PeelResult where they overlap)
+        url: result.url,
+        status: 200,
+        content: result.content,
+        metadata: {
+            title: result.title,
+            ...result.metadata,
+        },
+        tokens: result.tokens,
+        cached: extra.cached ?? false,
+        elapsed: result.elapsed,
+    };
+    // Optional envelope fields — only include when meaningful
+    if (extra.structured !== undefined)
+        envelope.structured = extra.structured;
+    if (extra.truncated)
+        envelope.truncated = true;
+    if (extra.totalAvailable !== undefined)
+        envelope.totalAvailable = extra.totalAvailable;
+    return envelope;
+}
+// ─── Output result ───────────────────────────────────────────────────────────
+export async function outputResult(result, options, extra = {}) {
+    // --links: output only links
+    if (options.links) {
+        if (options.json) {
+            const jsonStr = JSON.stringify(result.links, null, 2);
+            await writeStdout(jsonStr + '\n');
+        }
+        else {
+            for (const link of result.links) {
+                await writeStdout(link + '\n');
+            }
+        }
+        return;
+    }
+    // --images: output only image URLs
+    if (options.images) {
+        // Extract image URLs from links that point to images
+        const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico'];
+        const imageUrls = result.links.filter(link => {
+            const urlLower = link.toLowerCase();
+            return imageExtensions.some(ext => urlLower.includes(ext));
+        });
+        if (options.json) {
+            const jsonStr = JSON.stringify(imageUrls, null, 2);
+            await writeStdout(jsonStr + '\n');
+        }
+        else {
+            for (const imageUrl of imageUrls) {
+                await writeStdout(imageUrl + '\n');
+            }
+        }
+        return;
+    }
+    // --meta: output only metadata
+    if (options.meta) {
+        const meta = {
+            url: result.url,
+            title: result.title,
+            method: result.method,
+            elapsed: result.elapsed,
+            tokens: result.tokens,
+            cached: extra.cached ?? false,
+            ...result.metadata,
+        };
+        if (options.json) {
+            await writeStdout(JSON.stringify(meta, null, 2) + '\n');
+        }
+        else {
+            console.log(`Title:       ${meta.title || '(none)'}`);
+            console.log(`URL:         ${meta.url}`);
+            if (meta.description)
+                console.log(`Description: ${meta.description}`);
+            if (meta.author)
+                console.log(`Author:      ${meta.author}`);
+            if (meta.published)
+                console.log(`Published:   ${meta.published}`);
+            if (meta.canonical)
+                console.log(`Canonical:   ${meta.canonical}`);
+            if (meta.image)
+                console.log(`OG Image:    ${meta.image}`);
+            console.log(`Method:      ${meta.method}`);
+            console.log(`Elapsed:     ${meta.elapsed}ms`);
+            console.log(`Tokens:      ${meta.tokens}`);
+            console.log(`Cached:      ${meta.cached}`);
+        }
+        return;
+    }
+    // Default: full output
+    if (options.json) {
+        // Build clean JSON output with guaranteed top-level fields
+        // Note: elapsed/method/tokens are placed at the END so `tail -3` shows perf metrics
+        const output = {
+            url: result.url,
+            title: result.metadata?.title || result.title || null,
+            fetchedAt: new Date().toISOString(),
+            content: result.content,
+        };
+        // Add optional fields only if present (filter out undefined/null values from metadata)
+        if (result.metadata) {
+            const cleanMeta = {};
+            for (const [k, v] of Object.entries(result.metadata)) {
+                if (v !== undefined && v !== null)
+                    cleanMeta[k] = v;
+            }
+            if (Object.keys(cleanMeta).length > 0)
+                output.metadata = cleanMeta;
+        }
+        if (result.links?.length)
+            output.links = result.links;
+        if (result.tokenSavingsPercent !== undefined)
+            output.tokenSavingsPercent = result.tokenSavingsPercent;
+        if (result.rawTokenEstimate !== undefined)
+            output.rawTokenEstimate = result.rawTokenEstimate;
+        if (result.images?.length)
+            output.images = result.images;
+        if (result.structured)
+            output.structured = result.structured;
+        if (result.domainData)
+            output.domainData = result.domainData;
+        if (result.readability)
+            output.readability = result.readability;
+        if (result.quickAnswer)
+            output.quickAnswer = result.quickAnswer;
+        if (result.quality)
+            output.quality = result.quality;
+        if (result.contentType)
+            output.contentType = result.contentType;
+        if (result.chunks)
+            output.chunks = result.chunks;
+        if (result.totalChunks)
+            output.totalChunks = result.totalChunks;
+        if (result.warning)
+            output.warning = result.warning;
+        if (result.focusQuery)
+            output.focusQuery = result.focusQuery;
+        if (result.focusReduction)
+            output.focusReduction = result.focusReduction;
+        if (result.extracted)
+            output.extracted = result.extracted;
+        if (result.trust)
+            output.trust = result.trust;
+        if (result.safeBrowsing)
+            output.safeBrowsing = result.safeBrowsing;
+        if (extra.cached)
+            output.cached = true;
+        if (extra.truncated)
+            output.truncated = true;
+        if (extra.totalAvailable !== undefined)
+            output.totalAvailable = extra.totalAvailable;
+        output._meta = { version: cliVersion, method: result.method || 'simple', timing: result.timing, serverMarkdown: result.serverMarkdown || false };
+        // Perf metrics at the end — `tail -3` shows: elapsed | method | tokens
+        output.elapsed = result.elapsed;
+        output.method = result.method || 'simple';
+        output.tokens = result.tokens || 0;
+        await writeStdout(JSON.stringify(output, null, 2) + '\n');
+    }
+    else {
+        // Smart terminal header (interactive mode only)
+        const isTerminalOutput = process.stdout.isTTY && !options.silent;
+        if (isTerminalOutput) {
+            const meta = result.metadata || {};
+            const parts = [];
+            if (meta.title || result.title)
+                parts.push(`\x1b[1m${meta.title || result.title}\x1b[0m`);
+            if (meta.author)
+                parts.push(`By ${meta.author}`);
+            if (meta.wordCount)
+                parts.push(`${meta.wordCount} words`);
+            const totalMs = result.timing?.total ?? result.elapsed;
+            if (totalMs)
+                parts.push(`${totalMs}ms`);
+            if (parts.length > 0) {
+                await writeStdout(`\n  ${parts.join(' · ')}\n`);
+                await writeStdout('  ' + '─'.repeat(60) + '\n\n');
+            }
+        }
+        // Stream content immediately to stdout — consumer gets it without waiting
+        await writeStdout(result.content + '\n');
+        // Append timing summary to stderr (always — doesn't pollute stdout pipe)
+        {
+            const totalMs = result.timing?.total ?? result.elapsed;
+            const method = result.method || 'simple';
+            process.stderr.write(`\n--- ${totalMs}ms | ${method} | ${result.tokens} tokens ---\n`);
+        }
+    }
+}
+// ─── Write helpers ────────────────────────────────────────────────────────────
+export function writeStdout(data) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(data, (err) => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+// ─── Listings / CSV / table helpers ──────────────────────────────────────────
+/**
+ * Convert an array of listing items to CSV.
+ */
+export function formatListingsCsv(items) {
+    if (items.length === 0)
+        return '';
+    // Collect all keys
+    const keySet = new Set();
+    for (const item of items) {
+        for (const key of Object.keys(item)) {
+            if (item[key] !== undefined)
+                keySet.add(key);
+        }
+    }
+    const keys = Array.from(keySet);
+    const escapeCsv = (s) => {
+        if (s === undefined || s === null)
+            return '""';
+        const str = String(s);
+        if (str.includes('"') || str.includes(',') || str.includes('\n') || str.includes('\r')) {
+            return '"' + str.replace(/"/g, '""') + '"';
+        }
+        return '"' + str + '"';
+    };
+    const lines = [keys.join(',')];
+    for (const item of items) {
+        lines.push(keys.map(k => escapeCsv(item[k])).join(','));
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Normalise the result of --extract (which may be a flat object or contain
+ * arrays) into an array of row objects suitable for CSV / table rendering.
+ */
+export function normaliseExtractedToRows(extracted) {
+    // If every value is an array of the same length, zip them into rows
+    const values = Object.values(extracted);
+    const allArrays = values.length > 0 && values.every(v => Array.isArray(v));
+    if (allArrays) {
+        const length = values[0].length;
+        const rows = [];
+        for (let i = 0; i < length; i++) {
+            const row = {};
+            for (const key of Object.keys(extracted)) {
+                const val = extracted[key][i];
+                row[key] = val != null ? String(val) : undefined;
+            }
+            rows.push(row);
+        }
+        return rows;
+    }
+    // Otherwise treat as a single row
+    const row = {};
+    for (const [k, v] of Object.entries(extracted)) {
+        row[k] = v != null ? String(v) : undefined;
+    }
+    return [row];
+}
+// ─── Branding helpers ────────────────────────────────────────────────────────
+/** Helper function to extract colors from content */
+export function extractColors(content) {
+    const colors = [];
+    const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
+    const matches = content.match(hexRegex);
+    if (matches) {
+        colors.push(...[...new Set(matches)].slice(0, 10));
+    }
+    return colors;
+}
+/** Helper function to extract font information */
+export function extractFonts(content) {
+    const fonts = [];
+    const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
+    let match;
+    while ((match = fontRegex.exec(content)) !== null) {
+        fonts.push(match[1].trim());
+    }
+    return [...new Set(fonts)].slice(0, 5);
+}