@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/cli/commands/setup.d.ts

@@ -0,0 +1,13 @@
+/**
+ * `webpeel setup` — Interactive onboarding wizard.
+ *
+ * Walks the user through:
+ *   1. API key configuration (optional — local mode is fine)
+ *   2. Browser installation (Chromium via Playwright)
+ *   3. LLM key for AI extraction (optional)
+ *   4. Quick smoke test
+ *
+ * Non-interactive when piped (detects !isTTY).
+ */
+import type { Command } from 'commander';
+export declare function registerSetupCommand(program: Command): void;

package/dist/cli/commands/setup.js

@@ -0,0 +1,244 @@
+/**
+ * `webpeel setup` — Interactive onboarding wizard.
+ *
+ * Walks the user through:
+ *   1. API key configuration (optional — local mode is fine)
+ *   2. Browser installation (Chromium via Playwright)
+ *   3. LLM key for AI extraction (optional)
+ *   4. Quick smoke test
+ *
+ * Non-interactive when piped (detects !isTTY).
+ */
+import { createInterface } from 'readline';
+import { existsSync } from 'fs';
+import { loadConfig, saveConfig } from '../../cli-auth.js';
+import { cliVersion, NO_COLOR } from '../utils.js';
+import { runDoctor } from './doctor.js';
+// ── ANSI helpers ────────────────────────────────────────────────────────────
+const bold = (s) => NO_COLOR ? s : `\x1b[1m${s}\x1b[0m`;
+const dim = (s) => NO_COLOR ? s : `\x1b[2m${s}\x1b[0m`;
+const green = (s) => NO_COLOR ? s : `\x1b[32m${s}\x1b[0m`;
+const yellow = (s) => NO_COLOR ? s : `\x1b[33m${s}\x1b[0m`;
+const cyan = (s) => NO_COLOR ? s : `\x1b[36m${s}\x1b[0m`;
+// ── Prompt helper ───────────────────────────────────────────────────────────
+function ask(rl, question) {
+    return new Promise((resolve) => {
+        rl.question(question, (answer) => resolve(answer.trim()));
+    });
+}
+async function confirm(rl, question, defaultYes = true) {
+    const suffix = defaultYes ? '[Y/n]' : '[y/N]';
+    const answer = await ask(rl, `${question} ${dim(suffix)} `);
+    if (answer === '')
+        return defaultYes;
+    return /^y(es)?$/i.test(answer);
+}
+// ── Wizard steps ────────────────────────────────────────────────────────────
+async function stepApiKey(rl) {
+    const cfg = loadConfig();
+    const existing = cfg.apiKey || process.env.WEBPEEL_API_KEY;
+    console.log('');
+    console.log(bold('Step 1: API Key'));
+    console.log(dim('An API key unlocks cloud features (faster, higher limits, no browser needed).'));
+    console.log(dim('WebPeel works locally without one — this is optional.'));
+    console.log('');
+    if (existing) {
+        console.log(green(`  ✅ API key already configured: ${existing.slice(0, 8)}...${existing.slice(-4)}`));
+        const change = await confirm(rl, '  Change it?', false);
+        if (!change)
+            return;
+    }
+    const key = await ask(rl, `  Enter API key ${dim('(or press Enter to skip)')}: `);
+    if (!key) {
+        console.log(dim('  Skipped — using local mode.'));
+        return;
+    }
+    // Quick validation
+    const apiUrl = process.env.WEBPEEL_API_URL || 'https://api.webpeel.dev';
+    process.stdout.write('  Verifying...');
+    try {
+        const res = await fetch(`${apiUrl}/v1/usage`, {
+            headers: { Authorization: `Bearer ${key}` },
+            signal: AbortSignal.timeout(8000),
+        });
+        if (res.status === 401) {
+            console.log(` ${yellow('invalid key')}. Skipping.`);
+            console.log(dim('  Get a free key at: https://app.webpeel.dev/keys'));
+            return;
+        }
+        if (res.ok) {
+            const data = await res.json();
+            const plan = data?.tier || (typeof data?.plan === 'string' ? data?.plan : data?.plan?.tier) || 'free';
+            cfg.apiKey = key;
+            saveConfig(cfg);
+            console.log(` ${green(`✅ Verified (${plan} plan)`)}`);
+            return;
+        }
+        // Non-200, non-401 — save anyway
+        cfg.apiKey = key;
+        saveConfig(cfg);
+        console.log(` ${yellow(`saved (API returned ${res.status})`)}`);
+    }
+    catch (e) {
+        // Network error — save key, warn
+        cfg.apiKey = key;
+        saveConfig(cfg);
+        console.log(` ${yellow(`saved (couldn't verify: ${e.message})`)}`);
+    }
+}
+async function stepBrowser(rl) {
+    console.log('');
+    console.log(bold('Step 2: Browser (Chromium)'));
+    console.log(dim('Required for --render, --stealth, --screenshot, and JS-heavy sites.'));
+    console.log('');
+    let installed = false;
+    try {
+        const pw = await import('playwright');
+        const execPath = pw.chromium.executablePath();
+        installed = existsSync(execPath);
+    }
+    catch {
+        // playwright not importable
+    }
+    if (installed) {
+        console.log(green('  ✅ Chromium is already installed.'));
+        return;
+    }
+    const install = await confirm(rl, '  Install Chromium now? (~150MB, takes ~30s)', true);
+    if (!install) {
+        console.log(dim('  Skipped — browser features will auto-install on first use.'));
+        return;
+    }
+    console.log(dim('  Installing Chromium via Playwright...'));
+    try {
+        const { execSync } = await import('child_process');
+        execSync('npx playwright install chromium', {
+            stdio: 'inherit',
+            timeout: 120_000,
+        });
+        console.log(green('  ✅ Chromium installed.'));
+    }
+    catch (e) {
+        console.log(yellow(`  ⚠️  Install failed: ${e.message}`));
+        console.log(dim('  You can install later: npx playwright install chromium'));
+    }
+}
+async function stepLlm(rl) {
+    const cfg = loadConfig();
+    const existing = cfg.llm?.apiKey || process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.GOOGLE_API_KEY;
+    console.log('');
+    console.log(bold('Step 3: LLM Key (AI Extraction)'));
+    console.log(dim('Enables --extract for structured data extraction using AI.'));
+    console.log(dim('Supports OpenAI, Anthropic, or Google keys. Optional.'));
+    console.log('');
+    if (existing) {
+        const source = cfg.llm?.apiKey ? 'config' : 'environment variable';
+        console.log(green(`  ✅ LLM key found (via ${source}).`));
+        const change = await confirm(rl, '  Change it?', false);
+        if (!change)
+            return;
+    }
+    const key = await ask(rl, `  Enter OpenAI/Anthropic/Google API key ${dim('(or Enter to skip)')}: `);
+    if (!key) {
+        console.log(dim('  Skipped — set later with `webpeel config set llm.apiKey <key>`.'));
+        return;
+    }
+    if (!cfg.llm)
+        cfg.llm = {};
+    cfg.llm.apiKey = key;
+    saveConfig(cfg);
+    console.log(green('  ✅ LLM key saved.'));
+    // Ask for model preference
+    const model = await ask(rl, `  Model name ${dim('(default: gpt-4o-mini, Enter to accept)')}: `);
+    if (model) {
+        cfg.llm.model = model;
+        saveConfig(cfg);
+        console.log(green(`  ✅ Model set to ${model}.`));
+    }
+}
+async function stepSmokeTest(rl) {
+    console.log('');
+    console.log(bold('Step 4: Quick Test'));
+    console.log('');
+    const runTest = await confirm(rl, '  Run a quick fetch test?', true);
+    if (!runTest) {
+        console.log(dim('  Skipped.'));
+        return;
+    }
+    process.stdout.write('  Fetching https://example.com...');
+    try {
+        // Use local peel to test
+        const { peel } = await import('../../index.js');
+        const start = Date.now();
+        const result = await peel('https://example.com');
+        const elapsed = Date.now() - start;
+        const tokens = result.tokens ?? result.content.split(/\s+/).length;
+        console.log(` ${green(`✅ ${tokens} tokens in ${elapsed}ms`)}`);
+    }
+    catch (e) {
+        console.log(` ${yellow(`⚠️  ${e.message}`)}`);
+        console.log(dim('  This is expected for some network configurations. CLI still works fine.'));
+    }
+}
+// ── Main wizard ─────────────────────────────────────────────────────────────
+async function runSetup() {
+    console.log('');
+    console.log(bold('🔧 WebPeel Setup'));
+    console.log(dim(`v${cliVersion} — Fast web fetcher for AI agents`));
+    console.log(dim('─'.repeat(50)));
+    const isTTY = process.stdin.isTTY && process.stdout.isTTY;
+    if (!isTTY) {
+        console.log('');
+        console.log('Non-interactive mode detected. Running doctor instead...');
+        console.log('');
+        const report = await runDoctor();
+        console.log(JSON.stringify(report, null, 2));
+        process.exit(report.summary.error > 0 ? 1 : 0);
+    }
+    const rl = createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    try {
+        await stepApiKey(rl);
+        await stepBrowser(rl);
+        await stepLlm(rl);
+        await stepSmokeTest(rl);
+        // Final summary — run doctor
+        console.log('');
+        console.log(dim('─'.repeat(50)));
+        console.log(bold('📋 Final Status'));
+        const report = await runDoctor();
+        // Print compact summary
+        for (const check of report.checks) {
+            const icon = check.status === 'ok' ? green('✓') : check.status === 'warn' ? yellow('○') : check.status === 'error' ? '✗' : dim('·');
+            console.log(`  ${icon} ${check.name}: ${check.detail}`);
+        }
+        console.log('');
+        if (report.summary.error === 0) {
+            console.log(green(bold('🎉 Setup complete! WebPeel is ready.')));
+        }
+        else {
+            console.log(yellow('⚠️  Setup complete with some issues. Run `webpeel doctor` for details.'));
+        }
+        console.log('');
+        console.log(dim('Quick start:'));
+        console.log(cyan('  webpeel "https://news.ycombinator.com"'));
+        console.log(cyan('  webpeel search "latest AI news"'));
+        console.log(cyan('  webpeel doctor'));
+        console.log('');
+    }
+    finally {
+        rl.close();
+    }
+}
+// ── Commander registration ──────────────────────────────────────────────────
+export function registerSetupCommand(program) {
+    program
+        .command('setup')
+        .description('Interactive setup wizard — configure API key, browser, and LLM')
+        .action(async () => {
+        await runSetup();
+        process.exit(0);
+    });
+}

package/dist/cli/commands/skill.d.ts

@@ -0,0 +1,15 @@
+/**
+ * `webpeel skill` — Manage optional capabilities.
+ *
+ * Lists, installs, and uninstalls optional components:
+ *   - browser     — Chromium via Playwright (--render, --stealth, --screenshot)
+ *   - yt-dlp      — YouTube transcript extraction (auto-detected)
+ *
+ * Usage:
+ *   webpeel skill                     — list all skills with status
+ *   webpeel skill --install browser   — install Chromium
+ *   webpeel skill --uninstall browser — remove Chromium
+ *   webpeel skill --json              — machine-readable output
+ */
+import type { Command } from 'commander';
+export declare function registerSkillCommand(program: Command): void;

package/dist/cli/commands/skill.js

@@ -0,0 +1,195 @@
+/**
+ * `webpeel skill` — Manage optional capabilities.
+ *
+ * Lists, installs, and uninstalls optional components:
+ *   - browser     — Chromium via Playwright (--render, --stealth, --screenshot)
+ *   - yt-dlp      — YouTube transcript extraction (auto-detected)
+ *
+ * Usage:
+ *   webpeel skill                     — list all skills with status
+ *   webpeel skill --install browser   — install Chromium
+ *   webpeel skill --uninstall browser — remove Chromium
+ *   webpeel skill --json              — machine-readable output
+ */
+import { existsSync } from 'fs';
+import { execSync } from 'child_process';
+import { NO_COLOR } from '../utils.js';
+// ── ANSI helpers ────────────────────────────────────────────────────────────
+const bold = (s) => NO_COLOR ? s : `\x1b[1m${s}\x1b[0m`;
+const dim = (s) => NO_COLOR ? s : `\x1b[2m${s}\x1b[0m`;
+const green = (s) => NO_COLOR ? s : `\x1b[32m${s}\x1b[0m`;
+const yellow = (s) => NO_COLOR ? s : `\x1b[33m${s}\x1b[0m`;
+const red = (s) => NO_COLOR ? s : `\x1b[31m${s}\x1b[0m`;
+async function isBrowserInstalled() {
+    try {
+        const pw = await import('playwright');
+        return existsSync(pw.chromium.executablePath());
+    }
+    catch {
+        return false;
+    }
+}
+function isYtdlpInstalled() {
+    try {
+        execSync('yt-dlp --version', { stdio: 'pipe', timeout: 5000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const SKILLS = [
+    {
+        name: 'browser',
+        description: 'Chromium browser via Playwright',
+        features: '--render, --stealth, --screenshot, --action, interactive sessions',
+        installed: isBrowserInstalled,
+        install: async () => {
+            console.log(dim('Installing Chromium via Playwright...'));
+            execSync('npx playwright install chromium', { stdio: 'inherit', timeout: 120_000 });
+        },
+        uninstall: async () => {
+            console.log(dim('Removing Playwright browsers...'));
+            execSync('npx playwright uninstall', { stdio: 'inherit', timeout: 60_000 });
+        },
+        installHint: 'npx playwright install chromium',
+        uninstallHint: 'npx playwright uninstall',
+    },
+    {
+        name: 'yt-dlp',
+        description: 'YouTube transcript & metadata extractor',
+        features: 'Enhanced YouTube extraction (transcripts, metadata, chapters)',
+        installed: () => isYtdlpInstalled(),
+        install: async () => {
+            const platform = process.platform;
+            if (platform === 'darwin') {
+                console.log(dim('Installing yt-dlp via Homebrew...'));
+                execSync('brew install yt-dlp', { stdio: 'inherit', timeout: 120_000 });
+            }
+            else if (platform === 'linux') {
+                console.log(dim('Installing yt-dlp via pip...'));
+                execSync('pip3 install yt-dlp', { stdio: 'inherit', timeout: 120_000 });
+            }
+            else {
+                console.log(dim('Installing yt-dlp via pip...'));
+                execSync('pip install yt-dlp', { stdio: 'inherit', timeout: 120_000 });
+            }
+        },
+        uninstall: async () => {
+            const platform = process.platform;
+            if (platform === 'darwin') {
+                console.log(dim('Uninstalling yt-dlp via Homebrew...'));
+                execSync('brew uninstall yt-dlp', { stdio: 'inherit', timeout: 60_000 });
+            }
+            else {
+                console.log(dim('Uninstalling yt-dlp via pip...'));
+                execSync('pip3 uninstall -y yt-dlp', { stdio: 'inherit', timeout: 60_000 });
+            }
+        },
+        installHint: process.platform === 'darwin' ? 'brew install yt-dlp' : 'pip3 install yt-dlp',
+        uninstallHint: process.platform === 'darwin' ? 'brew uninstall yt-dlp' : 'pip3 uninstall yt-dlp',
+    },
+];
+async function getSkillStatuses() {
+    const results = [];
+    for (const skill of SKILLS) {
+        const installed = await skill.installed();
+        results.push({
+            name: skill.name,
+            description: skill.description,
+            features: skill.features,
+            installed,
+            installHint: skill.installHint,
+        });
+    }
+    return results;
+}
+// ── Commander registration ──────────────────────────────────────────────────
+export function registerSkillCommand(program) {
+    program
+        .command('skill')
+        .description('Manage optional capabilities (browser, yt-dlp)')
+        .option('--install <name>', 'Install a skill')
+        .option('--uninstall <name>', 'Uninstall a skill')
+        .option('--json', 'Output as JSON')
+        .action(async (opts) => {
+        // ── Install mode ──────────────────────────────────────────────────
+        if (opts.install) {
+            const skill = SKILLS.find((s) => s.name === opts.install);
+            if (!skill) {
+                console.error(`Unknown skill: ${opts.install}`);
+                console.error(`Available: ${SKILLS.map((s) => s.name).join(', ')}`);
+                process.exit(1);
+            }
+            const alreadyInstalled = await skill.installed();
+            if (alreadyInstalled) {
+                console.log(green(`✅ ${skill.name} is already installed.`));
+                process.exit(0);
+            }
+            try {
+                await skill.install();
+                const nowInstalled = await skill.installed();
+                if (nowInstalled) {
+                    console.log(green(`✅ ${skill.name} installed successfully.`));
+                }
+                else {
+                    console.log(yellow(`⚠️  Install command ran but ${skill.name} not detected. Try manually: ${skill.installHint}`));
+                }
+            }
+            catch (e) {
+                console.error(red(`❌ Failed to install ${skill.name}: ${e.message}`));
+                console.error(dim(`   Try manually: ${skill.installHint}`));
+                process.exit(1);
+            }
+            process.exit(0);
+        }
+        // ── Uninstall mode ────────────────────────────────────────────────
+        if (opts.uninstall) {
+            const skill = SKILLS.find((s) => s.name === opts.uninstall);
+            if (!skill) {
+                console.error(`Unknown skill: ${opts.uninstall}`);
+                console.error(`Available: ${SKILLS.map((s) => s.name).join(', ')}`);
+                process.exit(1);
+            }
+            const isInstalled = await skill.installed();
+            if (!isInstalled) {
+                console.log(dim(`${skill.name} is not installed.`));
+                process.exit(0);
+            }
+            try {
+                await skill.uninstall();
+                console.log(green(`✅ ${skill.name} uninstalled.`));
+            }
+            catch (e) {
+                console.error(red(`❌ Failed to uninstall ${skill.name}: ${e.message}`));
+                console.error(dim(`   Try manually: ${skill.uninstallHint}`));
+                process.exit(1);
+            }
+            process.exit(0);
+        }
+        // ── List mode (default) ───────────────────────────────────────────
+        const statuses = await getSkillStatuses();
+        if (opts.json) {
+            console.log(JSON.stringify(statuses, null, 2));
+            process.exit(0);
+        }
+        console.log('');
+        console.log(bold('WebPeel Skills'));
+        console.log(dim('Optional capabilities that enhance WebPeel'));
+        console.log('');
+        for (const skill of statuses) {
+            const icon = skill.installed ? green('✅') : yellow('○');
+            const status = skill.installed ? green('installed') : dim('not installed');
+            console.log(`  ${icon} ${bold(skill.name.padEnd(12))} ${status}`);
+            console.log(`     ${skill.description}`);
+            console.log(`     ${dim(`Enables: ${skill.features}`)}`);
+            if (!skill.installed) {
+                console.log(`     ${dim(`Install: webpeel skill --install ${skill.name}`)}`);
+            }
+            console.log('');
+        }
+        console.log(dim('Usage: webpeel skill --install <name> | --uninstall <name>'));
+        console.log('');
+        process.exit(0);
+    });
+}

package/dist/cli/utils.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Shared CLI utilities — config, API client, output formatting, helpers.
+ * Imported by all command modules.
+ */
+import type { PeelResult, PeelEnvelope, PageAction, PeelOptions } from '../types.js';
+export declare const cliVersion: string;
+export declare const VERB_ALIASES: Set<string>;
+export declare function checkForUpdates(): Promise<void>;
+export declare const NO_COLOR: boolean;
+export declare const bold: (s: string) => string;
+export declare const dim: (s: string) => string;
+export declare const cyan: (s: string) => string;
+/**
+ * Parse action strings into PageAction array
+ * Formats:
+ *   click:.selector         — click an element
+ *   type:.selector=text     — type text into an input
+ *   fill:.selector=text     — fill an input (replaces existing value)
+ *   scroll:down:500         — scroll direction + amount
+ *   scroll:bottom           — scroll to bottom (legacy)
+ *   scroll:top              — scroll to top (legacy)
+ *   wait:2000               — wait N ms
+ *   press:Enter             — press a keyboard key
+ *   hover:.selector         — hover over an element
+ *   waitFor:.selector       — wait for a selector to appear
+ *   select:.selector=value  — select dropdown option
+ *   screenshot              — take a screenshot
+ */
+export declare function parseActions(actionStrings: string[]): PageAction[];
+/**
+ * Format an error with actionable suggestions based on error type
+ */
+export declare function formatError(error: Error, _url: string, options: any): string;
+/**
+ * Routes ALL fetch requests through the WebPeel API.
+ * CLI is a pure API client — no local Playwright.
+ */
+export declare function fetchViaApi(url: string, options: PeelOptions, apiKey: string, apiUrl: string): Promise<any>;
+/**
+ * Reconstruct the standard Commander help layout for --help-all and subcommands.
+ * This mirrors Commander's own default formatHelp() so subcommand help keeps working.
+ */
+export declare function buildCommanderHelp(cmd: any, helper: any): string;
+/**
+ * Condensed, Anthropic-style help for the root command (default --help).
+ */
+export declare function buildCondensedHelp(): string;
+/**
+ * Format a past Date relative to now (e.g. "2h ago", "5m ago").
+ */
+export declare function formatRelativeTime(past: Date): string;
+export declare function classifyErrorCode(error: unknown): string;
+export interface OutputExtra {
+    /** Was this result served from the local cache? */
+    cached?: boolean;
+    /** Structured listings data (from --extract-all) */
+    structured?: Record<string, unknown>[];
+    /** Was content distilled/truncated to fit a budget? */
+    truncated?: boolean;
+    /** Total items available before budget limiting (listings only) */
+    totalAvailable?: number;
+}
+/**
+ * Build a unified PeelEnvelope from a PeelResult.
+ *
+ * All existing PeelResult fields are spread first (backward compatibility),
+ * then canonical envelope fields override/extend them.
+ */
+export declare function buildEnvelope(result: PeelResult, extra: OutputExtra): PeelEnvelope & Record<string, unknown>;
+export declare function outputResult(result: PeelResult, options: any, extra?: OutputExtra): Promise<void>;
+export declare function writeStdout(data: string): Promise<void>;
+/**
+ * Convert an array of listing items to CSV.
+ */
+export declare function formatListingsCsv(items: Array<Record<string, string | undefined>>): string;
+/**
+ * Normalise the result of --extract (which may be a flat object or contain
+ * arrays) into an array of row objects suitable for CSV / table rendering.
+ */
+export declare function normaliseExtractedToRows(extracted: Record<string, any>): Array<Record<string, string | undefined>>;
+/** Helper function to extract colors from content */
+export declare function extractColors(content: string): string[];
+/** Helper function to extract font information */
+export declare function extractFonts(content: string): string[];