@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/cli-auth.d.ts

@@ -0,0 +1,75 @@
+/**
+ * CLI Authentication & Usage Tracking
+ *
+ * Handles:
+ * - Anonymous usage (25 free fetches)
+ * - API key authentication
+ * - Usage checking against API
+ * - Config file management (~/.webpeel/config.json)
+ */
+interface CLIConfig {
+    apiKey?: string;
+    /** BYOK key for Brave Search (optional) */
+    braveApiKey?: string;
+    anonymousUsage: number;
+    lastReset: string;
+    planTier?: string;
+    planCachedAt?: string;
+    /** LLM settings for AI-powered features (extract, summarize, agent) */
+    llm?: {
+        apiKey?: string;
+        model?: string;
+        baseUrl?: string;
+    };
+}
+interface UsageCheckResult {
+    allowed: boolean;
+    message?: string;
+    isAnonymous?: boolean;
+    usageInfo?: {
+        used: number;
+        limit: number;
+        remaining: number;
+    };
+}
+/**
+ * Load config from ~/.webpeel/config.json
+ */
+export declare function loadConfig(): CLIConfig;
+/**
+ * Save config to ~/.webpeel/config.json
+ */
+export declare function saveConfig(config: CLIConfig): void;
+/**
+ * Delete config file
+ */
+export declare function deleteConfig(): void;
+/**
+ * Check usage quota before making a request
+ */
+export declare function checkUsage(): Promise<UsageCheckResult>;
+/**
+ * Show usage footer after successful fetch (for free/anonymous users only)
+ */
+export declare function showUsageFooter(usageInfo: {
+    used: number;
+    limit: number;
+    remaining: number;
+} | undefined, isAnonymous: boolean, stealth?: boolean): void;
+/**
+ * Prompt user for API key via stdin
+ */
+export declare function promptForApiKey(): Promise<string>;
+/**
+ * Login command - save API key to config
+ */
+export declare function handleLogin(): Promise<void>;
+/**
+ * Logout command - remove API key from config
+ */
+export declare function handleLogout(): void;
+/**
+ * Usage command - show current quota
+ */
+export declare function handleUsage(): Promise<void>;
+export {};

package/dist/cli-auth.js

@@ -0,0 +1,369 @@
+/**
+ * CLI Authentication & Usage Tracking
+ *
+ * Handles:
+ * - Anonymous usage (25 free fetches)
+ * - API key authentication
+ * - Usage checking against API
+ * - Config file management (~/.webpeel/config.json)
+ */
+import { readFileSync, writeFileSync, existsSync, mkdirSync, unlinkSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+import * as readline from 'readline';
+// Config file location: ~/.webpeel/config.json
+const CONFIG_DIR = join(homedir(), '.webpeel');
+const CONFIG_FILE = join(CONFIG_DIR, 'config.json');
+// API base URL (configurable via env var)
+const API_BASE_URL = process.env.WEBPEEL_API_URL || 'https://api.webpeel.dev';
+/**
+ * Load config from ~/.webpeel/config.json
+ */
+export function loadConfig() {
+    try {
+        if (!existsSync(CONFIG_FILE)) {
+            return {
+                anonymousUsage: 0,
+                lastReset: getLastMonday().toISOString(),
+            };
+        }
+        const content = readFileSync(CONFIG_FILE, 'utf-8');
+        const config = JSON.parse(content);
+        // Ensure lastReset exists
+        if (!config.lastReset) {
+            config.lastReset = getLastMonday().toISOString();
+        }
+        return config;
+    }
+    catch (error) {
+        // If config is corrupted, start fresh
+        return {
+            anonymousUsage: 0,
+            lastReset: getLastMonday().toISOString(),
+        };
+    }
+}
+/**
+ * Save config to ~/.webpeel/config.json
+ */
+export function saveConfig(config) {
+    try {
+        // Ensure directory exists
+        if (!existsSync(CONFIG_DIR)) {
+            mkdirSync(CONFIG_DIR, { recursive: true });
+        }
+        writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2), 'utf-8');
+    }
+    catch (error) {
+        console.error(`Warning: Failed to save config: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Delete config file
+ */
+export function deleteConfig() {
+    try {
+        if (existsSync(CONFIG_FILE)) {
+            unlinkSync(CONFIG_FILE);
+        }
+    }
+    catch (error) {
+        console.error(`Warning: Failed to delete config: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Get the last Monday 00:00 UTC (start of current week)
+ */
+function getLastMonday() {
+    const now = new Date();
+    const day = now.getUTCDay();
+    const diff = day === 0 ? 6 : day - 1; // Days since last Monday
+    const lastMonday = new Date(now);
+    lastMonday.setUTCDate(now.getUTCDate() - diff);
+    lastMonday.setUTCHours(0, 0, 0, 0);
+    return lastMonday;
+}
+/**
+ * Check if anonymous usage counter needs to be reset (weekly reset)
+ */
+function shouldResetAnonymousUsage(config) {
+    const lastReset = new Date(config.lastReset);
+    const currentWeekStart = getLastMonday();
+    return lastReset < currentWeekStart;
+}
+/**
+ * Check usage quota before making a request
+ */
+export async function checkUsage() {
+    const config = loadConfig();
+    // Support WEBPEEL_API_KEY env var (e.g. for CI, admin use, non-interactive)
+    const envApiKey = process.env.WEBPEEL_API_KEY;
+    if (envApiKey && !config.apiKey) {
+        config.apiKey = envApiKey;
+    }
+    // Check if anonymous usage needs reset
+    if (shouldResetAnonymousUsage(config)) {
+        config.anonymousUsage = 0;
+        config.lastReset = getLastMonday().toISOString();
+        saveConfig(config);
+    }
+    // Anonymous user - allow first 25 fetches
+    if (!config.apiKey) {
+        const limit = 25;
+        const used = config.anonymousUsage;
+        const remaining = limit - used;
+        if (used >= limit) {
+            return {
+                allowed: false,
+                message: `You've used your ${limit} free fetches this week.\n\n` +
+                    `📦 Sign up free → 500 fetches/week:  webpeel login\n` +
+                    `⚡ Pro ($9/mo) → 1,250 fetches/week: https://webpeel.dev/pricing\n` +
+                    `🚀 Max ($29/mo) → 6,250 fetches/week\n`,
+            };
+        }
+        // Increment usage counter
+        config.anonymousUsage++;
+        saveConfig(config);
+        return {
+            allowed: true,
+            isAnonymous: true,
+            usageInfo: {
+                used: used + 1,
+                limit,
+                remaining: remaining - 1,
+            },
+        };
+    }
+    // Authenticated user - check with API
+    try {
+        const response = await fetch(`${API_BASE_URL}/v1/cli/usage`, {
+            headers: {
+                'Authorization': `Bearer ${config.apiKey}`,
+            },
+            signal: AbortSignal.timeout(5000),
+        });
+        if (!response.ok) {
+            if (response.status === 401) {
+                return {
+                    allowed: false,
+                    message: `Authentication failed. Your API key may be invalid.\n\nRun: webpeel logout\nThen: webpeel login\n`,
+                };
+            }
+            // If API returns other errors, allow gracefully
+            return { allowed: true };
+        }
+        const data = await response.json();
+        // Check burst limit
+        if (data.burst.used >= data.burst.limit) {
+            return {
+                allowed: false,
+                message: `Burst limit reached (${data.burst.used}/${data.burst.limit}). Resets in ${data.burst.resetsIn}.\nUpgrade: ${data.upgradeUrl}`,
+            };
+        }
+        // Quick canFetch check from API
+        if (!data.canFetch) {
+            return {
+                allowed: false,
+                message: `Weekly limit reached (${data.weekly.used}/${data.weekly.limit}).\nResets: ${data.weekly.resetsAt}\nUpgrade: ${data.upgradeUrl}`,
+            };
+        }
+        // Cache plan tier for offline feature gating
+        const planName = (data.plan?.tier || 'free').toLowerCase();
+        config.planTier = planName;
+        config.planCachedAt = new Date().toISOString();
+        saveConfig(config);
+        return {
+            allowed: true,
+            isAnonymous: false,
+            usageInfo: {
+                used: data.weekly.used,
+                limit: data.weekly.limit,
+                remaining: data.weekly.remaining,
+            },
+        };
+    }
+    catch (error) {
+        // If API is unreachable, allow the request (graceful degradation)
+        return { allowed: true };
+    }
+}
+/**
+ * Show usage footer after successful fetch (for free/anonymous users only)
+ */
+export function showUsageFooter(usageInfo, isAnonymous, stealth = false) {
+    if (!usageInfo)
+        return;
+    // Only show footer for anonymous or free users
+    if (isAnonymous) {
+        const costText = stealth ? ' (costs 5 credits)' : '';
+        console.error(`⚡ ${usageInfo.remaining}/${usageInfo.limit} free fetches remaining${costText} · webpeel login for 500/week free · webpeel.dev/pricing for unlimited`);
+    }
+    else if (usageInfo.limit <= 500) {
+        // Free tier authenticated users — show upgrade CTA
+        const costText = stealth ? ' (costs 5 credits)' : '';
+        const pct = Math.round((usageInfo.used / usageInfo.limit) * 100);
+        if (pct >= 80) {
+            console.error(`⚠️  ${usageInfo.remaining}/${usageInfo.limit} fetches remaining this week${costText}.`);
+            console.error(`   Upgrade to Pro ($9/mo) for 1,250/week → webpeel.dev/pricing`);
+        }
+        else if (pct >= 50) {
+            console.error(`⚡ ${usageInfo.remaining}/${usageInfo.limit} fetches remaining this week${costText}. Upgrade: webpeel.dev/pricing`);
+        }
+        else {
+            console.error(`⚡ ${usageInfo.remaining}/${usageInfo.limit} fetches remaining this week${costText}.`);
+        }
+    }
+    // Don't show footer for paid users
+}
+/**
+ * Prompt user for API key via stdin
+ */
+export async function promptForApiKey() {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    return new Promise((resolve) => {
+        rl.question('Enter your API key (get one at https://app.webpeel.dev/keys): ', (answer) => {
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+}
+/**
+ * Login command - save API key to config
+ */
+export async function handleLogin() {
+    const config = loadConfig();
+    if (config.apiKey) {
+        console.log('You are already logged in.');
+        console.log('Run `webpeel logout` first if you want to use a different API key.');
+        return;
+    }
+    console.log('\n🔑 WebPeel CLI Authentication');
+    console.log('==============================\n');
+    console.log('Get your API key at: https://app.webpeel.dev/keys\n');
+    const apiKey = await promptForApiKey();
+    if (!apiKey) {
+        console.error('Error: API key cannot be empty');
+        process.exit(1);
+    }
+    // Validate API key format (should start with wp_)
+    if (!apiKey.startsWith('wp_')) {
+        console.error('Warning: API key should start with "wp_". Make sure you entered it correctly.');
+    }
+    // Validate API key against server before saving
+    console.log('\nVerifying API key...');
+    try {
+        const response = await fetch(`${API_BASE_URL}/v1/cli/usage`, {
+            headers: { 'Authorization': `Bearer ${apiKey}` },
+            signal: AbortSignal.timeout(5000),
+        });
+        if (response.ok) {
+            const data = await response.json();
+            const tierLabel = data.plan.tier.charAt(0).toUpperCase() + data.plan.tier.slice(1);
+            // Save to config with plan info
+            config.apiKey = apiKey;
+            config.planTier = data.plan.tier;
+            config.planCachedAt = new Date().toISOString();
+            saveConfig(config);
+            console.log(`\n✅ Successfully logged in!`);
+            console.log(`Plan: ${tierLabel} (${data.weekly.limit} fetches/week)`);
+            console.log(`Usage this week: ${data.weekly.used}/${data.weekly.limit}`);
+        }
+        else if (response.status === 401) {
+            console.error('\n❌ Invalid API key. Please check and try again.');
+            console.error('Get your API key at https://app.webpeel.dev/keys');
+            process.exit(1);
+        }
+        else {
+            // Server returned non-401 error — save key anyway (might be temporary)
+            config.apiKey = apiKey;
+            saveConfig(config);
+            console.log('\n✓ API key saved (could not verify — server may be temporarily unavailable).');
+        }
+    }
+    catch {
+        // Network error — save key anyway (graceful)
+        config.apiKey = apiKey;
+        saveConfig(config);
+        console.log('\n✓ API key saved (could not reach server to verify).');
+    }
+    console.log('Run `webpeel usage` to check your quota.');
+}
+/**
+ * Logout command - remove API key from config
+ */
+export function handleLogout() {
+    const config = loadConfig();
+    if (!config.apiKey) {
+        console.log('You are not logged in.');
+        return;
+    }
+    deleteConfig();
+    console.log('✓ Logged out successfully');
+}
+/**
+ * Usage command - show current quota
+ */
+export async function handleUsage() {
+    const config = loadConfig();
+    // Check for weekly reset
+    if (shouldResetAnonymousUsage(config)) {
+        config.anonymousUsage = 0;
+        config.lastReset = getLastMonday().toISOString();
+        saveConfig(config);
+    }
+    // Anonymous user
+    if (!config.apiKey) {
+        const limit = 25;
+        const used = config.anonymousUsage;
+        const remaining = limit - used;
+        const nextMonday = new Date(getLastMonday());
+        nextMonday.setUTCDate(nextMonday.getUTCDate() + 7);
+        console.log('\nWebPeel Usage (Anonymous)');
+        console.log('=========================\n');
+        console.log(`Plan: Anonymous (25 free fetches/week)`);
+        console.log(`Used this week: ${used}/${limit}`);
+        console.log(`Remaining: ${remaining}`);
+        console.log(`Resets: ${nextMonday.toUTCString()}`);
+        console.log('\n💡 Run `webpeel login` to get 500 fetches/week for free!');
+        console.log('   Or sign up at https://app.webpeel.dev/signup\n');
+        return;
+    }
+    // Authenticated user - fetch from API
+    try {
+        const response = await fetch(`${API_BASE_URL}/v1/cli/usage`, {
+            headers: {
+                'Authorization': `Bearer ${config.apiKey}`,
+            },
+            signal: AbortSignal.timeout(5000),
+        });
+        if (!response.ok) {
+            if (response.status === 401) {
+                console.error('Error: Authentication failed. Your API key may be invalid.');
+                console.error('Run `webpeel logout` and `webpeel login` to re-authenticate.');
+                process.exit(1);
+            }
+            throw new Error(`API returned status ${response.status}`);
+        }
+        const data = await response.json();
+        const tierLabel = data.plan.tier.charAt(0).toUpperCase() + data.plan.tier.slice(1);
+        console.log('\nWebPeel Usage');
+        console.log('=============\n');
+        console.log(`Plan: ${tierLabel} (${data.weekly.limit}/week)`);
+        console.log(`Used this week: ${data.weekly.used}/${data.weekly.limit} (${data.weekly.percentUsed}%)`);
+        console.log(`Remaining: ${data.weekly.remaining}`);
+        console.log(`Burst: ${data.burst.used}/${data.burst.limit} this hour (resets in ${data.burst.resetsIn})`);
+        console.log(`Weekly reset: ${new Date(data.weekly.resetsAt).toUTCString()}`);
+        if (data.weekly.remaining <= 10) {
+            console.log(`\n⚠️  Running low on credits. Upgrade at ${data.upgradeUrl}`);
+        }
+        console.log();
+    }
+    catch (error) {
+        console.error(`Error fetching usage data: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        console.error('The API may be temporarily unavailable. Try again later.');
+        process.exit(1);
+    }
+}

package/dist/cli.d.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+/**
+ * WebPeel CLI — Entry point
+ *
+ * Registers all command groups and starts the Commander program.
+ * The heavy implementation lives in src/cli/commands/*.ts
+ *
+ * Usage:
+ *   npx webpeel <url>                  - Fetch and convert to markdown
+ *   npx webpeel <url> --json           - Output as JSON
+ *   npx webpeel <url> --render         - Force browser mode
+ *   npx webpeel search "query"         - DuckDuckGo search
+ *   npx webpeel mcp                    - Start MCP server
+ *   npx webpeel --help                 - Condensed help
+ *   npx webpeel --help-all             - Full option reference
+ */
+export {};

package/dist/cli.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * WebPeel CLI — Entry point
+ *
+ * Registers all command groups and starts the Commander program.
+ * The heavy implementation lives in src/cli/commands/*.ts
+ *
+ * Usage:
+ *   npx webpeel <url>                  - Fetch and convert to markdown
+ *   npx webpeel <url> --json           - Output as JSON
+ *   npx webpeel <url> --render         - Force browser mode
+ *   npx webpeel search "query"         - DuckDuckGo search
+ *   npx webpeel mcp                    - Start MCP server
+ *   npx webpeel --help                 - Condensed help
+ *   npx webpeel --help-all             - Full option reference
+ */
+// ── Auto-load .env from cwd (lightweight, no dotenv dependency) ──────────────
+// Must happen BEFORE any imports that read env vars (e.g., WEBPEEL_API_KEY)
+import { readFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+{
+    const envPath = resolve(process.cwd(), '.env');
+    if (existsSync(envPath)) {
+        for (const line of readFileSync(envPath, 'utf-8').split('\n')) {
+            const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
+            if (m && !process.env[m[1]])
+                process.env[m[1]] = m[2].replace(/^["']|["']$/g, '');
+        }
+    }
+}
+import { Command } from 'commander';
+import { VERB_ALIASES, cliVersion, checkForUpdates, buildCommanderHelp, buildCondensedHelp, } from './cli/utils.js';
+import { registerFetchCommands } from './cli/commands/fetch.js';
+import { registerSearchCommands } from './cli/commands/search.js';
+import { registerInteractCommands } from './cli/commands/interact.js';
+import { registerAuthCommands } from './cli/commands/auth.js';
+import { registerScreenshotCommands } from './cli/commands/screenshot.js';
+import { registerJobsCommands } from './cli/commands/jobs.js';
+import { registerMonitorCommands } from './cli/commands/monitor.js';
+import { registerGuideCommand } from './cli/commands/guide.js';
+import { registerDoctorCommand } from './cli/commands/doctor.js';
+import { registerSetupCommand } from './cli/commands/setup.js';
+import { registerSkillCommand } from './cli/commands/skill.js';
+import { registerObserveCommand } from './cli/commands/observe.js';
+// ── Early silent/log-level detection (must happen before any async module code) ──
+// Set WEBPEEL_LOG_LEVEL early so logger checks see it when async IIFEs fire.
+if (!process.env.WEBPEEL_LOG_LEVEL && process.argv.includes('--silent')) {
+    process.env.WEBPEEL_LOG_LEVEL = 'silent';
+}
+// ── Verb alias intercept (before Commander parses) ────────────────────────────
+// "webpeel fetch <url>" → "webpeel <url>"
+// Note: 'read' is intentionally excluded — it's a registered subcommand.
+if (process.argv.length >= 3 && VERB_ALIASES.has(process.argv[2]?.toLowerCase())) {
+    process.argv.splice(2, 1);
+}
+// ── --help-all detection (must happen before Commander parses) ────────────────
+const isHelpAll = process.argv.slice(2).some(a => a === '--help-all');
+if (isHelpAll) {
+    const idx = process.argv.indexOf('--help-all');
+    if (idx !== -1)
+        process.argv[idx] = '--help';
+}
+// ── Program setup ─────────────────────────────────────────────────────────────
+const program = new Command();
+program
+    .name('webpeel')
+    .description('Fast web fetcher for AI agents')
+    .version(cliVersion)
+    .enablePositionalOptions();
+// ── Help formatting ───────────────────────────────────────────────────────────
+program.configureHelp({
+    sortSubcommands: true,
+    showGlobalOptions: false,
+    formatHelp: (cmd, helper) => {
+        if (cmd.parent !== null || isHelpAll) {
+            return buildCommanderHelp(cmd, helper);
+        }
+        return buildCondensedHelp();
+    },
+});
+// ── Update check (non-blocking, background) ───────────────────────────────────
+void checkForUpdates();
+// ── Register all command groups ───────────────────────────────────────────────
+// Doctor, setup, and other subcommands must register BEFORE fetch,
+// because fetch registers a catch-all `[url]` default argument.
+registerDoctorCommand(program);
+registerSetupCommand(program);
+registerSkillCommand(program);
+registerSearchCommands(program);
+registerInteractCommands(program);
+registerAuthCommands(program);
+registerScreenshotCommands(program);
+registerJobsCommands(program);
+registerMonitorCommands(program);
+registerObserveCommand(program);
+registerGuideCommand(program);
+registerFetchCommands(program);
+// ── Parse ─────────────────────────────────────────────────────────────────────
+program.parse();

package/dist/core/actions.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Page actions executor for browser automation
+ *
+ * This is WebPeel's "Actions API" — click/scroll/type/wait before extracting.
+ *
+ * Timeouts:
+ * - Default per action: 5s
+ * - Max total across all actions: 30s
+ */
+import type { Page } from 'playwright';
+import type { PageAction } from '../types.js';
+export declare const DEFAULT_ACTION_TIMEOUT_MS = 5000;
+export declare const MAX_TOTAL_ACTIONS_MS = 30000;
+export interface AutoScrollOptions {
+    /** Maximum number of scroll iterations (default: 20) */
+    maxScrolls?: number;
+    /** Milliseconds to wait between scrolls (default: 1000) */
+    scrollDelay?: number;
+    /** Total timeout in milliseconds (default: 30000) */
+    timeout?: number;
+    /** Optional: wait for this CSS selector after each scroll */
+    waitForSelector?: string;
+}
+export interface AutoScrollResult {
+    /** Number of scroll iterations performed */
+    scrollCount: number;
+    /** Final document height in pixels */
+    finalHeight: number;
+    /** Whether the page content grew during scrolling */
+    contentGrew: boolean;
+    /** Whether a virtual/inner scrollable container was found and used */
+    scrollContainerFound?: boolean;
+    /** Total number of DOM mutations detected during scrolling */
+    mutationsDetected?: number;
+}
+/**
+ * Normalize a raw actions array to WebPeel's internal PageAction shape.
+ * Accepts Firecrawl-style fields (milliseconds, text, direction/amount).
+ */
+export declare function normalizeActions(input?: unknown): PageAction[] | undefined;
+/**
+ * Detect the most likely scrollable container on the page.
+ * Returns a CSS selector string for the container, or null if only window scrolling is needed.
+ *
+ * Looks for elements with overflow-y: auto|scroll whose scrollHeight > clientHeight,
+ * preferring the largest such element. Used by autoScroll and scrollThrough.
+ */
+export declare function detectScrollContainer(page: Page): Promise<string | null>;
+/**
+ * Intelligently scroll the page to load all lazy/infinite-scroll content.
+ *
+ * Improvements over the basic version:
+ * 1. Detects virtual/inner scroll containers (Polymarket, React virtualized lists)
+ * 2. Uses MutationObserver to detect DOM additions (not just height changes)
+ * 3. Gracefully handles execution context destruction (SPA navigation)
+ * 4. Stability requires BOTH no height change AND no DOM mutations
+ *
+ * Stops when:
+ * - Height is stable AND no DOM mutations for 2 consecutive checks
+ * - maxScrolls limit is reached
+ * - Total timeout is exceeded
+ * - Execution context is destroyed (SPA navigation)
+ */
+export declare function autoScroll(page: Page, options?: AutoScrollOptions): Promise<AutoScrollResult>;
+export declare function executeActions(page: Page, actions: PageAction[], screenshotOptions?: {
+    fullPage?: boolean;
+    type?: 'png' | 'jpeg';
+    quality?: number;
+}): Promise<Buffer | undefined>;