@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/actions.js

@@ -0,0 +1,495 @@
+/**
+ * Page actions executor for browser automation
+ *
+ * This is WebPeel's "Actions API" — click/scroll/type/wait before extracting.
+ *
+ * Timeouts:
+ * - Default per action: 5s
+ * - Max total across all actions: 30s
+ */
+import { TimeoutError, WebPeelError } from '../types.js';
+export const DEFAULT_ACTION_TIMEOUT_MS = 5_000;
+export const MAX_TOTAL_ACTIONS_MS = 30_000;
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+async function withTimeout(promise, ms, message) {
+    if (!Number.isFinite(ms) || ms <= 0) {
+        throw new TimeoutError(message);
+    }
+    return Promise.race([
+        promise,
+        new Promise((_, reject) => setTimeout(() => reject(new TimeoutError(message)), ms)),
+    ]);
+}
+/**
+ * Normalize a raw actions array to WebPeel's internal PageAction shape.
+ * Accepts Firecrawl-style fields (milliseconds, text, direction/amount).
+ */
+export function normalizeActions(input) {
+    if (!input)
+        return undefined;
+    if (!Array.isArray(input))
+        throw new WebPeelError('Invalid actions: must be an array');
+    return input.map((raw) => {
+        if (!raw || typeof raw !== 'object')
+            throw new WebPeelError('Invalid action: must be an object');
+        if (typeof raw.type !== 'string')
+            throw new WebPeelError('Invalid action: missing type');
+        const type = raw.type;
+        // Common aliases
+        const selector = typeof raw.selector === 'string' ? raw.selector : undefined;
+        const timeout = typeof raw.timeout === 'number' ? raw.timeout : undefined;
+        switch (type) {
+            case 'wait': {
+                const ms = typeof raw.milliseconds === 'number'
+                    ? raw.milliseconds
+                    : typeof raw.ms === 'number'
+                        ? raw.ms
+                        : typeof raw.value === 'number'
+                            ? raw.value
+                            : undefined;
+                return {
+                    type: 'wait',
+                    ms: ms ?? 1000,
+                    timeout,
+                };
+            }
+            case 'click':
+                return { type: 'click', selector, timeout };
+            case 'type':
+            case 'fill': {
+                const value = typeof raw.value === 'string' ? raw.value
+                    : typeof raw.text === 'string' ? raw.text
+                        : undefined;
+                return { type, selector, value, timeout };
+            }
+            case 'select': {
+                const value = typeof raw.value === 'string' ? raw.value : undefined;
+                return { type: 'select', selector, value, timeout };
+            }
+            case 'press': {
+                const key = typeof raw.key === 'string' ? raw.key : (typeof raw.value === 'string' ? raw.value : undefined);
+                return { type: 'press', key, timeout };
+            }
+            case 'hover':
+                return { type: 'hover', selector, timeout };
+            case 'waitForSelector':
+                return { type: 'waitForSelector', selector, timeout };
+            case 'scroll': {
+                const direction = typeof raw.direction === 'string' ? raw.direction : undefined;
+                const amount = typeof raw.amount === 'number' ? raw.amount : undefined;
+                // Legacy/internal: to can be top|bottom|number|{x,y}
+                let to;
+                if (raw.to === 'top' || raw.to === 'bottom' || typeof raw.to === 'number') {
+                    to = raw.to;
+                }
+                else if (typeof raw.to === 'object' && raw.to !== null && 'x' in raw.to && 'y' in raw.to) {
+                    to = { x: raw.to.x, y: raw.to.y };
+                }
+                else {
+                    to = undefined;
+                }
+                return {
+                    type: 'scroll',
+                    direction: (direction === 'up' || direction === 'down' || direction === 'left' || direction === 'right') ? direction : undefined,
+                    amount,
+                    to,
+                    timeout,
+                };
+            }
+            case 'screenshot':
+                return { type: 'screenshot', timeout };
+            default:
+                // Allow forward compatibility — but still pass through known fields.
+                return { ...raw };
+        }
+    });
+}
+/**
+ * Check if an error message indicates the execution context was destroyed.
+ * This happens on SPAs (like Polymarket) when scrolling triggers navigation.
+ */
+function isContextDestroyedError(err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return (msg.includes('Execution context was destroyed') ||
+        msg.includes('Target closed') ||
+        msg.includes('frame was detached') ||
+        msg.includes('Session closed'));
+}
+/**
+ * Detect the most likely scrollable container on the page.
+ * Returns a CSS selector string for the container, or null if only window scrolling is needed.
+ *
+ * Looks for elements with overflow-y: auto|scroll whose scrollHeight > clientHeight,
+ * preferring the largest such element. Used by autoScroll and scrollThrough.
+ */
+export async function detectScrollContainer(page) {
+    try {
+        return await page.evaluate(() => {
+            const candidates = [];
+            const elements = document.querySelectorAll('*');
+            for (const el of Array.from(elements)) {
+                // Skip body/html (that's the default window scroll)
+                if (el === document.body || el === document.documentElement)
+                    continue;
+                const style = window.getComputedStyle(el);
+                const overflowY = style.overflowY;
+                if (overflowY !== 'auto' && overflowY !== 'scroll')
+                    continue;
+                const scrollable = el.scrollHeight - el.clientHeight;
+                if (scrollable < 100)
+                    continue; // Must have meaningful scrollable space
+                // Build a reasonable selector
+                let selector = el.tagName.toLowerCase();
+                if (el.id) {
+                    selector = `#${CSS.escape(el.id)}`;
+                }
+                else if (el.className && typeof el.className === 'string') {
+                    const cls = el.className.trim().split(/\s+/)[0];
+                    if (cls)
+                        selector = `${el.tagName.toLowerCase()}.${CSS.escape(cls)}`;
+                }
+                candidates.push({ selector, scrollable });
+            }
+            if (candidates.length === 0)
+                return null;
+            // Sort by most scrollable content and return the best candidate
+            candidates.sort((a, b) => b.scrollable - a.scrollable);
+            return candidates[0].selector;
+        });
+    }
+    catch {
+        // Context may be gone — safe to ignore
+        return null;
+    }
+}
+/**
+ * Intelligently scroll the page to load all lazy/infinite-scroll content.
+ *
+ * Improvements over the basic version:
+ * 1. Detects virtual/inner scroll containers (Polymarket, React virtualized lists)
+ * 2. Uses MutationObserver to detect DOM additions (not just height changes)
+ * 3. Gracefully handles execution context destruction (SPA navigation)
+ * 4. Stability requires BOTH no height change AND no DOM mutations
+ *
+ * Stops when:
+ * - Height is stable AND no DOM mutations for 2 consecutive checks
+ * - maxScrolls limit is reached
+ * - Total timeout is exceeded
+ * - Execution context is destroyed (SPA navigation)
+ */
+export async function autoScroll(page, options = {}) {
+    const { maxScrolls = 20, scrollDelay = 1000, timeout = 30_000, waitForSelector, } = options;
+    const startTime = Date.now();
+    let scrollCount = 0;
+    let stableCount = 0;
+    const stableThreshold = 2;
+    let totalMutations = 0;
+    let scrollContainerFound = false;
+    // ── Step 1: Detect the actual scrollable container ─────────────────────────
+    // Use || null to normalize undefined (returned by test mocks) to null.
+    const containerSelector = (await detectScrollContainer(page)) || null;
+    scrollContainerFound = containerSelector !== null;
+    // ── Step 2: Get initial height (body or container) ─────────────────────────
+    const getHeight = async () => {
+        try {
+            if (containerSelector) {
+                const h = await page.evaluate((sel) => {
+                    const el = document.querySelector(sel);
+                    return el ? el.scrollHeight : document.body.scrollHeight;
+                }, containerSelector);
+                return h || 0;
+            }
+            // Use string form for backward compatibility with test mocks
+            return (await page.evaluate('document.body.scrollHeight')) || 0;
+        }
+        catch (err) {
+            if (isContextDestroyedError(err))
+                return -1; // Sentinel: context gone
+            throw err;
+        }
+    };
+    // ── Step 3: Set up MutationObserver inside the page ─────────────────────────
+    // We store a mutation count in window.__wpMutationCount so we can poll it.
+    const setupObserver = async () => {
+        try {
+            await page.evaluate(() => {
+                window.__wpMutationCount = 0;
+                const observer = new MutationObserver((mutations) => {
+                    window.__wpMutationCount += mutations.reduce((sum, m) => sum + m.addedNodes.length, 0);
+                });
+                observer.observe(document.body, { childList: true, subtree: true });
+                window.__wpMutationObserver = observer;
+            });
+        }
+        catch {
+            // If we can't set up the observer, we'll fall back to height-only detection
+        }
+    };
+    const getMutationCount = async () => {
+        try {
+            const count = await page.evaluate(() => window.__wpMutationCount ?? 0);
+            return typeof count === 'number' ? count : 0;
+        }
+        catch {
+            return 0;
+        }
+    };
+    const resetMutationCount = async () => {
+        try {
+            await page.evaluate(() => { window.__wpMutationCount = 0; });
+        }
+        catch {
+            // Ignore
+        }
+    };
+    // ── Step 4: Perform the scroll function ────────────────────────────────────
+    const scrollToBottom = async () => {
+        try {
+            if (containerSelector) {
+                await page.evaluate((sel) => {
+                    const el = document.querySelector(sel);
+                    if (el) {
+                        el.scrollTop = el.scrollHeight;
+                    }
+                    else {
+                        window.scrollTo(0, document.body.scrollHeight);
+                    }
+                }, containerSelector);
+            }
+            else {
+                // Use string form for backward compatibility with test mocks
+                await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+            }
+        }
+        catch (err) {
+            if (!isContextDestroyedError(err))
+                throw err;
+            // Context destroyed — will be caught in main loop
+        }
+    };
+    // ── Main loop ──────────────────────────────────────────────────────────────
+    await setupObserver();
+    await resetMutationCount();
+    const initialHeight = await getHeight();
+    if (initialHeight === -1) {
+        // Context was already destroyed before we started
+        return { scrollCount: 0, finalHeight: 0, contentGrew: false, scrollContainerFound, mutationsDetected: 0 };
+    }
+    let lastHeight = initialHeight;
+    let finalHeight = initialHeight;
+    let lastMutationCount = 0;
+    while (scrollCount < maxScrolls) {
+        // Check timeout
+        if (Date.now() - startTime >= timeout) {
+            break;
+        }
+        // Scroll to bottom (with error recovery)
+        try {
+            await scrollToBottom();
+        }
+        catch (err) {
+            if (isContextDestroyedError(err)) {
+                // SPA navigation destroyed the context — stop gracefully
+                break;
+            }
+            throw err;
+        }
+        scrollCount++;
+        // Wait for new content
+        const remainingTime = timeout - (Date.now() - startTime);
+        const waitMs = Math.min(scrollDelay, Math.max(remainingTime, 0));
+        if (waitMs > 0) {
+            await sleep(waitMs);
+        }
+        // Optionally wait for a specific selector
+        if (waitForSelector) {
+            const selectorTimeout = Math.max(0, timeout - (Date.now() - startTime));
+            if (selectorTimeout > 0) {
+                await page.waitForSelector(waitForSelector, { timeout: selectorTimeout }).catch(() => { });
+            }
+        }
+        // Check height (with error recovery)
+        let currentHeight;
+        try {
+            currentHeight = await getHeight();
+            if (currentHeight === -1)
+                break; // Context destroyed
+        }
+        catch (err) {
+            if (isContextDestroyedError(err))
+                break;
+            throw err;
+        }
+        finalHeight = currentHeight;
+        // Check mutation count
+        const currentMutations = await getMutationCount();
+        const mutationsSinceLastCheck = currentMutations - lastMutationCount;
+        totalMutations += mutationsSinceLastCheck;
+        // Stability check: stable = no height growth AND no new DOM mutations
+        const heightStable = currentHeight <= lastHeight;
+        const mutationsStable = mutationsSinceLastCheck === 0;
+        if (heightStable && mutationsStable) {
+            stableCount++;
+            if (stableCount >= stableThreshold) {
+                break;
+            }
+        }
+        else {
+            stableCount = 0;
+            if (!heightStable)
+                lastHeight = currentHeight;
+            lastMutationCount = currentMutations;
+            await resetMutationCount();
+            lastMutationCount = 0;
+        }
+    }
+    return {
+        scrollCount,
+        finalHeight,
+        contentGrew: finalHeight > initialHeight,
+        scrollContainerFound,
+        mutationsDetected: totalMutations,
+    };
+}
+export async function executeActions(page, actions, screenshotOptions) {
+    let lastScreenshot;
+    const screenshotType = screenshotOptions?.type || 'png';
+    const screenshotFullPage = screenshotOptions?.fullPage ?? true;
+    const screenshotQuality = screenshotOptions?.quality;
+    const start = Date.now();
+    const deadline = start + MAX_TOTAL_ACTIONS_MS;
+    // Normalize once to handle Firecrawl-style aliases even if caller didn't.
+    const normalized = normalizeActions(actions) ?? [];
+    for (let i = 0; i < normalized.length; i++) {
+        const action = normalized[i];
+        const remainingTotal = deadline - Date.now();
+        if (remainingTotal <= 0) {
+            throw new TimeoutError(`Actions timed out after ${MAX_TOTAL_ACTIONS_MS}ms`);
+        }
+        const perActionTimeout = Math.min(typeof action.timeout === 'number' && action.timeout > 0 ? action.timeout : DEFAULT_ACTION_TIMEOUT_MS, remainingTotal);
+        const label = `Action ${i + 1}/${normalized.length} (${action.type})`;
+        switch (action.type) {
+            case 'wait': {
+                const ms = (typeof action.ms === 'number' ? action.ms : undefined)
+                    ?? (typeof action.milliseconds === 'number' ? action.milliseconds : undefined)
+                    ?? 1000;
+                const waitMs = Math.min(Math.max(ms, 0), remainingTotal);
+                await withTimeout(page.waitForTimeout(waitMs), waitMs + 50, `${label} timed out`);
+                break;
+            }
+            case 'click': {
+                if (!action.selector)
+                    throw new WebPeelError('click action requires selector');
+                await withTimeout(page.click(action.selector, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'scroll': {
+                const dir = action.direction;
+                const amount = typeof action.amount === 'number' ? action.amount : undefined;
+                const scrollPromise = (async () => {
+                    // Relative scroll (Firecrawl-style)
+                    if (dir && amount !== undefined) {
+                        const a = Math.max(0, amount);
+                        let dx = 0;
+                        let dy = 0;
+                        if (dir === 'down')
+                            dy = a;
+                        if (dir === 'up')
+                            dy = -a;
+                        if (dir === 'right')
+                            dx = a;
+                        if (dir === 'left')
+                            dx = -a;
+                        await page.evaluate(`window.scrollBy(${dx}, ${dy})`);
+                        return;
+                    }
+                    // Legacy absolute scroll target
+                    if (action.to === 'bottom') {
+                        await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+                        return;
+                    }
+                    if (action.to === 'top') {
+                        await page.evaluate('window.scrollTo(0, 0)');
+                        return;
+                    }
+                    if (typeof action.to === 'number') {
+                        await page.evaluate(`window.scrollTo(0, ${action.to})`);
+                        return;
+                    }
+                    if (typeof action.to === 'object' && action.to !== null && 'x' in action.to && 'y' in action.to) {
+                        await page.evaluate(`window.scrollTo(${action.to.x}, ${action.to.y})`);
+                        return;
+                    }
+                    // Default: scroll to bottom
+                    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+                })();
+                await withTimeout(scrollPromise, perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'type': {
+                if (!action.selector)
+                    throw new WebPeelError('type action requires selector');
+                const value = action.value ?? action.text;
+                if (!value)
+                    throw new WebPeelError('type action requires text');
+                await withTimeout(page.type(action.selector, value, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'fill': {
+                if (!action.selector)
+                    throw new WebPeelError('fill action requires selector');
+                const value = action.value ?? action.text;
+                if (!value)
+                    throw new WebPeelError('fill action requires value');
+                await withTimeout(page.fill(action.selector, value, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'select': {
+                if (!action.selector)
+                    throw new WebPeelError('select action requires selector');
+                if (!action.value)
+                    throw new WebPeelError('select action requires value');
+                await withTimeout(page.selectOption(action.selector, action.value, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'press': {
+                const key = action.key;
+                if (!key)
+                    throw new WebPeelError('press action requires key');
+                await withTimeout(page.keyboard.press(key), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'hover': {
+                if (!action.selector)
+                    throw new WebPeelError('hover action requires selector');
+                await withTimeout(page.hover(action.selector, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'waitForSelector': {
+                if (!action.selector)
+                    throw new WebPeelError('waitForSelector action requires selector');
+                await withTimeout(page.waitForSelector(action.selector, { timeout: perActionTimeout }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            case 'screenshot': {
+                lastScreenshot = await withTimeout(page.screenshot({
+                    fullPage: screenshotFullPage,
+                    type: screenshotType,
+                    ...(screenshotType === 'jpeg' && typeof screenshotQuality === 'number'
+                        ? { quality: screenshotQuality }
+                        : {}),
+                }), perActionTimeout + 50, `${label} timed out`);
+                break;
+            }
+            default: {
+                // This should not happen due to our type union, but keep a safe fallback.
+                throw new WebPeelError(`Unknown action type: ${action.type}`);
+            }
+        }
+        // Small yield to avoid starving the event loop in tight action sequences
+        await sleep(0);
+    }
+    return lastScreenshot;
+}

package/dist/core/agent.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Autonomous web research agent
+ * Searches the web, fetches pages, and extracts structured data based on natural language prompts
+ *
+ * Supports:
+ * - depth: "basic" (1 search, top 3) vs "thorough" (multi-step, up to 3 searches, top 10)
+ * - maxSources: control how many sources to include (default 5, max 20)
+ * - topic: "general" | "news" | "technical" | "academic" — adjusts queries & prioritization
+ * - outputSchema: JSON Schema for structured output with validation
+ * - streaming callbacks for SSE support
+ */
+export type AgentDepth = 'basic' | 'thorough';
+export type AgentTopic = 'general' | 'news' | 'technical' | 'academic';
+export interface AgentOptions {
+    /** Natural language description of what data to extract */
+    prompt: string;
+    /** Optional URLs to start from */
+    urls?: string[];
+    /** JSON schema for structured output (legacy — prefer outputSchema) */
+    schema?: Record<string, any>;
+    /** JSON Schema for structured output with validation */
+    outputSchema?: Record<string, any>;
+    /** LLM API key (BYOK - bring your own key) */
+    llmApiKey: string;
+    /** LLM API base URL (default: OpenAI) */
+    llmApiBase?: string;
+    /** LLM model (default: gpt-4o-mini) */
+    llmModel?: string;
+    /** Max pages to visit (default: 10) — legacy param */
+    maxPages?: number;
+    /** Max sources to include (default 5, max 20) */
+    maxSources?: number;
+    /** Research depth: "basic" or "thorough" */
+    depth?: AgentDepth;
+    /** Topic filter */
+    topic?: AgentTopic;
+    /** Max credits/cost to spend */
+    maxCredits?: number;
+    /** Progress callback (legacy — still supported) */
+    onProgress?: (progress: AgentProgress) => void;
+    /** Streaming event callback for SSE */
+    onEvent?: (event: AgentStreamEvent) => void;
+}
+export interface AgentProgress {
+    status: 'searching' | 'visiting' | 'extracting' | 'done';
+    currentUrl?: string;
+    pagesVisited: number;
+    message: string;
+}
+export interface AgentResult {
+    success: boolean;
+    data: any;
+    /** The synthesised answer (text). Only present when no outputSchema. */
+    answer?: string;
+    sources: string[];
+    sourcesDetailed?: Array<{
+        url: string;
+        title: string;
+    }>;
+    pagesVisited: number;
+    creditsUsed: number;
+    tokensUsed?: {
+        input: number;
+        output: number;
+    };
+}
+/** Events emitted during streaming */
+export type AgentStreamEvent = {
+    type: 'step';
+    action: 'searching';
+    query: string;
+} | {
+    type: 'step';
+    action: 'fetching';
+    url: string;
+} | {
+    type: 'step';
+    action: 'analyzing';
+    summary: string;
+} | {
+    type: 'chunk';
+    text: string;
+} | {
+    type: 'done';
+    answer: string;
+    sources: Array<{
+        url: string;
+        title: string;
+    }>;
+    tokensUsed: {
+        input: number;
+        output: number;
+    };
+};
+/**
+ * Run autonomous web research agent
+ */
+export declare function runAgent(options: AgentOptions): Promise<AgentResult>;