@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/engine-ranker.js

@@ -0,0 +1,395 @@
+/**
+ * Engine Quality-Ranked Fallback System
+ *
+ * Ranks extraction strategies by quality for a given URL, producing a
+ * dynamic fallback chain. Inspired by Firecrawl's engine cascade approach
+ * but tailored to WebPeel's architecture.
+ *
+ * Usage:
+ * ```ts
+ * import { buildFallbackChain } from './engine-ranker.js';
+ * const chain = buildFallbackChain('https://twitter.com/user', { render: true });
+ * // Returns engines sorted by quality, with domain-specific adjustments
+ * ```
+ *
+ * @module engine-ranker
+ */
+// ---------------------------------------------------------------------------
+// Default engine configurations
+// ---------------------------------------------------------------------------
+/**
+ * Baseline configuration for each engine type.
+ *
+ * Quality reflects extraction fidelity (not speed or cost):
+ * - domain-api:      Best quality — structured data direct from source APIs
+ * - browser:         High quality — full JS rendering captures dynamic content
+ * - stealth:         Good quality — same as browser but with anti-bot bypass
+ * - cloaked:         Decent quality — heavy stealth, sometimes degrades content
+ * - simple:          Moderate — plain HTTP fetch, misses JS-rendered content
+ * - search-fallback: Low — cached/snippet data from search engine caches
+ */
+const ENGINE_DEFAULTS = {
+    'domain-api': {
+        type: 'domain-api',
+        quality: 95,
+        speed: 95,
+        cost: 5,
+        maxTimeoutMs: 5000,
+        features: { javascript: false, antibot: false, screenshots: false, stealth: false },
+    },
+    'simple': {
+        type: 'simple',
+        quality: 70,
+        speed: 90,
+        cost: 10,
+        maxTimeoutMs: 8000,
+        features: { javascript: false, antibot: false, screenshots: false, stealth: false },
+    },
+    'browser': {
+        type: 'browser',
+        quality: 85,
+        speed: 40,
+        cost: 60,
+        maxTimeoutMs: 15000,
+        features: { javascript: true, antibot: false, screenshots: true, stealth: false },
+    },
+    'stealth': {
+        type: 'stealth',
+        quality: 80,
+        speed: 30,
+        cost: 80,
+        maxTimeoutMs: 20000,
+        features: { javascript: true, antibot: true, screenshots: true, stealth: true },
+    },
+    'cloaked': {
+        type: 'cloaked',
+        quality: 75,
+        speed: 20,
+        cost: 90,
+        maxTimeoutMs: 25000,
+        features: { javascript: true, antibot: true, screenshots: true, stealth: true },
+    },
+    'search-fallback': {
+        type: 'search-fallback',
+        quality: 40,
+        speed: 50,
+        cost: 30,
+        maxTimeoutMs: 10000,
+        features: { javascript: false, antibot: false, screenshots: false, stealth: false },
+    },
+};
+/**
+ * Domain rules that adjust engine scores for known site categories.
+ *
+ * Patterns use suffix matching: "twitter.com" matches both "twitter.com"
+ * and "www.twitter.com" but not "nottwitter.com".
+ */
+const DOMAIN_RULES = [
+    // ── Social media: heavy JS, aggressive anti-bot ──────────────────────
+    {
+        pattern: 'twitter.com',
+        overrides: {
+            'simple': { quality: 20, speed: 95 },
+            'stealth': { quality: 90 },
+            'cloaked': { quality: 85 },
+            'browser': { quality: 80 },
+        },
+    },
+    {
+        pattern: 'x.com',
+        overrides: {
+            'simple': { quality: 20, speed: 95 },
+            'stealth': { quality: 90 },
+            'cloaked': { quality: 85 },
+            'browser': { quality: 80 },
+        },
+    },
+    {
+        pattern: 'instagram.com',
+        overrides: {
+            'simple': { quality: 15 },
+            'stealth': { quality: 90 },
+            'cloaked': { quality: 88 },
+            'browser': { quality: 75 },
+        },
+    },
+    {
+        pattern: 'tiktok.com',
+        overrides: {
+            'simple': { quality: 15 },
+            'stealth': { quality: 90 },
+            'cloaked': { quality: 88 },
+            'browser': { quality: 70 },
+        },
+    },
+    {
+        pattern: 'facebook.com',
+        overrides: {
+            'simple': { quality: 20 },
+            'stealth': { quality: 88 },
+            'cloaked': { quality: 85 },
+        },
+    },
+    {
+        pattern: 'linkedin.com',
+        overrides: {
+            'simple': { quality: 25 },
+            'stealth': { quality: 88 },
+            'browser': { quality: 78 },
+        },
+    },
+    {
+        pattern: 'reddit.com',
+        overrides: {
+            'simple': { quality: 30 },
+            'browser': { quality: 88 },
+            'stealth': { quality: 85 },
+        },
+    },
+    {
+        pattern: 'threads.net',
+        overrides: {
+            'simple': { quality: 15 },
+            'stealth': { quality: 90 },
+            'cloaked': { quality: 85 },
+        },
+    },
+    // ── SPA-heavy / JS-rendered sites ────────────────────────────────────
+    {
+        pattern: 'vercel.app',
+        overrides: {
+            'browser': { quality: 90 },
+            'simple': { quality: 50 },
+        },
+    },
+    {
+        pattern: 'netlify.app',
+        overrides: {
+            'browser': { quality: 90 },
+            'simple': { quality: 50 },
+        },
+    },
+    {
+        pattern: 'notion.so',
+        overrides: {
+            'browser': { quality: 92 },
+            'simple': { quality: 20 },
+        },
+    },
+    {
+        pattern: 'figma.com',
+        overrides: {
+            'browser': { quality: 90 },
+            'simple': { quality: 15 },
+        },
+    },
+    // ── Static / well-structured sites ───────────────────────────────────
+    {
+        pattern: 'wikipedia.org',
+        overrides: {
+            'simple': { quality: 92 },
+            'browser': { quality: 80, cost: 70 },
+        },
+    },
+    {
+        pattern: 'github.com',
+        overrides: {
+            'simple': { quality: 85 },
+            'browser': { quality: 78, cost: 65 },
+        },
+    },
+    {
+        pattern: 'stackoverflow.com',
+        overrides: {
+            'simple': { quality: 88 },
+            'browser': { quality: 78 },
+        },
+    },
+    {
+        pattern: 'docs.python.org',
+        overrides: {
+            'simple': { quality: 90 },
+        },
+    },
+    {
+        pattern: 'developer.mozilla.org',
+        overrides: {
+            'simple': { quality: 90 },
+        },
+    },
+    {
+        pattern: 'news.ycombinator.com',
+        overrides: {
+            'simple': { quality: 92 },
+            'browser': { quality: 75 },
+        },
+    },
+    // ── Known-blocked / aggressive anti-bot ──────────────────────────────
+    {
+        pattern: 'zillow.com',
+        overrides: {
+            'simple': { quality: 10 },
+            'browser': { quality: 50 },
+            'cloaked': { quality: 90 },
+            'stealth': { quality: 85 },
+        },
+    },
+    {
+        pattern: 'yelp.com',
+        overrides: {
+            'simple': { quality: 15 },
+            'cloaked': { quality: 88 },
+            'stealth': { quality: 82 },
+        },
+    },
+    {
+        pattern: 'pinterest.com',
+        overrides: {
+            'simple': { quality: 15 },
+            'cloaked': { quality: 88 },
+            'stealth': { quality: 85 },
+        },
+    },
+    {
+        pattern: 'ticketmaster.com',
+        overrides: {
+            'simple': { quality: 10 },
+            'cloaked': { quality: 90 },
+            'stealth': { quality: 82 },
+        },
+    },
+];
+/**
+ * Returns domain-specific engine config overrides for a given hostname.
+ *
+ * Matches against known domain patterns using suffix matching.
+ * A pattern "twitter.com" matches hostnames "twitter.com", "www.twitter.com",
+ * "mobile.twitter.com", etc.
+ *
+ * @param hostname - The hostname to look up (e.g. "www.twitter.com")
+ * @returns Partial config overrides keyed by engine type, or an empty object
+ */
+export function getDomainOverrides(hostname) {
+    const lower = hostname.toLowerCase();
+    const merged = {};
+    for (const rule of DOMAIN_RULES) {
+        if (lower === rule.pattern || lower.endsWith(`.${rule.pattern}`)) {
+            // Merge overrides — last match wins for conflicting fields
+            for (const [engineKey, overrideValue] of Object.entries(rule.overrides)) {
+                const engine = engineKey;
+                merged[engine] = { ...merged[engine], ...overrideValue };
+            }
+        }
+    }
+    return merged;
+}
+// ---------------------------------------------------------------------------
+// Fallback chain builder
+// ---------------------------------------------------------------------------
+/**
+ * Deep-clones an EngineConfig and merges partial overrides onto it.
+ */
+function applyOverrides(base, overrides) {
+    return {
+        ...base,
+        ...overrides,
+        features: {
+            ...base.features,
+            ...(overrides.features ?? {}),
+        },
+        // Ensure type is always preserved from base
+        type: base.type,
+    };
+}
+/**
+ * Builds an ordered fallback chain of extraction engines for a given URL.
+ *
+ * The chain is constructed by:
+ * 1. Starting with default engine configurations
+ * 2. Applying domain-specific quality/score overrides
+ * 3. Filtering engines based on the provided options
+ * 4. Sorting by quality descending (ties broken by speed descending)
+ *
+ * @param url - The target URL to build a fallback chain for
+ * @param options - Controls which engines are eligible
+ * @returns Ordered array of engine entries, highest quality first
+ *
+ * @example
+ * ```ts
+ * // Basic chain for a static site
+ * const chain = buildFallbackChain('https://wikipedia.org/wiki/Test');
+ * // → [domain-api, simple, browser, stealth, cloaked, search-fallback]
+ *
+ * // Chain for a social media URL with rendering
+ * const chain = buildFallbackChain('https://twitter.com/user', { render: true });
+ * // → [domain-api, stealth, cloaked, browser, simple, search-fallback]
+ *
+ * // No browser rendering, no domain API
+ * const chain = buildFallbackChain('https://example.com', {
+ *   render: false,
+ *   noDomainApi: true,
+ * });
+ * // → [simple, search-fallback]
+ * ```
+ */
+export function buildFallbackChain(url, options = {}) {
+    const { render, stealth, noDomainApi } = options;
+    // 1. Parse hostname for domain overrides
+    let hostname = '';
+    try {
+        hostname = new URL(url).hostname;
+    }
+    catch {
+        // Invalid URL — proceed with no domain overrides
+    }
+    const domainOverrides = hostname ? getDomainOverrides(hostname) : {};
+    // 2. Build full config for each engine (base + domain overrides)
+    const allEngines = Object.keys(ENGINE_DEFAULTS);
+    const configs = allEngines.map((engineType) => {
+        const base = { ...ENGINE_DEFAULTS[engineType] };
+        const override = domainOverrides[engineType];
+        const config = override ? applyOverrides(base, override) : { ...base };
+        return { engine: engineType, config };
+    });
+    // 3. Filter engines based on options
+    const filtered = configs.filter(({ engine, config }) => {
+        // Remove domain-api if explicitly excluded
+        if (noDomainApi && engine === 'domain-api')
+            return false;
+        // When render is explicitly false, remove engines that require a browser
+        // UNLESS stealth is explicitly requested
+        if (render === false) {
+            if (config.features.javascript) {
+                // Keep stealth/cloaked engines if stealth was explicitly requested
+                if (stealth && config.features.stealth)
+                    return true;
+                return false;
+            }
+        }
+        return true;
+    });
+    // 4. Sort by quality descending, tie-break by speed descending
+    filtered.sort((a, b) => {
+        const qualityDiff = b.config.quality - a.config.quality;
+        if (qualityDiff !== 0)
+            return qualityDiff;
+        return b.config.speed - a.config.speed;
+    });
+    return filtered;
+}
+/**
+ * Returns the default engine configuration for a given engine type.
+ * Useful for inspecting baseline values without domain overrides.
+ *
+ * @param type - The engine type to look up
+ * @returns A copy of the default EngineConfig
+ */
+export function getEngineDefaults(type) {
+    return { ...ENGINE_DEFAULTS[type], features: { ...ENGINE_DEFAULTS[type].features } };
+}
+/**
+ * Returns all available engine types.
+ */
+export function getAvailableEngines() {
+    return Object.keys(ENGINE_DEFAULTS);
+}

package/dist/core/extract-inline.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Inline structured extraction using BYOK LLM
+ *
+ * After fetching page content, pass it + a JSON schema + optional prompt
+ * to an LLM and get back structured JSON matching the schema.
+ *
+ * Supports OpenAI, Anthropic, and Google (same BYOK pattern as /v1/answer).
+ */
+export type LLMProvider = 'openai' | 'anthropic' | 'google';
+export interface InlineExtractOptions {
+    /** JSON Schema describing the desired output structure */
+    schema?: Record<string, any>;
+    /** Natural language prompt describing what to extract */
+    prompt?: string;
+    /** LLM provider (required) */
+    llmProvider: LLMProvider;
+    /** LLM API key — BYOK (required) */
+    llmApiKey: string;
+    /** LLM model name (optional — uses provider default) */
+    llmModel?: string;
+}
+export interface InlineExtractResult {
+    /** Extracted structured data */
+    data: Record<string, any>;
+    /** Tokens consumed */
+    tokensUsed: {
+        input: number;
+        output: number;
+    };
+}
+/**
+ * Extract structured JSON from page content using an LLM (BYOK).
+ *
+ * @param content - Page content (markdown or text)
+ * @param options - Extraction options including schema, prompt, and LLM credentials
+ * @returns Extracted structured data + token usage
+ */
+export declare function extractInlineJson(content: string, options: InlineExtractOptions): Promise<InlineExtractResult>;

package/dist/core/extract-inline.js

@@ -0,0 +1,215 @@
+/**
+ * Inline structured extraction using BYOK LLM
+ *
+ * After fetching page content, pass it + a JSON schema + optional prompt
+ * to an LLM and get back structured JSON matching the schema.
+ *
+ * Supports OpenAI, Anthropic, and Google (same BYOK pattern as /v1/answer).
+ */
+function defaultModel(provider) {
+    switch (provider) {
+        case 'openai':
+            return 'gpt-4o-mini';
+        case 'anthropic':
+            return 'claude-3-5-sonnet-latest';
+        case 'google':
+            return 'gemini-1.5-flash';
+    }
+}
+function buildSystemPrompt(schema, prompt) {
+    const parts = [
+        'You are a structured data extraction assistant.',
+        'Extract data from the provided web page content and return ONLY valid JSON — no markdown fences, no explanation, no extra text.',
+    ];
+    if (prompt) {
+        parts.push(`\nInstruction: ${prompt}`);
+    }
+    if (schema) {
+        parts.push(`\nReturn a JSON object that conforms to this JSON Schema:\n${JSON.stringify(schema, null, 2)}`);
+    }
+    parts.push('\nReturn ONLY the JSON object.');
+    return parts.join('\n');
+}
+function truncateContent(content, maxChars = 24_000) {
+    if (content.length <= maxChars)
+        return content;
+    return content.slice(0, maxChars) + '\n\n[Content truncated]';
+}
+function parseJsonResponse(text) {
+    // Try direct parse first
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // Strip markdown code fences if present
+        const fenceMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+        if (fenceMatch) {
+            try {
+                return JSON.parse(fenceMatch[1].trim());
+            }
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', 'fence json parse failed:', e instanceof Error ? e.message : e);
+            }
+        }
+        // Try to find the first { ... } block
+        const braceStart = text.indexOf('{');
+        const braceEnd = text.lastIndexOf('}');
+        if (braceStart !== -1 && braceEnd > braceStart) {
+            try {
+                return JSON.parse(text.slice(braceStart, braceEnd + 1));
+            }
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', 'brace json parse failed:', e instanceof Error ? e.message : e);
+            }
+        }
+        throw new Error(`LLM returned invalid JSON: ${text.slice(0, 300)}`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Provider-specific calls (mirrors core/answer.ts patterns)
+// ---------------------------------------------------------------------------
+async function callOpenAI(apiKey, model, systemPrompt, userContent) {
+    const resp = await fetch('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model,
+            messages: [
+                { role: 'system', content: systemPrompt },
+                { role: 'user', content: userContent },
+            ],
+            temperature: 0,
+            response_format: { type: 'json_object' },
+        }),
+    });
+    if (!resp.ok) {
+        const errText = await resp.text().catch(() => '');
+        throw new Error(`OpenAI API error: HTTP ${resp.status}${errText ? ` - ${errText}` : ''}`);
+    }
+    const json = (await resp.json());
+    return {
+        text: String(json?.choices?.[0]?.message?.content || '').trim(),
+        usage: {
+            input: Number(json?.usage?.prompt_tokens || 0),
+            output: Number(json?.usage?.completion_tokens || 0),
+        },
+    };
+}
+async function callAnthropic(apiKey, model, systemPrompt, userContent) {
+    const resp = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-api-key': apiKey,
+            'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify({
+            model,
+            system: systemPrompt,
+            messages: [{ role: 'user', content: userContent }],
+            max_tokens: 4096,
+            temperature: 0,
+        }),
+    });
+    if (!resp.ok) {
+        const errText = await resp.text().catch(() => '');
+        throw new Error(`Anthropic API error: HTTP ${resp.status}${errText ? ` - ${errText}` : ''}`);
+    }
+    const json = (await resp.json());
+    const blocks = Array.isArray(json?.content) ? json.content : [];
+    const text = blocks
+        .map((b) => (typeof b?.text === 'string' ? b.text : ''))
+        .join('')
+        .trim();
+    return {
+        text,
+        usage: {
+            input: Number(json?.usage?.input_tokens || 0),
+            output: Number(json?.usage?.output_tokens || 0),
+        },
+    };
+}
+async function callGoogle(apiKey, model, systemPrompt, userContent) {
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent?key=${encodeURIComponent(apiKey)}`;
+    const resp = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            contents: [
+                {
+                    role: 'user',
+                    parts: [{ text: `${systemPrompt}\n\n${userContent}` }],
+                },
+            ],
+            generationConfig: {
+                temperature: 0,
+                responseMimeType: 'application/json',
+            },
+        }),
+    });
+    if (!resp.ok) {
+        const errText = await resp.text().catch(() => '');
+        throw new Error(`Google API error: HTTP ${resp.status}${errText ? ` - ${errText}` : ''}`);
+    }
+    const json = (await resp.json());
+    const parts = json?.candidates?.[0]?.content?.parts;
+    const text = Array.isArray(parts)
+        ? parts.map((p) => (typeof p?.text === 'string' ? p.text : '')).join('')
+        : '';
+    return {
+        text: String(text || '').trim(),
+        usage: {
+            input: Number(json?.usageMetadata?.promptTokenCount || 0),
+            output: Number(json?.usageMetadata?.candidatesTokenCount || 0),
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Extract structured JSON from page content using an LLM (BYOK).
+ *
+ * @param content - Page content (markdown or text)
+ * @param options - Extraction options including schema, prompt, and LLM credentials
+ * @returns Extracted structured data + token usage
+ */
+export async function extractInlineJson(content, options) {
+    const { schema, prompt, llmProvider, llmApiKey, llmModel } = options;
+    if (!llmApiKey) {
+        throw new Error('Inline extraction requires "llmApiKey" (BYOK)');
+    }
+    if (!llmProvider) {
+        throw new Error('Inline extraction requires "llmProvider" (openai, anthropic, or google)');
+    }
+    if (!schema && !prompt) {
+        throw new Error('Inline extraction requires "schema" or "prompt" (or both)');
+    }
+    const model = (llmModel || '').trim() || defaultModel(llmProvider);
+    const systemPrompt = buildSystemPrompt(schema, prompt);
+    const userContent = truncateContent(content);
+    let result;
+    switch (llmProvider) {
+        case 'openai':
+            result = await callOpenAI(llmApiKey, model, systemPrompt, userContent);
+            break;
+        case 'anthropic':
+            result = await callAnthropic(llmApiKey, model, systemPrompt, userContent);
+            break;
+        case 'google':
+            result = await callGoogle(llmApiKey, model, systemPrompt, userContent);
+            break;
+        default:
+            throw new Error(`Unsupported llmProvider: ${llmProvider}`);
+    }
+    const data = parseJsonResponse(result.text);
+    return {
+        data,
+        tokensUsed: result.usage,
+    };
+}

package/dist/core/extract-listings.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Auto-extract repeated listing patterns from HTML pages.
+ *
+ * Given raw HTML (e.g. an eBay search results page), this module detects the
+ * largest group of sibling elements with a consistent internal structure and
+ * extracts structured fields (title, price, image, link, description, rating)
+ * from each item.
+ *
+ * @module extract-listings
+ */
+/** A single extracted listing item. */
+export interface ListingItem {
+    title?: string;
+    price?: string;
+    image?: string;
+    link?: string;
+    description?: string;
+    rating?: string;
+    [key: string]: string | undefined;
+}
+/**
+ * Automatically detect repeated listing patterns in raw HTML and extract
+ * structured items.
+ *
+ * @param html - Raw HTML string to parse.
+ * @param url  - Optional base URL for resolving relative links and images.
+ * @returns    Array of extracted listing items (may be empty).
+ *
+ * @example
+ * ```typescript
+ * import { extractListings } from 'webpeel';
+ *
+ * const items = extractListings(ebayHtml, 'https://ebay.com/sch?q=card');
+ * console.log(items[0].title);  // "Charizard VMAX 020/189"
+ * console.log(items[0].price);  // "$24.99"
+ * ```
+ */
+export declare function extractListings(html: string, url?: string): ListingItem[];