@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/screenshot.d.ts

@@ -0,0 +1,224 @@
+/**
+ * Screenshot helper (Playwright)
+ *
+ * Returns a base64-encoded screenshot for a given URL.
+ */
+import type { PageAction } from '../types.js';
+export type ScreenshotFormat = 'png' | 'jpeg';
+export interface ScreenshotOptions {
+    fullPage?: boolean;
+    width?: number;
+    height?: number;
+    /** png | jpeg | jpg (jpg is treated as jpeg) */
+    format?: 'png' | 'jpeg' | 'jpg';
+    /** JPEG quality (1-100). Ignored for PNG. */
+    quality?: number;
+    /** Wait in ms after page load (domcontentloaded) */
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    actions?: PageAction[];
+    scrollThrough?: boolean;
+    selector?: string;
+}
+export interface ScreenshotResult {
+    url: string;
+    format: ScreenshotFormat;
+    contentType: string;
+    /** Base64-encoded image bytes (no data: prefix) */
+    screenshot: string;
+}
+export declare function takeScreenshot(url: string, options?: ScreenshotOptions): Promise<ScreenshotResult>;
+export interface FilmstripOptions {
+    frames?: number;
+    width?: number;
+    height?: number;
+    /** png | jpeg | jpg (jpg is treated as jpeg) */
+    format?: 'png' | 'jpeg' | 'jpg';
+    quality?: number;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}
+export interface FilmstripResult {
+    url: string;
+    format: ScreenshotFormat;
+    contentType: string;
+    /** Array of base64-encoded screenshots (no data: prefix) */
+    frames: string[];
+    frameCount: number;
+}
+export declare function takeFilmstrip(url: string, options?: FilmstripOptions): Promise<FilmstripResult>;
+export interface ScreenshotDiffOptions {
+    width?: number;
+    height?: number;
+    fullPage?: boolean;
+    threshold?: number;
+    format?: 'png' | 'jpeg' | 'jpg';
+    quality?: number;
+    stealth?: boolean;
+    waitFor?: number;
+    timeout?: number;
+}
+export interface ScreenshotDiffResult {
+    diff: string;
+    diffPixels: number;
+    totalPixels: number;
+    diffPercent: number;
+    dimensions: {
+        width: number;
+        height: number;
+    };
+}
+export declare function takeScreenshotDiff(url1: string, url2: string, options?: ScreenshotDiffOptions): Promise<ScreenshotDiffResult>;
+export type { DesignAuditResult, DesignAnalysis, EffectInstance } from './fetcher.js';
+import type { DesignAnalysis } from './fetcher.js';
+import { type DesignComparison } from './design-compare.js';
+export type { DesignComparison, DesignGap } from './design-compare.js';
+export interface AuditOptions {
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg' | 'jpg';
+    quality?: number;
+    selector?: string;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    scrollThrough?: boolean;
+}
+export interface AuditSection {
+    index: number;
+    tag: string;
+    id: string;
+    className: string;
+    top: number;
+    height: number;
+    screenshot: string;
+}
+export interface AuditResult {
+    url: string;
+    format: ScreenshotFormat;
+    sections: AuditSection[];
+}
+export declare function takeAuditScreenshots(url: string, options?: AuditOptions): Promise<AuditResult>;
+export interface AnimationCaptureOptions {
+    frames?: number;
+    intervalMs?: number;
+    scrollTo?: number;
+    selector?: string;
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg' | 'jpg';
+    quality?: number;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}
+export interface AnimationFrame {
+    index: number;
+    timestampMs: number;
+    screenshot: string;
+}
+export interface AnimationCaptureResult {
+    url: string;
+    format: ScreenshotFormat;
+    frames: AnimationFrame[];
+    frameCount: number;
+}
+export declare function takeAnimationCapture(url: string, options?: AnimationCaptureOptions): Promise<AnimationCaptureResult>;
+export interface ViewportSpec {
+    width: number;
+    height: number;
+    label?: string;
+}
+export interface ViewportsBatchOptions {
+    viewports: ViewportSpec[];
+    fullPage?: boolean;
+    format?: 'png' | 'jpeg' | 'jpg';
+    quality?: number;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    scrollThrough?: boolean;
+}
+export interface ViewportResult {
+    width: number;
+    height: number;
+    label: string;
+    screenshot: string;
+}
+export interface ViewportsBatchResult {
+    url: string;
+    format: ScreenshotFormat;
+    viewports: ViewportResult[];
+}
+export declare function takeViewportsBatch(url: string, options: ViewportsBatchOptions): Promise<ViewportsBatchResult>;
+export interface DesignAuditOptions {
+    rules?: {
+        spacingGrid?: number;
+        minTouchTarget?: number;
+        minContrast?: number;
+    };
+    selector?: string;
+    width?: number;
+    height?: number;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}
+export interface DesignAuditSummaryResult {
+    url: string;
+    audit: import('./fetcher.js').DesignAuditResult;
+}
+export declare function takeDesignAudit(url: string, options?: DesignAuditOptions): Promise<DesignAuditSummaryResult>;
+export interface DesignAnalysisOptions {
+    selector?: string;
+    width?: number;
+    height?: number;
+    waitFor?: number;
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}
+export interface DesignAnalysisSummaryResult {
+    url: string;
+    analysis: DesignAnalysis;
+}
+export declare function takeDesignAnalysis(url: string, options?: DesignAnalysisOptions): Promise<DesignAnalysisSummaryResult>;
+export interface DesignCompareOptions {
+    width?: number;
+    height?: number;
+    waitFor?: number;
+    timeout?: number;
+    stealth?: boolean;
+}
+export interface DesignComparisonResult {
+    subjectUrl: string;
+    referenceUrl: string;
+    comparison: DesignComparison;
+}
+/**
+ * Compare the design of two URLs by extracting design tokens from each in
+ * parallel and diffing the result into a structured DesignComparison.
+ */
+export declare function takeDesignComparison(subjectUrl: string, referenceUrl: string, options?: DesignCompareOptions): Promise<DesignComparisonResult>;

package/dist/core/screenshot.js

@@ -0,0 +1,207 @@
+/**
+ * Screenshot helper (Playwright)
+ *
+ * Returns a base64-encoded screenshot for a given URL.
+ */
+import { browserScreenshot, browserFilmstrip, browserDiff } from './fetcher.js';
+export async function takeScreenshot(url, options = {}) {
+    const format = (options.format === 'jpg' ? 'jpeg' : (options.format || 'png'));
+    const { buffer, finalUrl } = await browserScreenshot(url, {
+        fullPage: options.fullPage || false,
+        width: options.width,
+        height: options.height,
+        format,
+        quality: options.quality,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+        actions: options.actions,
+        scrollThrough: options.scrollThrough,
+        selector: options.selector,
+    });
+    return {
+        url: finalUrl,
+        format,
+        contentType: format === 'png' ? 'image/png' : 'image/jpeg',
+        screenshot: buffer.toString('base64'),
+    };
+}
+export async function takeFilmstrip(url, options = {}) {
+    const format = (options.format === 'jpg' ? 'jpeg' : (options.format || 'png'));
+    const { frames, finalUrl } = await browserFilmstrip(url, {
+        frames: options.frames,
+        width: options.width,
+        height: options.height,
+        format,
+        quality: options.quality,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+    });
+    return {
+        url: finalUrl,
+        format,
+        contentType: format === 'png' ? 'image/png' : 'image/jpeg',
+        frames: frames.map(f => f.toString('base64')),
+        frameCount: frames.length,
+    };
+}
+export async function takeScreenshotDiff(url1, url2, options = {}) {
+    const { diffBuffer, diffPixels, totalPixels, diffPercent, dimensions } = await browserDiff(url1, url2, {
+        width: options.width,
+        height: options.height,
+        fullPage: options.fullPage,
+        threshold: options.threshold,
+        stealth: options.stealth,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+    });
+    return {
+        diff: diffBuffer.toString('base64'),
+        diffPixels,
+        totalPixels,
+        diffPercent,
+        dimensions,
+    };
+}
+// ── Audit ─────────────────────────────────────────────────────────────────────
+import { browserAudit, browserAnimationCapture, browserViewports, browserDesignAudit, browserDesignAnalysis } from './fetcher.js';
+import { buildDesignComparison } from './design-compare.js';
+export async function takeAuditScreenshots(url, options = {}) {
+    const format = (options.format === 'jpg' ? 'jpeg' : (options.format || 'jpeg'));
+    const { frames, finalUrl } = await browserAudit(url, {
+        width: options.width,
+        height: options.height,
+        format,
+        quality: options.quality,
+        selector: options.selector,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+        scrollThrough: options.scrollThrough,
+    });
+    return {
+        url: finalUrl,
+        format,
+        sections: frames.map(f => ({
+            index: f.index,
+            tag: f.tag,
+            id: f.id,
+            className: f.className,
+            top: f.top,
+            height: f.height,
+            screenshot: f.buffer.toString('base64'),
+        })),
+    };
+}
+export async function takeAnimationCapture(url, options = {}) {
+    const format = (options.format === 'jpg' ? 'jpeg' : (options.format || 'jpeg'));
+    const { frames, finalUrl } = await browserAnimationCapture(url, {
+        frames: options.frames,
+        intervalMs: options.intervalMs,
+        scrollTo: options.scrollTo,
+        selector: options.selector,
+        width: options.width,
+        height: options.height,
+        format,
+        quality: options.quality,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+    });
+    return {
+        url: finalUrl,
+        format,
+        frames: frames.map(f => ({
+            index: f.index,
+            timestampMs: f.timestampMs,
+            screenshot: f.buffer.toString('base64'),
+        })),
+        frameCount: frames.length,
+    };
+}
+export async function takeViewportsBatch(url, options) {
+    const format = (options.format === 'jpg' ? 'jpeg' : (options.format || 'jpeg'));
+    const { frames, finalUrl } = await browserViewports(url, {
+        viewports: options.viewports,
+        fullPage: options.fullPage,
+        format,
+        quality: options.quality,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+        scrollThrough: options.scrollThrough,
+    });
+    return {
+        url: finalUrl,
+        format,
+        viewports: frames.map(f => ({
+            width: f.width,
+            height: f.height,
+            label: f.label,
+            screenshot: f.buffer.toString('base64'),
+        })),
+    };
+}
+export async function takeDesignAudit(url, options = {}) {
+    const { audit, finalUrl } = await browserDesignAudit(url, {
+        rules: options.rules,
+        selector: options.selector,
+        width: options.width,
+        height: options.height,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+    });
+    return { url: finalUrl, audit };
+}
+export async function takeDesignAnalysis(url, options = {}) {
+    const { analysis, finalUrl } = await browserDesignAnalysis(url, {
+        selector: options.selector,
+        width: options.width,
+        height: options.height,
+        waitMs: options.waitFor,
+        timeoutMs: options.timeout,
+        userAgent: options.userAgent,
+        headers: options.headers,
+        cookies: options.cookies,
+        stealth: options.stealth,
+    });
+    return { url: finalUrl, analysis };
+}
+/**
+ * Compare the design of two URLs by extracting design tokens from each in
+ * parallel and diffing the result into a structured DesignComparison.
+ */
+export async function takeDesignComparison(subjectUrl, referenceUrl, options = {}) {
+    const { width = 1440, height = 900, waitFor, timeout = 60000, stealth, } = options;
+    const browserOpts = { width, height, waitMs: waitFor, timeoutMs: timeout, stealth };
+    const [subjectResult, referenceResult] = await Promise.all([
+        browserDesignAnalysis(subjectUrl, browserOpts),
+        browserDesignAnalysis(referenceUrl, browserOpts),
+    ]);
+    const comparison = buildDesignComparison(subjectResult.finalUrl, referenceResult.finalUrl, subjectResult.analysis, referenceResult.analysis);
+    return {
+        subjectUrl: subjectResult.finalUrl,
+        referenceUrl: referenceResult.finalUrl,
+        comparison,
+    };
+}

package/dist/core/search-engines.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Additional search engine providers: Baidu, Yandex
+ * HTTP-only scraping with cheerio — no browser, no API key required.
+ */
+import type { WebSearchResult, WebSearchOptions, SearchProvider, SearchProviderId } from './search-provider.js';
+export declare class BaiduSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+export declare class NaverSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+export declare class YahooJapanSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+export declare class YandexSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}

package/dist/core/search-engines.js

@@ -0,0 +1,182 @@
+/**
+ * Additional search engine providers: Baidu, Yandex
+ * HTTP-only scraping with cheerio — no browser, no API key required.
+ */
+import { load } from 'cheerio';
+import { simpleFetch } from './fetcher.js';
+// ── Baidu Search ──────────────────────────────────────────────────────────
+export class BaiduSearchProvider {
+    id = 'baidu';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        // Baidu search URL
+        const params = new URLSearchParams({
+            wd: query,
+            rn: String(Math.min(count, 50)),
+            ie: 'utf-8',
+        });
+        const url = `https://www.baidu.com/s?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            const seen = new Set();
+            // Baidu result selectors: .result or .c-container
+            $('.result, .c-container').each((_, elem) => {
+                const el = $(elem);
+                const linkEl = el.find('h3 a, .t a').first();
+                const title = linkEl.text().trim();
+                // Baidu uses redirect URLs — get the data-url or mu attribute for real URL
+                const href = el.attr('mu') || linkEl.attr('href') || '';
+                const snippet = el.find('.c-abstract, .c-span-last, .content-right_8Zs40').first().text().trim();
+                if (title && href && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Naver Search (Korea) ──────────────────────────────────────────────────
+export class NaverSearchProvider {
+    id = 'naver';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            query: query,
+            where: 'web',
+        });
+        const url = `https://search.naver.com/search.naver?${params}`;
+        try {
+            // Naver is heavily JS-rendered — use peel with render for full content
+            const { peel } = await import('../index.js');
+            const result = await peel(url, { render: true, format: 'html', wait: 2000, timeout: 15000 });
+            const html = result.content || '';
+            if (!html)
+                return [];
+            const $ = load(html);
+            const results = [];
+            const seen = new Set();
+            // Naver uses .type-web class for web results, titles in *-title-text classes
+            $('.type-web').each((_, elem) => {
+                const el = $(elem);
+                const parent = el.closest('li, section, [class*=item]').length ? el.closest('li, section, [class*=item]') : el.parent();
+                const title = parent.find('[class*="title-text"], [class*="Title"]').first().text().trim();
+                const href = parent.find('a[href^="http"]').first().attr('href') || '';
+                const snippet = parent.find('[class*="text-type-body"], [class*="desc"]').first().text().trim();
+                if (title && href && !href.includes('naver.com/search') && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            // Fallback: try generic link extraction if .type-web yielded nothing
+            if (results.length === 0) {
+                $('a[href^="http"]').each((_, elem) => {
+                    const el = $(elem);
+                    const href = el.attr('href') || '';
+                    const title = el.text().trim();
+                    if (title.length > 5 && title.length < 200 && href && !href.includes('naver.com') && !seen.has(href)) {
+                        seen.add(href);
+                        results.push({ title, url: href, snippet: '' });
+                    }
+                });
+            }
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Yahoo Japan Search ────────────────────────────────────────────────────
+export class YahooJapanSearchProvider {
+    id = 'yahoo_japan';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            p: query,
+            n: String(Math.min(count, 50)),
+            ei: 'UTF-8',
+        });
+        const url = `https://search.yahoo.co.jp/search?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            // Yahoo Japan result selectors (2026 layout uses sw-Card components)
+            const seen = new Set();
+            $('.sw-Card__title, .algo, .dd').each((_, elem) => {
+                const el = $(elem);
+                // Walk up to the card container to find the link and snippet
+                const card = el.closest('[class*="sw-Card"], .algo, .dd, li').length
+                    ? el.closest('[class*="sw-Card"], .algo, .dd, li')
+                    : el.parent();
+                const title = el.find('.sw-Card__titleMain, h3, a').first().text().trim() || el.text().trim();
+                const href = card.find('a[href^="http"]').first().attr('href') || '';
+                const snippet = card.find('.sw-Card__description, .sw-Card__floatText, p').first().text().trim();
+                // Filter Yahoo internal links
+                if (title && title.length > 3 && href && !href.includes('yahoo.co.jp/search') && !href.includes('cache.yahoofs') && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Yandex Search ──────────────────────────────────────────────────────────
+export class YandexSearchProvider {
+    id = 'yandex';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            text: query,
+            numdoc: String(Math.min(count, 50)),
+            lr: '84', // Default to US region; can be overridden
+        });
+        // Use Yandex HTML search
+        const url = `https://yandex.com/search/?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            const seen = new Set();
+            // Yandex result selectors
+            $('.serp-item, .organic').each((_, elem) => {
+                const el = $(elem);
+                const linkEl = el.find('.organic__url, .link, a[href]').first();
+                const title = el.find('.organic__title, .OrganicTitle-LinkText, h2').first().text().trim();
+                const href = linkEl.attr('href') || '';
+                const snippet = el.find('.organic__text, .OrganicText, .text-container').first().text().trim();
+                // Filter internal Yandex links
+                if (title && href && !href.includes('yandex.') && !seen.has(href)) {
+                    seen.add(href);
+                    // Normalize URL (Yandex sometimes uses relative paths)
+                    const fullUrl = href.startsWith('http') ? href : `https://${href}`;
+                    results.push({ title, url: fullUrl, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}