@jshookmcp/jshook 0.1.7 → 0.1.9

package/dist/modules/captcha/CaptchaDetector.js

@@ -1,388 +0,0 @@
-import { logger } from '../../utils/logger.js';
-import { CAPTCHA_KEYWORDS, CAPTCHA_SELECTORS, EXCLUDE_KEYWORDS, EXCLUDE_SELECTORS, } from '../captcha/CaptchaDetector.constants.js';
-export class CaptchaDetector {
-    static EXCLUDE_SELECTORS = EXCLUDE_SELECTORS;
-    static CAPTCHA_SELECTORS = CAPTCHA_SELECTORS;
-    static CAPTCHA_KEYWORDS = CAPTCHA_KEYWORDS;
-    static EXCLUDE_KEYWORDS = EXCLUDE_KEYWORDS;
-    async detect(page) {
-        try {
-            logger.info('Starting CAPTCHA detection checks');
-            const urlCheck = await this.checkUrl(page);
-            if (urlCheck.detected) {
-                return urlCheck;
-            }
-            const titleCheck = await this.checkTitle(page);
-            if (titleCheck.detected) {
-                return titleCheck;
-            }
-            const domCheck = await this.checkDOMElements(page);
-            if (domCheck.detected) {
-                return domCheck;
-            }
-            const textCheck = await this.checkPageText(page);
-            if (textCheck.detected) {
-                return textCheck;
-            }
-            const vendorCheck = await this.checkVendorSpecific(page);
-            if (vendorCheck.detected) {
-                return vendorCheck;
-            }
-            logger.info('No CAPTCHA detected by current heuristics');
-            return { detected: false, confidence: 0 };
-        }
-        catch (error) {
-            logger.error('CAPTCHA detection failed', error);
-            return { detected: false, confidence: 0 };
-        }
-    }
-    async checkUrl(page) {
-        const url = page.url();
-        const lowerUrl = url.toLowerCase();
-        for (const excludeKeyword of CaptchaDetector.EXCLUDE_KEYWORDS.url) {
-            if (lowerUrl.includes(excludeKeyword)) {
-                logger.debug(`URL matched exclusion keyword: ${excludeKeyword}`);
-                return { detected: false, confidence: 0, falsePositiveReason: `URL exclusion: ${excludeKeyword}` };
-            }
-        }
-        for (const keyword of CaptchaDetector.CAPTCHA_KEYWORDS.url) {
-            if (lowerUrl.includes(keyword)) {
-                let type = 'url_redirect';
-                let vendor = 'unknown';
-                let confidence = 70;
-                if (url.includes('cloudflare') || url.includes('cdn-cgi')) {
-                    type = 'cloudflare';
-                    vendor = 'cloudflare';
-                    confidence = 95;
-                }
-                else if (url.includes('recaptcha')) {
-                    type = 'recaptcha';
-                    vendor = 'recaptcha';
-                    confidence = 95;
-                }
-                else if (url.includes('hcaptcha')) {
-                    type = 'hcaptcha';
-                    vendor = 'hcaptcha';
-                    confidence = 95;
-                }
-                else if (url.includes('geetest')) {
-                    type = 'slider';
-                    vendor = 'geetest';
-                    confidence = 90;
-                }
-                if (confidence < 80) {
-                    const domCheck = await this.verifyByDOM(page);
-                    if (!domCheck) {
-                        logger.debug(`URL keyword match in DOM, skipping: ${keyword}`);
-                        return { detected: false, confidence: 0, falsePositiveReason: `URLDOM: ${keyword}` };
-                    }
-                    confidence = 85;
-                }
-                logger.warn(`CAPTCHA URL keyword detected: ${keyword} (confidence: ${confidence}%)`);
-                return {
-                    detected: true,
-                    type,
-                    url,
-                    vendor,
-                    confidence,
-                };
-            }
-        }
-        return { detected: false, confidence: 0 };
-    }
-    async checkTitle(page) {
-        const title = await page.title();
-        const lowerTitle = title.toLowerCase();
-        for (const excludeKeyword of CaptchaDetector.EXCLUDE_KEYWORDS.title) {
-            if (lowerTitle.includes(excludeKeyword.toLowerCase())) {
-                logger.debug(`Title matched exclusion keyword: ${excludeKeyword}`);
-                return { detected: false, confidence: 0, falsePositiveReason: `Title exclusion: ${excludeKeyword}` };
-            }
-        }
-        for (const keyword of CaptchaDetector.CAPTCHA_KEYWORDS.title) {
-            if (lowerTitle.includes(keyword)) {
-                const domCheck = await this.verifyByDOM(page);
-                if (!domCheck) {
-                    logger.debug(`DOM keyword is common UI element, skipping: ${keyword}`);
-                    return { detected: false, confidence: 0, falsePositiveReason: `DOM: ${keyword}` };
-                }
-                logger.warn(`CAPTCHA DOM keyword detected: ${keyword}`);
-                return {
-                    detected: true,
-                    type: 'page_redirect',
-                    title,
-                    confidence: 85,
-                };
-            }
-        }
-        return { detected: false, confidence: 0 };
-    }
-    async checkDOMElements(page) {
-        for (const selector of CaptchaDetector.CAPTCHA_SELECTORS.slider) {
-            const element = await page.$(selector);
-            if (element) {
-                const isVisible = await element.isIntersectingViewport();
-                if (isVisible) {
-                    const isRealSlider = await this.verifySliderElement(page, selector);
-                    if (!isRealSlider) {
-                        logger.debug(`Selector is generic, skipping: ${selector}`);
-                        continue;
-                    }
-                    logger.warn(`CAPTCHA selector detected: ${selector}`);
-                    let vendor = 'unknown';
-                    if (selector.includes('geetest'))
-                        vendor = 'geetest';
-                    else if (selector.includes('nc_') || selector.includes('aliyun'))
-                        vendor = 'aliyun';
-                    else if (selector.includes('tcaptcha') || selector.includes('tencent'))
-                        vendor = 'tencent';
-                    return {
-                        detected: true,
-                        type: 'slider',
-                        selector,
-                        vendor,
-                        confidence: 95,
-                    };
-                }
-            }
-        }
-        for (const selector of CaptchaDetector.CAPTCHA_SELECTORS.recaptcha) {
-            const element = await page.$(selector);
-            if (element) {
-                logger.warn(`reCAPTCHA element detected: ${selector}`);
-                return {
-                    detected: true,
-                    type: 'recaptcha',
-                    selector,
-                    vendor: 'recaptcha',
-                    confidence: 98,
-                };
-            }
-        }
-        for (const selector of CaptchaDetector.CAPTCHA_SELECTORS.hcaptcha) {
-            const element = await page.$(selector);
-            if (element) {
-                logger.warn(`hCaptcha element detected: ${selector}`);
-                return {
-                    detected: true,
-                    type: 'hcaptcha',
-                    selector,
-                    vendor: 'hcaptcha',
-                    confidence: 98,
-                };
-            }
-        }
-        for (const selector of CaptchaDetector.CAPTCHA_SELECTORS.cloudflare) {
-            const element = await page.$(selector);
-            if (element) {
-                logger.warn(`Cloudflare challenge detected: ${selector}`);
-                return {
-                    detected: true,
-                    type: 'cloudflare',
-                    selector,
-                    vendor: 'cloudflare',
-                    confidence: 97,
-                };
-            }
-        }
-        return { detected: false, confidence: 0 };
-    }
-    async checkPageText(page) {
-        const bodyText = await page.evaluate(() => document.body.innerText);
-        for (const excludeKeyword of CaptchaDetector.EXCLUDE_KEYWORDS.text) {
-            if (bodyText.includes(excludeKeyword)) {
-                logger.debug(`Body text matched exclusion keyword: ${excludeKeyword}`);
-                return { detected: false, confidence: 0, falsePositiveReason: `Text exclusion: ${excludeKeyword}` };
-            }
-        }
-        for (const keyword of CaptchaDetector.CAPTCHA_KEYWORDS.text) {
-            if (bodyText.includes(keyword)) {
-                const domCheck = await this.verifyByDOM(page);
-                if (!domCheck) {
-                    logger.debug(`Keyword is common element, skipping: ${keyword}`);
-                    return { detected: false, confidence: 0, falsePositiveReason: `DOM: ${keyword}` };
-                }
-                logger.warn(`CAPTCHA keyword detected: ${keyword}`);
-                return {
-                    detected: true,
-                    type: 'unknown',
-                    confidence: 75,
-                    details: { keyword },
-                };
-            }
-        }
-        return { detected: false, confidence: 0 };
-    }
-    async checkVendorSpecific(page) {
-        const geetestCheck = await page.evaluate(() => {
-            const win = window;
-            return !!win.initGeetest || document.querySelector('.geetest_holder');
-        });
-        if (geetestCheck) {
-            logger.warn('Geetest CAPTCHA indicators detected');
-            return {
-                detected: true,
-                type: 'slider',
-                vendor: 'geetest',
-                confidence: 95,
-            };
-        }
-        const tencentCheck = await page.evaluate(() => {
-            const win = window;
-            return !!win.TencentCaptcha || document.querySelector('.tcaptcha-transform');
-        });
-        if (tencentCheck) {
-            logger.warn('Tencent CAPTCHA indicators detected');
-            return {
-                detected: true,
-                type: 'slider',
-                vendor: 'tencent',
-                confidence: 95,
-            };
-        }
-        return { detected: false, confidence: 0 };
-    }
-    async waitForCompletion(page, timeout = 300000) {
-        logger.info('Waiting for CAPTCHA to be solved...');
-        const startTime = Date.now();
-        while (Date.now() - startTime < timeout) {
-            const result = await this.detect(page);
-            if (!result.detected) {
-                logger.info('CAPTCHA no longer detected; continuing workflow');
-                return true;
-            }
-            await new Promise((resolve) => setTimeout(resolve, 2000));
-        }
-        logger.error('Timed out while waiting for CAPTCHA completion');
-        return false;
-    }
-    async verifyByDOM(page) {
-        try {
-            const hasSlider = await page.evaluate(() => {
-                const sliderSelectors = [
-                    '.captcha-slider',
-                    '.geetest_slider',
-                    '.tcaptcha-transform',
-                    '#nc_1_wrapper',
-                    '.slide-verify',
-                ];
-                return sliderSelectors.some((sel) => document.querySelector(sel) !== null);
-            });
-            const hasRecaptcha = await page.evaluate(() => {
-                return (!!document.querySelector('iframe[src*="recaptcha"]') ||
-                    !!document.querySelector('.g-recaptcha'));
-            });
-            const hasHcaptcha = await page.evaluate(() => {
-                return (!!document.querySelector('iframe[src*="hcaptcha"]') ||
-                    !!document.querySelector('.h-captcha'));
-            });
-            const hasCloudflare = await page.evaluate(() => {
-                return (!!document.querySelector('#challenge-form') || !!document.querySelector('.cf-challenge'));
-            });
-            return hasSlider || hasRecaptcha || hasHcaptcha || hasCloudflare;
-        }
-        catch (error) {
-            logger.error('DOM verification failed during CAPTCHA detection', error);
-            return false;
-        }
-    }
-    async verifySliderElement(page, selector) {
-        try {
-            const excludeSelectors = CaptchaDetector.EXCLUDE_SELECTORS;
-            const result = await page.evaluate((sel, excludeSels) => {
-                const element = document.querySelector(sel);
-                if (!element)
-                    return false;
-                for (const excludeSel of excludeSels) {
-                    if (element.matches(excludeSel)) {
-                        console.warn(`[CaptchaDetector] Excluded selector match: ${excludeSel}`);
-                        return false;
-                    }
-                    if (element.closest(excludeSel)) {
-                        console.warn(`[CaptchaDetector] Excluded selector ancestor: ${excludeSel}`);
-                        return false;
-                    }
-                }
-                const rect = element.getBoundingClientRect();
-                if (rect.width === 0 || rect.height === 0)
-                    return false;
-                const className = element.className.toLowerCase();
-                const id = element.id.toLowerCase();
-                const excludeKeywords = [
-                    'video',
-                    'player',
-                    'swiper',
-                    'carousel',
-                    'banner',
-                    'gallery',
-                    'douyin',
-                    'tiktok',
-                    'scroll',
-                    'progress',
-                    'range',
-                    'volume',
-                    'seek',
-                    'timeline',
-                ];
-                for (const keyword of excludeKeywords) {
-                    if (className.includes(keyword) || id.includes(keyword)) {
-                        console.warn(`[CaptchaDetector] Excluded class/id keyword: ${keyword}`);
-                        return false;
-                    }
-                }
-                const hasCaptchaKeyword = className.includes('captcha') ||
-                    className.includes('verify') ||
-                    className.includes('challenge') ||
-                    id.includes('captcha') ||
-                    id.includes('verify') ||
-                    id.includes('challenge');
-                const style = window.getComputedStyle(element);
-                const hasDraggableStyle = style.cursor === 'move' || style.cursor === 'grab' || style.cursor === 'grabbing';
-                const hasSliderClass = className.includes('slider') || className.includes('slide');
-                const hasDragAttribute = element.hasAttribute('draggable') ||
-                    element.hasAttribute('data-slide') ||
-                    element.hasAttribute('data-captcha') ||
-                    element.hasAttribute('data-verify');
-                let parent = element.parentElement;
-                let hasParentCaptcha = false;
-                for (let i = 0; i < 3 && parent; i++) {
-                    const parentClass = parent.className.toLowerCase();
-                    const parentId = parent.id.toLowerCase();
-                    if (parentClass.includes('captcha') ||
-                        parentClass.includes('verify') ||
-                        parentClass.includes('challenge') ||
-                        parentId.includes('captcha') ||
-                        parentId.includes('verify')) {
-                        hasParentCaptcha = true;
-                        break;
-                    }
-                    parent = parent.parentElement;
-                }
-                const width = rect.width;
-                const height = rect.height;
-                const hasReasonableSize = width >= 30 && width <= 500 && height >= 30 && height <= 200;
-                if (!hasReasonableSize) {
-                    console.warn(`[CaptchaDetector] Rejected by size heuristic: ${width}x${height}`);
-                    return false;
-                }
-                const conditionA = hasCaptchaKeyword && (hasSliderClass || hasDraggableStyle);
-                const conditionB = hasParentCaptcha && hasSliderClass && hasDragAttribute;
-                const isVendorSpecific = className.includes('geetest') ||
-                    className.includes('nc_') ||
-                    className.includes('tcaptcha') ||
-                    className.includes('yidun') ||
-                    id.includes('nc_1_wrapper');
-                const isValid = conditionA || conditionB || isVendorSpecific;
-                if (!isValid) {
-                    console.warn(`[CaptchaDetector] Slider verification rejected - captcha:${hasCaptchaKeyword}, slider:${hasSliderClass}, parent:${hasParentCaptcha}`);
-                }
-                return isValid;
-            }, selector, excludeSelectors);
-            return result;
-        }
-        catch (error) {
-            logger.error('Slider element verification failed', error);
-            return false;
-        }
-    }
-}

package/dist/modules/collector/CodeCollector.d.ts

@@ -1,115 +0,0 @@
-import type { Browser, Page } from 'rebrowser-puppeteer-core';
-import type { CollectCodeOptions, CollectCodeResult, CodeFile, PuppeteerConfig } from '../../types/index.js';
-import { CodeCache } from '../collector/CodeCache.js';
-import { SmartCodeCollector } from '../collector/SmartCodeCollector.js';
-import { CodeCompressor } from '../collector/CodeCompressor.js';
-export declare class CodeCollector {
-    private config;
-    private browser;
-    private collectedUrls;
-    private initPromise;
-    private collectLock;
-    private readonly MAX_COLLECTED_URLS;
-    private readonly MAX_FILES_PER_COLLECT;
-    private readonly MAX_RESPONSE_SIZE;
-    private readonly MAX_SINGLE_FILE_SIZE;
-    private readonly viewport;
-    private readonly userAgent;
-    private collectedFilesCache;
-    private cache;
-    cacheEnabled: boolean;
-    smartCollector: SmartCodeCollector;
-    private compressor;
-    private cdpSession;
-    cdpListeners: {
-        responseReceived?: (params: unknown) => void;
-    };
-    private activePageIndex;
-    private currentHeadless;
-    constructor(config: PuppeteerConfig);
-    setCacheEnabled(enabled: boolean): void;
-    clearFileCache(): Promise<void>;
-    getFileCacheStats(): Promise<{
-        memoryEntries: number;
-        diskEntries: number;
-        totalSize: number;
-    }>;
-    clearAllData(): Promise<void>;
-    getAllStats(): Promise<{
-        cache: {
-            memoryEntries: number;
-            diskEntries: number;
-            totalSize: number;
-        };
-        compression: {
-            cacheSize: number;
-            totalCompressed: number;
-            totalOriginalSize: number;
-            totalCompressedSize: number;
-            averageRatio: number;
-            cacheHits: number;
-            cacheMisses: number;
-            totalTime: number;
-        };
-        collector: {
-            collectedUrls: number;
-            maxCollectedUrls: number;
-        };
-    }>;
-    getCache(): CodeCache;
-    getCompressor(): CodeCompressor;
-    cleanupCollectedUrls(): void;
-    init(headless?: boolean): Promise<void>;
-    private initInner;
-    private resolveExecutablePath;
-    close(): Promise<void>;
-    getActivePage(): Promise<Page>;
-    listPages(): Promise<Array<{
-        index: number;
-        url: string;
-        title: string;
-    }>>;
-    selectPage(index: number): Promise<void>;
-    createPage(url?: string): Promise<Page>;
-    private applyAntiDetection;
-    getStatus(): Promise<{
-        running: boolean;
-        pagesCount: number;
-        version?: string;
-        effectiveHeadless?: boolean;
-    }>;
-    collect(options: CollectCodeOptions): Promise<CollectCodeResult>;
-    private collectInner;
-    shouldCollectUrl(url: string, filterRules?: string[]): boolean;
-    navigateWithRetry(page: Page, url: string, options: NonNullable<Parameters<Page['goto']>[1]>, maxRetries?: number): Promise<void>;
-    getPerformanceMetrics(page: Page): Promise<Record<string, number>>;
-    collectPageMetadata(page: Page): Promise<Record<string, unknown>>;
-    connect(endpoint: string): Promise<void>;
-    getBrowser(): Browser | null;
-    getCollectionStats(): {
-        totalCollected: number;
-        uniqueUrls: number;
-    };
-    clearCache(): void;
-    getCollectedFilesSummary(): Array<{
-        url: string;
-        size: number;
-        type: string;
-        truncated?: boolean;
-        originalSize?: number;
-    }>;
-    getFileByUrl(url: string): CodeFile | null;
-    getFilesByPattern(pattern: string, limit?: number, maxTotalSize?: number): {
-        files: CodeFile[];
-        totalSize: number;
-        matched: number;
-        returned: number;
-        truncated: boolean;
-    };
-    getTopPriorityFiles(topN?: number, maxTotalSize?: number): {
-        files: CodeFile[];
-        totalSize: number;
-        totalFiles: number;
-    };
-    clearCollectedFilesCache(): void;
-}