@jshookmcp/jshook 0.1.8 → 0.2.0

package/dist/src/modules/collector/CodeCollector.js

@@ -0,0 +1,475 @@
+import { existsSync } from 'fs';
+import puppeteer from 'rebrowser-puppeteer-core';
+import { logger } from '../../utils/logger.js';
+import { PrerequisiteError } from '../../errors/PrerequisiteError.js';
+import { CodeCache } from '../collector/CodeCache.js';
+import { SmartCodeCollector } from '../collector/SmartCodeCollector.js';
+import { CodeCompressor } from '../collector/CodeCompressor.js';
+import { calculatePriorityScore } from '../collector/PageScriptCollectors.js';
+import { findBrowserExecutable } from '../../utils/browserExecutable.js';
+import { collectInnerImpl } from '../collector/CodeCollectorCollectInternal.js';
+import { shouldCollectUrlImpl, navigateWithRetryImpl, getPerformanceMetricsImpl, collectPageMetadataImpl } from '../collector/CodeCollectorUtilsInternal.js';
+export class CodeCollector {
+    config;
+    browser = null;
+    collectedUrls = new Set();
+    initPromise = null;
+    collectLock = null;
+    MAX_COLLECTED_URLS;
+    MAX_FILES_PER_COLLECT;
+    MAX_RESPONSE_SIZE;
+    MAX_SINGLE_FILE_SIZE;
+    viewport;
+    userAgent;
+    collectedFilesCache = new Map();
+    cache;
+    cacheEnabled = true;
+    smartCollector;
+    compressor;
+    cdpSession = null;
+    cdpListeners = {};
+    activePageIndex = null;
+    currentHeadless = null;
+    explicitlyClosed = false;
+    constructor(config) {
+        this.config = config;
+        this.MAX_COLLECTED_URLS = config.maxCollectedUrls ?? 10000;
+        this.MAX_FILES_PER_COLLECT = config.maxFilesPerCollect ?? 200;
+        this.MAX_RESPONSE_SIZE = config.maxTotalContentSize ?? 512 * 1024;
+        this.MAX_SINGLE_FILE_SIZE = config.maxSingleFileSize ?? 200 * 1024;
+        this.viewport = config.viewport ?? { width: 1920, height: 1080 };
+        this.userAgent =
+            config.userAgent ??
+                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+        this.cache = new CodeCache();
+        this.smartCollector = new SmartCodeCollector();
+        this.compressor = new CodeCompressor();
+        logger.info(` CodeCollector limits: maxCollect=${this.MAX_FILES_PER_COLLECT} files, maxResponse=${(this.MAX_RESPONSE_SIZE / 1024).toFixed(0)}KB, maxSingle=${(this.MAX_SINGLE_FILE_SIZE / 1024).toFixed(0)}KB`);
+        logger.info(` Strategy: Collect ALL files -> Cache -> Return summary/partial data to fit MCP limits`);
+    }
+    setCacheEnabled(enabled) {
+        this.cacheEnabled = enabled;
+        logger.info(`Code cache ${enabled ? 'enabled' : 'disabled'}`);
+    }
+    async clearFileCache() {
+        await this.cache.clear();
+    }
+    async getFileCacheStats() {
+        return await this.cache.getStats();
+    }
+    async clearAllData() {
+        logger.info('Clearing all collected data...');
+        await this.cache.clear();
+        this.compressor.clearCache();
+        this.compressor.resetStats();
+        this.collectedUrls.clear();
+        this.collectedFilesCache.clear();
+        logger.success('All data cleared');
+    }
+    async getAllStats() {
+        const cacheStats = await this.cache.getStats();
+        const compressionStats = this.compressor.getStats();
+        return {
+            cache: cacheStats,
+            compression: {
+                ...compressionStats,
+                cacheSize: this.compressor.getCacheSize(),
+            },
+            collector: {
+                collectedUrls: this.collectedUrls.size,
+                maxCollectedUrls: this.MAX_COLLECTED_URLS,
+            },
+        };
+    }
+    getCache() {
+        return this.cache;
+    }
+    getCompressor() {
+        return this.compressor;
+    }
+    cleanupCollectedUrls() {
+        if (this.collectedUrls.size > this.MAX_COLLECTED_URLS) {
+            logger.warn(`Collected URLs exceeded ${this.MAX_COLLECTED_URLS}, clearing...`);
+            const urls = Array.from(this.collectedUrls);
+            this.collectedUrls.clear();
+            urls
+                .slice(-Math.floor(this.MAX_COLLECTED_URLS / 2))
+                .forEach((url) => this.collectedUrls.add(url));
+        }
+    }
+    async init(headless) {
+        if (this.browser) {
+            return;
+        }
+        this.explicitlyClosed = false;
+        if (this.initPromise) {
+            return this.initPromise;
+        }
+        this.initPromise = this.initInner(headless);
+        try {
+            await this.initPromise;
+        }
+        finally {
+            this.initPromise = null;
+        }
+    }
+    async initInner(headless) {
+        const useHeadless = headless ?? this.config.headless;
+        const executablePath = this.resolveExecutablePath();
+        const launchOptions = {
+            headless: useHeadless,
+            args: [
+                ...(this.config.args || []),
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
+                '--disable-blink-features=AutomationControlled',
+                '--disable-extensions',
+                '--disable-component-extensions-with-background-pages',
+                '--disable-web-security',
+                '--disable-features=IsolateOrigins,site-per-process',
+                `--window-size=${this.viewport.width},${this.viewport.height}`,
+                '--ignore-certificate-errors',
+            ],
+            defaultViewport: this.viewport,
+            protocolTimeout: 60000,
+        };
+        if (executablePath) {
+            launchOptions.executablePath = executablePath;
+        }
+        logger.info('Initializing browser with anti-detection...');
+        this.browser = await puppeteer.launch(launchOptions);
+        this.currentHeadless = useHeadless === undefined ? true : useHeadless !== false;
+        this.browser.on('disconnected', () => {
+            logger.warn('Browser disconnected');
+            this.browser = null;
+            this.currentHeadless = null;
+            if (this.cdpSession) {
+                this.cdpSession = null;
+                this.cdpListeners = {};
+            }
+        });
+        logger.success('Browser initialized with enhanced anti-detection');
+    }
+    resolveExecutablePath() {
+        const configuredPath = this.config.executablePath?.trim();
+        if (configuredPath) {
+            if (existsSync(configuredPath)) {
+                return configuredPath;
+            }
+            throw new Error(`Configured browser executable was not found: ${configuredPath}. ` +
+                'Set a valid executablePath or configure CHROME_PATH / PUPPETEER_EXECUTABLE_PATH / BROWSER_EXECUTABLE_PATH.');
+        }
+        const detectedPath = findBrowserExecutable();
+        if (detectedPath) {
+            return detectedPath;
+        }
+        logger.info('No explicit browser executable configured. Falling back to Puppeteer-managed browser resolution.');
+        return undefined;
+    }
+    async close() {
+        await this.clearAllData();
+        this.explicitlyClosed = true;
+        this.activePageIndex = null;
+        const browser = this.browser;
+        this.browser = null;
+        this.currentHeadless = null;
+        if (this.cdpSession) {
+            this.cdpSession = null;
+            this.cdpListeners = {};
+        }
+        if (browser) {
+            await browser.close();
+        }
+        logger.info('Browser closed and all data cleared');
+    }
+    async getActivePage() {
+        if (!this.browser) {
+            if (this.explicitlyClosed) {
+                throw new PrerequisiteError('Browser was explicitly closed. Call browser_launch or browser_attach first.');
+            }
+            try {
+                await this.init();
+            }
+            catch (error) {
+                throw new PrerequisiteError(`Browser not available: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        const pages = await this.browser.pages();
+        if (pages.length === 0) {
+            return await this.browser.newPage();
+        }
+        if (this.activePageIndex !== null && this.activePageIndex < pages.length) {
+            return pages[this.activePageIndex];
+        }
+        const lastPage = pages[pages.length - 1];
+        if (!lastPage) {
+            throw new Error('Failed to get active page');
+        }
+        return lastPage;
+    }
+    async listPages() {
+        if (!this.browser) {
+            return [];
+        }
+        const pages = await this.browser.pages();
+        const results = await Promise.all(pages.map(async (page, index) => {
+            let url = '';
+            let title = '';
+            try {
+                url = page.url();
+            }
+            catch { }
+            try {
+                title = await page.title();
+            }
+            catch { }
+            return { index, url, title };
+        }));
+        return results;
+    }
+    async selectPage(index) {
+        if (!this.browser) {
+            throw new Error('Browser not connected');
+        }
+        const pages = await this.browser.pages();
+        if (index < 0 || index >= pages.length) {
+            throw new Error(`Page index ${index} out of range (0-${pages.length - 1})`);
+        }
+        this.activePageIndex = index;
+        logger.info(`Active page set to index ${index}: ${pages[index].url()}`);
+    }
+    async createPage(url) {
+        if (!this.browser) {
+            await this.init();
+        }
+        const page = await this.browser.newPage();
+        await page.setUserAgent(this.userAgent);
+        await this.applyAntiDetection(page);
+        if (url) {
+            await page.goto(url, {
+                waitUntil: 'networkidle2',
+                timeout: this.config.timeout,
+            });
+        }
+        logger.info(`New page created${url ? `: ${url}` : ''}`);
+        return page;
+    }
+    async applyAntiDetection(page) {
+        await page.evaluateOnNewDocument(() => {
+            Object.defineProperty(navigator, 'webdriver', {
+                get: () => false,
+            });
+            Object.defineProperty(navigator, 'plugins', {
+                get: () => [1, 2, 3, 4, 5],
+            });
+            Object.defineProperty(navigator, 'languages', {
+                get: () => ['en-US', 'en'],
+            });
+            const win = window;
+            if (!win.chrome) {
+                win.chrome = {
+                    runtime: {},
+                    loadTimes: function () { },
+                    csi: function () { },
+                    app: {},
+                };
+            }
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => {
+                if (parameters.name === 'notifications') {
+                    return Promise.resolve({ state: 'denied' });
+                }
+                return originalQuery(parameters);
+            };
+        });
+    }
+    async getStatus() {
+        if (!this.browser) {
+            return {
+                running: false,
+                pagesCount: 0,
+            };
+        }
+        try {
+            const pages = await this.browser.pages();
+            const version = await this.browser.version();
+            return {
+                running: true,
+                pagesCount: pages.length,
+                version,
+                effectiveHeadless: this.currentHeadless ?? undefined,
+            };
+        }
+        catch (error) {
+            logger.debug('Browser not running or disconnected:', error);
+            return {
+                running: false,
+                pagesCount: 0,
+            };
+        }
+    }
+    async collect(options) {
+        while (this.collectLock) {
+            try {
+                await this.collectLock;
+            }
+            catch { }
+        }
+        let resolve;
+        let reject;
+        this.collectLock = new Promise((res, rej) => { resolve = res; reject = rej; });
+        try {
+            const result = await this.collectInner(options);
+            resolve(result);
+            return result;
+        }
+        catch (e) {
+            reject(e);
+            throw e;
+        }
+        finally {
+            this.collectLock = null;
+        }
+    }
+    async collectInner(options) {
+        return collectInnerImpl(this, options);
+    }
+    shouldCollectUrl(url, filterRules) {
+        return shouldCollectUrlImpl(url, filterRules);
+    }
+    async navigateWithRetry(page, url, options, maxRetries = 3) {
+        return navigateWithRetryImpl(page, url, options, maxRetries);
+    }
+    async getPerformanceMetrics(page) {
+        return getPerformanceMetricsImpl(page);
+    }
+    async collectPageMetadata(page) {
+        return collectPageMetadataImpl(page);
+    }
+    async connect(endpoint) {
+        this.explicitlyClosed = false;
+        if (this.browser) {
+            try {
+                await this.browser.disconnect();
+            }
+            catch { }
+            this.browser = null;
+            this.currentHeadless = null;
+        }
+        this.activePageIndex = null;
+        logger.info(`Connecting to existing browser: ${endpoint}`);
+        const connectOptions = endpoint.startsWith('ws://') || endpoint.startsWith('wss://')
+            ? { browserWSEndpoint: endpoint }
+            : { browserURL: endpoint };
+        this.browser = await puppeteer.connect(connectOptions);
+        this.browser.on('disconnected', () => {
+            logger.warn('Browser disconnected');
+            this.browser = null;
+            this.currentHeadless = null;
+            if (this.cdpSession) {
+                this.cdpSession = null;
+                this.cdpListeners = {};
+            }
+        });
+        logger.success('Connected to existing browser successfully');
+    }
+    getBrowser() {
+        return this.browser;
+    }
+    getCollectionStats() {
+        return {
+            totalCollected: this.collectedUrls.size,
+            uniqueUrls: this.collectedUrls.size,
+        };
+    }
+    clearCache() {
+        this.collectedUrls.clear();
+        logger.info('Collection cache cleared');
+    }
+    getCollectedFilesSummary() {
+        const summaries = Array.from(this.collectedFilesCache.values()).map((file) => ({
+            url: file.url,
+            size: file.size,
+            type: file.type,
+            truncated: typeof file.metadata?.truncated === 'boolean' ? file.metadata.truncated : undefined,
+            originalSize: typeof file.metadata?.originalSize === 'number' ? file.metadata.originalSize : undefined,
+        }));
+        logger.info(`Returning summary of ${summaries.length} collected files`);
+        return summaries;
+    }
+    getFileByUrl(url) {
+        const file = this.collectedFilesCache.get(url);
+        if (file) {
+            logger.info(`Returning file: ${url} (${(file.size / 1024).toFixed(2)} KB)`);
+            return file;
+        }
+        logger.warn(`File not found: ${url}`);
+        return null;
+    }
+    getFilesByPattern(pattern, limit = 20, maxTotalSize = this.MAX_RESPONSE_SIZE) {
+        const regex = new RegExp(pattern);
+        const matched = [];
+        for (const file of this.collectedFilesCache.values()) {
+            if (regex.test(file.url)) {
+                matched.push(file);
+            }
+        }
+        const returned = [];
+        let totalSize = 0;
+        let truncated = false;
+        for (let i = 0; i < matched.length && i < limit; i++) {
+            const file = matched[i];
+            if (file && totalSize + file.size <= maxTotalSize) {
+                returned.push(file);
+                totalSize += file.size;
+            }
+            else {
+                truncated = true;
+                break;
+            }
+        }
+        if (truncated || matched.length > limit) {
+            logger.warn(`Pattern "${pattern}" matched ${matched.length} files, returning ${returned.length} (limited by size/count)`);
+        }
+        logger.info(` Pattern "${pattern}": matched ${matched.length}, returning ${returned.length} files (${(totalSize / 1024).toFixed(2)} KB)`);
+        return {
+            files: returned,
+            totalSize,
+            matched: matched.length,
+            returned: returned.length,
+            truncated,
+        };
+    }
+    getTopPriorityFiles(topN = 10, maxTotalSize = this.MAX_RESPONSE_SIZE) {
+        const allFiles = Array.from(this.collectedFilesCache.values());
+        const scoredFiles = allFiles.map((file) => ({
+            file,
+            score: calculatePriorityScore(file),
+        }));
+        scoredFiles.sort((a, b) => b.score - a.score);
+        const selected = [];
+        let totalSize = 0;
+        for (let i = 0; i < Math.min(topN, scoredFiles.length); i++) {
+            const item = scoredFiles[i];
+            if (item && item.file && totalSize + item.file.size <= maxTotalSize) {
+                selected.push(item.file);
+                totalSize += item.file.size;
+            }
+            else {
+                break;
+            }
+        }
+        logger.info(`Returning top ${selected.length}/${allFiles.length} priority files (${(totalSize / 1024).toFixed(2)} KB)`);
+        return {
+            files: selected,
+            totalSize,
+            totalFiles: allFiles.length,
+        };
+    }
+    clearCollectedFilesCache() {
+        const count = this.collectedFilesCache.size;
+        this.collectedFilesCache.clear();
+        logger.info(`Cleared collected files cache (${count} files)`);
+    }
+}

package/dist/src/modules/collector/PageController.d.ts

@@ -0,0 +1,102 @@
+import type { CodeCollector } from '../collector/CodeCollector.js';
+export interface NavigationOptions {
+    waitUntil?: 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2';
+    timeout?: number;
+}
+export interface ClickOptions {
+    button?: 'left' | 'right' | 'middle';
+    clickCount?: number;
+    delay?: number;
+}
+export interface TypeOptions {
+    delay?: number;
+}
+export interface ScrollOptions {
+    x?: number;
+    y?: number;
+}
+export interface ScreenshotClip {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface ScreenshotOptions {
+    path?: string;
+    type?: 'png' | 'jpeg';
+    quality?: number;
+    fullPage?: boolean;
+    clip?: ScreenshotClip;
+}
+interface WaitForSelectorElement {
+    tagName: string;
+    id?: string;
+    className?: string;
+    textContent?: string;
+    attributes: Record<string, string>;
+}
+export declare class PageController {
+    private collector;
+    constructor(collector: CodeCollector);
+    navigate(url: string, options?: NavigationOptions): Promise<{
+        url: string;
+        title: string;
+        loadTime: number;
+    }>;
+    reload(options?: NavigationOptions): Promise<void>;
+    goBack(): Promise<void>;
+    goForward(): Promise<void>;
+    click(selector: string, options?: ClickOptions): Promise<void>;
+    type(selector: string, text: string, options?: TypeOptions): Promise<void>;
+    select(selector: string, ...values: string[]): Promise<void>;
+    hover(selector: string): Promise<void>;
+    scroll(options: ScrollOptions): Promise<void>;
+    waitForSelector(selector: string, timeout?: number): Promise<{
+        success: boolean;
+        element?: WaitForSelectorElement | null;
+        message: string;
+    }>;
+    waitForNavigation(timeout?: number): Promise<void>;
+    evaluate<T>(code: string): Promise<T>;
+    getURL(): Promise<string>;
+    getTitle(): Promise<string>;
+    getContent(): Promise<string>;
+    screenshot(options?: ScreenshotOptions): Promise<Buffer>;
+    getPerformanceMetrics(): Promise<{
+        domContentLoaded: number;
+        loadComplete: number;
+        dns: number;
+        tcp: number;
+        request: number;
+        response: number;
+        total: number;
+        resources: number;
+    }>;
+    injectScript(scriptContent: string): Promise<void>;
+    setCookies(cookies: Array<{
+        name: string;
+        value: string;
+        domain?: string;
+        path?: string;
+        expires?: number;
+        httpOnly?: boolean;
+        secure?: boolean;
+        sameSite?: 'Strict' | 'Lax' | 'None';
+    }>): Promise<void>;
+    getCookies(): Promise<import("rebrowser-puppeteer-core").Cookie[]>;
+    clearCookies(): Promise<void>;
+    setViewport(width: number, height: number): Promise<void>;
+    emulateDevice(deviceName: string): Promise<'iPhone' | 'iPad' | 'Android'>;
+    waitForNetworkIdle(timeout?: number): Promise<void>;
+    getLocalStorage(): Promise<Record<string, string>>;
+    setLocalStorage(key: string, value: string): Promise<void>;
+    clearLocalStorage(): Promise<void>;
+    pressKey(key: string): Promise<void>;
+    uploadFile(selector: string, filePath: string): Promise<void>;
+    getAllLinks(): Promise<Array<{
+        text: string;
+        href: string;
+    }>>;
+    getPage(): Promise<import("rebrowser-puppeteer-core").Page>;
+}
+export {};