@jshookmcp/jshook 0.1.8 → 0.2.0

package/dist/modules/collector/CodeCollector.js

@@ -1,462 +0,0 @@
-import { existsSync } from 'fs';
-import puppeteer from 'rebrowser-puppeteer-core';
-import { logger } from '../../utils/logger.js';
-import { PrerequisiteError } from '../../errors/PrerequisiteError.js';
-import { CodeCache } from '../collector/CodeCache.js';
-import { SmartCodeCollector } from '../collector/SmartCodeCollector.js';
-import { CodeCompressor } from '../collector/CodeCompressor.js';
-import { calculatePriorityScore } from '../collector/PageScriptCollectors.js';
-import { findBrowserExecutable } from '../../utils/browserExecutable.js';
-import { collectInnerImpl } from '../collector/CodeCollectorCollectInternal.js';
-import { shouldCollectUrlImpl, navigateWithRetryImpl, getPerformanceMetricsImpl, collectPageMetadataImpl } from '../collector/CodeCollectorUtilsInternal.js';
-export class CodeCollector {
-    config;
-    browser = null;
-    collectedUrls = new Set();
-    initPromise = null;
-    collectLock = null;
-    MAX_COLLECTED_URLS;
-    MAX_FILES_PER_COLLECT;
-    MAX_RESPONSE_SIZE;
-    MAX_SINGLE_FILE_SIZE;
-    viewport;
-    userAgent;
-    collectedFilesCache = new Map();
-    cache;
-    cacheEnabled = true;
-    smartCollector;
-    compressor;
-    cdpSession = null;
-    cdpListeners = {};
-    activePageIndex = null;
-    currentHeadless = null;
-    constructor(config) {
-        this.config = config;
-        this.MAX_COLLECTED_URLS = config.maxCollectedUrls ?? 10000;
-        this.MAX_FILES_PER_COLLECT = config.maxFilesPerCollect ?? 200;
-        this.MAX_RESPONSE_SIZE = config.maxTotalContentSize ?? 512 * 1024;
-        this.MAX_SINGLE_FILE_SIZE = config.maxSingleFileSize ?? 200 * 1024;
-        this.viewport = config.viewport ?? { width: 1920, height: 1080 };
-        this.userAgent =
-            config.userAgent ??
-                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
-        this.cache = new CodeCache();
-        this.smartCollector = new SmartCodeCollector();
-        this.compressor = new CodeCompressor();
-        logger.info(` CodeCollector limits: maxCollect=${this.MAX_FILES_PER_COLLECT} files, maxResponse=${(this.MAX_RESPONSE_SIZE / 1024).toFixed(0)}KB, maxSingle=${(this.MAX_SINGLE_FILE_SIZE / 1024).toFixed(0)}KB`);
-        logger.info(` Strategy: Collect ALL files -> Cache -> Return summary/partial data to fit MCP limits`);
-    }
-    setCacheEnabled(enabled) {
-        this.cacheEnabled = enabled;
-        logger.info(`Code cache ${enabled ? 'enabled' : 'disabled'}`);
-    }
-    async clearFileCache() {
-        await this.cache.clear();
-    }
-    async getFileCacheStats() {
-        return await this.cache.getStats();
-    }
-    async clearAllData() {
-        logger.info('Clearing all collected data...');
-        await this.cache.clear();
-        this.compressor.clearCache();
-        this.compressor.resetStats();
-        this.collectedUrls.clear();
-        this.collectedFilesCache.clear();
-        logger.success('All data cleared');
-    }
-    async getAllStats() {
-        const cacheStats = await this.cache.getStats();
-        const compressionStats = this.compressor.getStats();
-        return {
-            cache: cacheStats,
-            compression: {
-                ...compressionStats,
-                cacheSize: this.compressor.getCacheSize(),
-            },
-            collector: {
-                collectedUrls: this.collectedUrls.size,
-                maxCollectedUrls: this.MAX_COLLECTED_URLS,
-            },
-        };
-    }
-    getCache() {
-        return this.cache;
-    }
-    getCompressor() {
-        return this.compressor;
-    }
-    cleanupCollectedUrls() {
-        if (this.collectedUrls.size > this.MAX_COLLECTED_URLS) {
-            logger.warn(`Collected URLs exceeded ${this.MAX_COLLECTED_URLS}, clearing...`);
-            const urls = Array.from(this.collectedUrls);
-            this.collectedUrls.clear();
-            urls
-                .slice(-Math.floor(this.MAX_COLLECTED_URLS / 2))
-                .forEach((url) => this.collectedUrls.add(url));
-        }
-    }
-    async init(headless) {
-        if (this.browser) {
-            return;
-        }
-        if (this.initPromise) {
-            return this.initPromise;
-        }
-        this.initPromise = this.initInner(headless);
-        try {
-            await this.initPromise;
-        }
-        finally {
-            this.initPromise = null;
-        }
-    }
-    async initInner(headless) {
-        const useHeadless = headless ?? this.config.headless;
-        const executablePath = this.resolveExecutablePath();
-        const launchOptions = {
-            headless: useHeadless,
-            args: [
-                ...(this.config.args || []),
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-blink-features=AutomationControlled',
-                '--disable-extensions',
-                '--disable-component-extensions-with-background-pages',
-                '--disable-web-security',
-                '--disable-features=IsolateOrigins,site-per-process',
-                `--window-size=${this.viewport.width},${this.viewport.height}`,
-                '--ignore-certificate-errors',
-            ],
-            defaultViewport: this.viewport,
-            protocolTimeout: 60000,
-        };
-        if (executablePath) {
-            launchOptions.executablePath = executablePath;
-        }
-        logger.info('Initializing browser with anti-detection...');
-        this.browser = await puppeteer.launch(launchOptions);
-        this.currentHeadless = useHeadless === undefined ? true : useHeadless !== false;
-        this.browser.on('disconnected', () => {
-            logger.warn('Browser disconnected');
-            this.browser = null;
-            this.currentHeadless = null;
-            if (this.cdpSession) {
-                this.cdpSession = null;
-                this.cdpListeners = {};
-            }
-        });
-        logger.success('Browser initialized with enhanced anti-detection');
-    }
-    resolveExecutablePath() {
-        const configuredPath = this.config.executablePath?.trim();
-        if (configuredPath) {
-            if (existsSync(configuredPath)) {
-                return configuredPath;
-            }
-            throw new Error(`Configured browser executable was not found: ${configuredPath}. ` +
-                'Set a valid executablePath or configure CHROME_PATH / PUPPETEER_EXECUTABLE_PATH / BROWSER_EXECUTABLE_PATH.');
-        }
-        const detectedPath = findBrowserExecutable();
-        if (detectedPath) {
-            return detectedPath;
-        }
-        logger.info('No explicit browser executable configured. Falling back to Puppeteer-managed browser resolution.');
-        return undefined;
-    }
-    async close() {
-        await this.clearAllData();
-        if (this.browser) {
-            await this.browser.close();
-            this.browser = null;
-            this.currentHeadless = null;
-            logger.info('Browser closed and all data cleared');
-        }
-    }
-    async getActivePage() {
-        if (!this.browser) {
-            try {
-                await this.init();
-            }
-            catch (error) {
-                throw new PrerequisiteError(`Browser not available: ${error instanceof Error ? error.message : String(error)}`);
-            }
-        }
-        const pages = await this.browser.pages();
-        if (pages.length === 0) {
-            return await this.browser.newPage();
-        }
-        if (this.activePageIndex !== null && this.activePageIndex < pages.length) {
-            return pages[this.activePageIndex];
-        }
-        const lastPage = pages[pages.length - 1];
-        if (!lastPage) {
-            throw new Error('Failed to get active page');
-        }
-        return lastPage;
-    }
-    async listPages() {
-        if (!this.browser) {
-            return [];
-        }
-        const pages = await this.browser.pages();
-        const results = await Promise.all(pages.map(async (page, index) => {
-            let url = '';
-            let title = '';
-            try {
-                url = page.url();
-            }
-            catch { }
-            try {
-                title = await page.title();
-            }
-            catch { }
-            return { index, url, title };
-        }));
-        return results;
-    }
-    async selectPage(index) {
-        if (!this.browser) {
-            throw new Error('Browser not connected');
-        }
-        const pages = await this.browser.pages();
-        if (index < 0 || index >= pages.length) {
-            throw new Error(`Page index ${index} out of range (0-${pages.length - 1})`);
-        }
-        this.activePageIndex = index;
-        logger.info(`Active page set to index ${index}: ${pages[index].url()}`);
-    }
-    async createPage(url) {
-        if (!this.browser) {
-            await this.init();
-        }
-        const page = await this.browser.newPage();
-        await page.setUserAgent(this.userAgent);
-        await this.applyAntiDetection(page);
-        if (url) {
-            await page.goto(url, {
-                waitUntil: 'networkidle2',
-                timeout: this.config.timeout,
-            });
-        }
-        logger.info(`New page created${url ? `: ${url}` : ''}`);
-        return page;
-    }
-    async applyAntiDetection(page) {
-        await page.evaluateOnNewDocument(() => {
-            Object.defineProperty(navigator, 'webdriver', {
-                get: () => false,
-            });
-            Object.defineProperty(navigator, 'plugins', {
-                get: () => [1, 2, 3, 4, 5],
-            });
-            Object.defineProperty(navigator, 'languages', {
-                get: () => ['en-US', 'en'],
-            });
-            const win = window;
-            if (!win.chrome) {
-                win.chrome = {
-                    runtime: {},
-                    loadTimes: function () { },
-                    csi: function () { },
-                    app: {},
-                };
-            }
-            const originalQuery = window.navigator.permissions.query;
-            window.navigator.permissions.query = (parameters) => {
-                if (parameters.name === 'notifications') {
-                    return Promise.resolve({ state: 'denied' });
-                }
-                return originalQuery(parameters);
-            };
-        });
-    }
-    async getStatus() {
-        if (!this.browser) {
-            return {
-                running: false,
-                pagesCount: 0,
-            };
-        }
-        try {
-            const pages = await this.browser.pages();
-            const version = await this.browser.version();
-            return {
-                running: true,
-                pagesCount: pages.length,
-                version,
-                effectiveHeadless: this.currentHeadless ?? undefined,
-            };
-        }
-        catch (error) {
-            logger.debug('Browser not running or disconnected:', error);
-            return {
-                running: false,
-                pagesCount: 0,
-            };
-        }
-    }
-    async collect(options) {
-        while (this.collectLock) {
-            try {
-                await this.collectLock;
-            }
-            catch { }
-        }
-        let resolve;
-        let reject;
-        this.collectLock = new Promise((res, rej) => { resolve = res; reject = rej; });
-        try {
-            const result = await this.collectInner(options);
-            resolve(result);
-            return result;
-        }
-        catch (e) {
-            reject(e);
-            throw e;
-        }
-        finally {
-            this.collectLock = null;
-        }
-    }
-    async collectInner(options) {
-        return collectInnerImpl(this, options);
-    }
-    shouldCollectUrl(url, filterRules) {
-        return shouldCollectUrlImpl(url, filterRules);
-    }
-    async navigateWithRetry(page, url, options, maxRetries = 3) {
-        return navigateWithRetryImpl(page, url, options, maxRetries);
-    }
-    async getPerformanceMetrics(page) {
-        return getPerformanceMetricsImpl(page);
-    }
-    async collectPageMetadata(page) {
-        return collectPageMetadataImpl(page);
-    }
-    async connect(endpoint) {
-        if (this.browser) {
-            try {
-                await this.browser.disconnect();
-            }
-            catch { }
-            this.browser = null;
-            this.currentHeadless = null;
-        }
-        this.activePageIndex = null;
-        logger.info(`Connecting to existing browser: ${endpoint}`);
-        const connectOptions = endpoint.startsWith('ws://') || endpoint.startsWith('wss://')
-            ? { browserWSEndpoint: endpoint }
-            : { browserURL: endpoint };
-        this.browser = await puppeteer.connect(connectOptions);
-        this.browser.on('disconnected', () => {
-            logger.warn('Browser disconnected');
-            this.browser = null;
-            this.currentHeadless = null;
-            if (this.cdpSession) {
-                this.cdpSession = null;
-                this.cdpListeners = {};
-            }
-        });
-        logger.success('Connected to existing browser successfully');
-    }
-    getBrowser() {
-        return this.browser;
-    }
-    getCollectionStats() {
-        return {
-            totalCollected: this.collectedUrls.size,
-            uniqueUrls: this.collectedUrls.size,
-        };
-    }
-    clearCache() {
-        this.collectedUrls.clear();
-        logger.info('Collection cache cleared');
-    }
-    getCollectedFilesSummary() {
-        const summaries = Array.from(this.collectedFilesCache.values()).map((file) => ({
-            url: file.url,
-            size: file.size,
-            type: file.type,
-            truncated: typeof file.metadata?.truncated === 'boolean' ? file.metadata.truncated : undefined,
-            originalSize: typeof file.metadata?.originalSize === 'number' ? file.metadata.originalSize : undefined,
-        }));
-        logger.info(`Returning summary of ${summaries.length} collected files`);
-        return summaries;
-    }
-    getFileByUrl(url) {
-        const file = this.collectedFilesCache.get(url);
-        if (file) {
-            logger.info(`Returning file: ${url} (${(file.size / 1024).toFixed(2)} KB)`);
-            return file;
-        }
-        logger.warn(`File not found: ${url}`);
-        return null;
-    }
-    getFilesByPattern(pattern, limit = 20, maxTotalSize = this.MAX_RESPONSE_SIZE) {
-        const regex = new RegExp(pattern);
-        const matched = [];
-        for (const file of this.collectedFilesCache.values()) {
-            if (regex.test(file.url)) {
-                matched.push(file);
-            }
-        }
-        const returned = [];
-        let totalSize = 0;
-        let truncated = false;
-        for (let i = 0; i < matched.length && i < limit; i++) {
-            const file = matched[i];
-            if (file && totalSize + file.size <= maxTotalSize) {
-                returned.push(file);
-                totalSize += file.size;
-            }
-            else {
-                truncated = true;
-                break;
-            }
-        }
-        if (truncated || matched.length > limit) {
-            logger.warn(`Pattern "${pattern}" matched ${matched.length} files, returning ${returned.length} (limited by size/count)`);
-        }
-        logger.info(` Pattern "${pattern}": matched ${matched.length}, returning ${returned.length} files (${(totalSize / 1024).toFixed(2)} KB)`);
-        return {
-            files: returned,
-            totalSize,
-            matched: matched.length,
-            returned: returned.length,
-            truncated,
-        };
-    }
-    getTopPriorityFiles(topN = 10, maxTotalSize = this.MAX_RESPONSE_SIZE) {
-        const allFiles = Array.from(this.collectedFilesCache.values());
-        const scoredFiles = allFiles.map((file) => ({
-            file,
-            score: calculatePriorityScore(file),
-        }));
-        scoredFiles.sort((a, b) => b.score - a.score);
-        const selected = [];
-        let totalSize = 0;
-        for (let i = 0; i < Math.min(topN, scoredFiles.length); i++) {
-            const item = scoredFiles[i];
-            if (item && item.file && totalSize + item.file.size <= maxTotalSize) {
-                selected.push(item.file);
-                totalSize += item.file.size;
-            }
-            else {
-                break;
-            }
-        }
-        logger.info(`Returning top ${selected.length}/${allFiles.length} priority files (${(totalSize / 1024).toFixed(2)} KB)`);
-        return {
-            files: selected,
-            totalSize,
-            totalFiles: allFiles.length,
-        };
-    }
-    clearCollectedFilesCache() {
-        const count = this.collectedFilesCache.size;
-        this.collectedFilesCache.clear();
-        logger.info(`Cleared collected files cache (${count} files)`);
-    }
-}

package/dist/modules/collector/PageController.d.ts

@@ -1,95 +0,0 @@
-import type { CodeCollector } from '../collector/CodeCollector.js';
-export interface NavigationOptions {
-    waitUntil?: 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2';
-    timeout?: number;
-}
-export interface ClickOptions {
-    button?: 'left' | 'right' | 'middle';
-    clickCount?: number;
-    delay?: number;
-}
-export interface TypeOptions {
-    delay?: number;
-}
-export interface ScrollOptions {
-    x?: number;
-    y?: number;
-}
-export interface ScreenshotOptions {
-    path?: string;
-    type?: 'png' | 'jpeg';
-    quality?: number;
-    fullPage?: boolean;
-}
-interface WaitForSelectorElement {
-    tagName: string;
-    id?: string;
-    className?: string;
-    textContent?: string;
-    attributes: Record<string, string>;
-}
-export declare class PageController {
-    private collector;
-    constructor(collector: CodeCollector);
-    navigate(url: string, options?: NavigationOptions): Promise<{
-        url: string;
-        title: string;
-        loadTime: number;
-    }>;
-    reload(options?: NavigationOptions): Promise<void>;
-    goBack(): Promise<void>;
-    goForward(): Promise<void>;
-    click(selector: string, options?: ClickOptions): Promise<void>;
-    type(selector: string, text: string, options?: TypeOptions): Promise<void>;
-    select(selector: string, ...values: string[]): Promise<void>;
-    hover(selector: string): Promise<void>;
-    scroll(options: ScrollOptions): Promise<void>;
-    waitForSelector(selector: string, timeout?: number): Promise<{
-        success: boolean;
-        element?: WaitForSelectorElement | null;
-        message: string;
-    }>;
-    waitForNavigation(timeout?: number): Promise<void>;
-    evaluate<T>(code: string): Promise<T>;
-    getURL(): Promise<string>;
-    getTitle(): Promise<string>;
-    getContent(): Promise<string>;
-    screenshot(options?: ScreenshotOptions): Promise<Buffer>;
-    getPerformanceMetrics(): Promise<{
-        domContentLoaded: number;
-        loadComplete: number;
-        dns: number;
-        tcp: number;
-        request: number;
-        response: number;
-        total: number;
-        resources: number;
-    }>;
-    injectScript(scriptContent: string): Promise<void>;
-    setCookies(cookies: Array<{
-        name: string;
-        value: string;
-        domain?: string;
-        path?: string;
-        expires?: number;
-        httpOnly?: boolean;
-        secure?: boolean;
-        sameSite?: 'Strict' | 'Lax' | 'None';
-    }>): Promise<void>;
-    getCookies(): Promise<import("rebrowser-puppeteer-core").Cookie[]>;
-    clearCookies(): Promise<void>;
-    setViewport(width: number, height: number): Promise<void>;
-    emulateDevice(deviceName: string): Promise<'iPhone' | 'iPad' | 'Android'>;
-    waitForNetworkIdle(timeout?: number): Promise<void>;
-    getLocalStorage(): Promise<Record<string, string>>;
-    setLocalStorage(key: string, value: string): Promise<void>;
-    clearLocalStorage(): Promise<void>;
-    pressKey(key: string): Promise<void>;
-    uploadFile(selector: string, filePath: string): Promise<void>;
-    getAllLinks(): Promise<Array<{
-        text: string;
-        href: string;
-    }>>;
-    getPage(): Promise<import("rebrowser-puppeteer-core").Page>;
-}
-export {};