@jshookmcp/jshook 0.2.5 → 0.2.7

package/dist/src/modules/collector/CodeCache.js

@@ -1,204 +0,0 @@
-import fs from 'fs/promises';
-import path from 'path';
-import crypto from 'crypto';
-import { logger } from '../../utils/logger.js';
-export class CodeCache {
-    cacheDir;
-    maxAge;
-    maxSize;
-    memoryCache = new Map();
-    MAX_MEMORY_CACHE_SIZE = 100;
-    writesSinceCleanup = 0;
-    static CLEANUP_INTERVAL = 20;
-    constructor(options = {}) {
-        this.cacheDir = options.cacheDir || path.join(process.cwd(), '.cache', 'code');
-        this.maxAge = options.maxAge || 24 * 60 * 60 * 1000;
-        this.maxSize = options.maxSize || 100 * 1024 * 1024;
-    }
-    async init() {
-        try {
-            await fs.mkdir(this.cacheDir, { recursive: true });
-            logger.debug(`Cache directory initialized: ${this.cacheDir}`);
-        }
-        catch (error) {
-            logger.error('Failed to initialize cache directory:', error);
-        }
-    }
-    generateKey(url, options) {
-        const data = JSON.stringify({ url, options });
-        return crypto.createHash('md5').update(data).digest('hex');
-    }
-    getCachePath(key) {
-        return path.join(this.cacheDir, `${key}.json`);
-    }
-    getDependenciesOrEmpty(dependencies) {
-        return dependencies ?? { nodes: [], edges: [] };
-    }
-    isExpired(entry) {
-        return Date.now() - entry.timestamp > this.maxAge;
-    }
-    async get(url, options) {
-        const key = this.generateKey(url, options);
-        if (this.memoryCache.has(key)) {
-            const entry = this.memoryCache.get(key);
-            if (!this.isExpired(entry)) {
-                logger.debug(`Cache hit (memory): ${url}`);
-                return {
-                    files: entry.files,
-                    dependencies: this.getDependenciesOrEmpty(entry.dependencies),
-                    totalSize: entry.totalSize,
-                    collectTime: entry.collectTime,
-                    summaries: entry.summaries,
-                };
-            }
-            else {
-                this.memoryCache.delete(key);
-            }
-        }
-        try {
-            const cachePath = this.getCachePath(key);
-            const data = await fs.readFile(cachePath, 'utf-8');
-            const entry = JSON.parse(data);
-            if (this.isExpired(entry)) {
-                logger.debug(`Cache expired: ${url}`);
-                await fs.unlink(cachePath);
-                return null;
-            }
-            this.memoryCache.set(key, entry);
-            logger.debug(`Cache hit (disk): ${url}`);
-            return {
-                files: entry.files,
-                dependencies: this.getDependenciesOrEmpty(entry.dependencies),
-                totalSize: entry.totalSize,
-                collectTime: entry.collectTime,
-                summaries: entry.summaries,
-            };
-        }
-        catch (err) {
-            logger.warn(`Cache read failed for ${url}: ${err instanceof Error ? err.message : String(err)}`);
-            return null;
-        }
-    }
-    async set(url, result, options) {
-        const key = this.generateKey(url, options);
-        const hash = crypto.createHash('md5').update(JSON.stringify(result.files)).digest('hex');
-        const entry = {
-            url,
-            files: result.files,
-            dependencies: result.dependencies,
-            totalSize: result.totalSize,
-            collectTime: result.collectTime,
-            summaries: result.summaries,
-            timestamp: Date.now(),
-            hash,
-        };
-        this.memoryCache.set(key, entry);
-        if (this.memoryCache.size > this.MAX_MEMORY_CACHE_SIZE) {
-            const firstKey = this.memoryCache.keys().next().value;
-            if (firstKey) {
-                this.memoryCache.delete(firstKey);
-                logger.debug(`Memory cache evicted: ${firstKey}`);
-            }
-        }
-        try {
-            const cachePath = this.getCachePath(key);
-            await fs.mkdir(this.cacheDir, { recursive: true });
-            await fs.writeFile(cachePath, JSON.stringify(entry, null, 2), 'utf-8');
-            logger.debug(`Cache saved: ${url} (${(result.totalSize / 1024).toFixed(2)} KB)`);
-        }
-        catch (error) {
-            logger.error('Failed to save cache:', error);
-        }
-        if (++this.writesSinceCleanup >= CodeCache.CLEANUP_INTERVAL) {
-            this.writesSinceCleanup = 0;
-            void this.cleanup();
-        }
-    }
-    async cleanup() {
-        try {
-            const files = await fs.readdir(this.cacheDir);
-            let totalSize = 0;
-            const entries = [];
-            for (const file of files) {
-                if (!file.endsWith('.json'))
-                    continue;
-                const filePath = path.join(this.cacheDir, file);
-                const stats = await fs.stat(filePath);
-                totalSize += stats.size;
-                entries.push({
-                    file: filePath,
-                    mtime: stats.mtime,
-                    size: stats.size,
-                });
-            }
-            if (totalSize > this.maxSize) {
-                entries.sort((a, b) => a.mtime.getTime() - b.mtime.getTime());
-                let removedSize = 0;
-                for (const entry of entries) {
-                    if (totalSize - removedSize <= this.maxSize * 0.8)
-                        break;
-                    await fs.unlink(entry.file);
-                    removedSize += entry.size;
-                    logger.debug(`Removed old cache: ${entry.file}`);
-                }
-                logger.info(`Cache cleanup: removed ${removedSize} bytes`);
-            }
-        }
-        catch (error) {
-            logger.error('Failed to cleanup cache:', error);
-        }
-    }
-    async clear() {
-        try {
-            this.memoryCache.clear();
-            const files = await fs.readdir(this.cacheDir);
-            for (const file of files) {
-                if (file.endsWith('.json')) {
-                    await fs.unlink(path.join(this.cacheDir, file));
-                }
-            }
-            logger.info('All cache cleared');
-        }
-        catch (error) {
-            if (error.code === 'ENOENT') {
-                return;
-            }
-            logger.error('Failed to clear cache:', error);
-        }
-    }
-    async getStats() {
-        try {
-            const files = await fs.readdir(this.cacheDir);
-            let totalSize = 0;
-            let diskEntries = 0;
-            for (const file of files) {
-                if (!file.endsWith('.json'))
-                    continue;
-                const filePath = path.join(this.cacheDir, file);
-                const stats = await fs.stat(filePath);
-                totalSize += stats.size;
-                diskEntries++;
-            }
-            return {
-                memoryEntries: this.memoryCache.size,
-                diskEntries,
-                totalSize,
-            };
-        }
-        catch (error) {
-            logger.error('Failed to get cache stats:', error);
-            return {
-                memoryEntries: this.memoryCache.size,
-                diskEntries: 0,
-                totalSize: 0,
-            };
-        }
-    }
-    async warmup(urls) {
-        logger.info(`Warming up cache for ${urls.length} URLs...`);
-        for (const url of urls) {
-            await this.get(url);
-        }
-        logger.info('Cache warmup completed');
-    }
-}

package/dist/src/modules/collector/CodeCollector.d.ts

@@ -1,154 +0,0 @@
-import type { Browser, Page } from 'rebrowser-puppeteer-core';
-import type { CollectCodeOptions, CollectCodeResult, CodeFile, PuppeteerConfig } from '../../types/index.js';
-import { CodeCache } from '../collector/CodeCache.js';
-import { SmartCodeCollector } from '../collector/SmartCodeCollector.js';
-import { CodeCompressor } from '../collector/CodeCompressor.js';
-type ChromeReleaseChannel = 'stable' | 'beta' | 'dev' | 'canary';
-export interface ChromeConnectOptions {
-    browserURL?: string;
-    wsEndpoint?: string;
-    autoConnect?: boolean;
-    channel?: ChromeReleaseChannel;
-    userDataDir?: string;
-}
-export interface ResolvedPageDescriptor {
-    index: number;
-    url: string;
-    title: string;
-    page: Page;
-}
-export declare class CodeCollector {
-    protected config: PuppeteerConfig;
-    private browser;
-    protected collectedUrls: Set<string>;
-    private initPromise;
-    private collectLock;
-    private connectAttemptId;
-    protected readonly MAX_COLLECTED_URLS: number;
-    protected readonly MAX_FILES_PER_COLLECT: number;
-    protected readonly MAX_RESPONSE_SIZE: number;
-    protected readonly MAX_SINGLE_FILE_SIZE: number;
-    private readonly CONNECT_TIMEOUT_MS;
-    protected readonly viewport: {
-        width: number;
-        height: number;
-    };
-    protected readonly userAgent: string;
-    protected collectedFilesCache: Map<string, CodeFile>;
-    private cache;
-    cacheEnabled: boolean;
-    smartCollector: SmartCodeCollector;
-    private compressor;
-    private cdpSession;
-    cdpListeners: {
-        responseReceived?: (params: unknown) => void;
-    };
-    private activePageIndex;
-    private currentHeadless;
-    private explicitlyClosed;
-    private connectedToExistingBrowser;
-    private chromePid;
-    private static readonly BROWSER_CLOSE_TIMEOUT_MS;
-    constructor(config: PuppeteerConfig);
-    setCacheEnabled(enabled: boolean): void;
-    clearFileCache(): Promise<void>;
-    getFileCacheStats(): Promise<{
-        memoryEntries: number;
-        diskEntries: number;
-        totalSize: number;
-    }>;
-    clearAllData(): Promise<void>;
-    getAllStats(): Promise<{
-        cache: {
-            memoryEntries: number;
-            diskEntries: number;
-            totalSize: number;
-        };
-        compression: {
-            cacheSize: number;
-            totalCompressed: number;
-            totalOriginalSize: number;
-            totalCompressedSize: number;
-            averageRatio: number;
-            cacheHits: number;
-            cacheMisses: number;
-            totalTime: number;
-        };
-        collector: {
-            collectedUrls: number;
-            maxCollectedUrls: number;
-        };
-    }>;
-    getCache(): CodeCache;
-    getCompressor(): CodeCompressor;
-    cleanupCollectedUrls(): void;
-    init(headless?: boolean): Promise<void>;
-    private initInner;
-    private resolveExecutablePath;
-    close(): Promise<void>;
-    private closeBrowserWithForceKill;
-    static forceKillPid(pid: number | null): void;
-    getChromePid(): number | null;
-    private getPageTargets;
-    private resolvePageTargetHandle;
-    isExistingBrowserConnection(): boolean;
-    getActivePage(): Promise<Page>;
-    listPages(): Promise<Array<{
-        index: number;
-        url: string;
-        title: string;
-    }>>;
-    listResolvedPages(timeoutMs?: number): Promise<ResolvedPageDescriptor[]>;
-    selectPage(index: number): Promise<void>;
-    createPage(url?: string): Promise<Page>;
-    private applyAntiDetection;
-    getStatus(): Promise<{
-        running: boolean;
-        pagesCount: number;
-        version?: string;
-        effectiveHeadless?: boolean;
-    }>;
-    collect(options: CollectCodeOptions): Promise<CollectCodeResult>;
-    private collectInner;
-    shouldCollectUrl(url: string, filterRules?: string[]): boolean;
-    navigateWithRetry(page: Page, url: string, options: NonNullable<Parameters<Page['goto']>[1]>, maxRetries?: number): Promise<void>;
-    getPerformanceMetrics(page: Page): Promise<Record<string, number>>;
-    collectPageMetadata(page: Page): Promise<Record<string, unknown>>;
-    private resolveDefaultChromeUserDataDir;
-    private resolveAutoConnectWsEndpoint;
-    private resolveConnectOptions;
-    private isAutoConnectRequest;
-    private getUnknownErrorMessage;
-    private normalizeConnectError;
-    private buildConnectTimeoutError;
-    private connectWithTimeout;
-    connect(endpointOrOptions: string | ChromeConnectOptions): Promise<void>;
-    getBrowser(): Browser | null;
-    getCollectionStats(): {
-        totalCollected: number;
-        uniqueUrls: number;
-    };
-    clearCache(): void;
-    getCollectedFilesSummary(): Array<{
-        url: string;
-        size: number;
-        type: string;
-        truncated?: boolean;
-        originalSize?: number;
-    }>;
-    getFileByUrl(url: string): CodeFile | null;
-    getFilesByPattern(pattern: string, limit?: number, maxTotalSize?: number): {
-        files: CodeFile[];
-        totalSize: number;
-        matched: number;
-        returned: number;
-        truncated: boolean;
-    };
-    getTopPriorityFiles(topN?: number, maxTotalSize?: number): {
-        files: CodeFile[];
-        totalSize: number;
-        totalFiles: number;
-    };
-    clearCollectedFilesCache(): void;
-}
-export {};