@jshookmcp/jshook 0.1.2

package/dist/modules/collector/CodeCache.d.ts

@@ -0,0 +1,39 @@
+import type { CodeFile, CollectCodeResult } from '../../types/index.js';
+export interface CacheEntry {
+    url: string;
+    files: CodeFile[];
+    totalSize: number;
+    collectTime: number;
+    timestamp: number;
+    hash: string;
+}
+export interface CacheOptions {
+    cacheDir?: string;
+    maxAge?: number;
+    maxSize?: number;
+}
+export declare class CodeCache {
+    private cacheDir;
+    private maxAge;
+    private maxSize;
+    private memoryCache;
+    private readonly MAX_MEMORY_CACHE_SIZE;
+    private writesSinceCleanup;
+    private static readonly CLEANUP_INTERVAL;
+    constructor(options?: CacheOptions);
+    init(): Promise<void>;
+    private generateKey;
+    private getCachePath;
+    private isExpired;
+    get(url: string, options?: Record<string, unknown>): Promise<CollectCodeResult | null>;
+    set(url: string, result: CollectCodeResult, options?: Record<string, unknown>): Promise<void>;
+    cleanup(): Promise<void>;
+    clear(): Promise<void>;
+    getStats(): Promise<{
+        memoryEntries: number;
+        diskEntries: number;
+        totalSize: number;
+    }>;
+    warmup(urls: string[]): Promise<void>;
+}
+//# sourceMappingURL=CodeCache.d.ts.map

package/dist/modules/collector/CodeCache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodeCache.d.ts","sourceRoot":"","sources":["../../../src/modules/collector/CodeCache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAEzE,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAsC;IAEzD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAO;IAC7C,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAM;gBAElC,OAAO,GAAE,YAAiB;IAMhC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,SAAS;IAIX,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IA4CtF,GAAG,CACP,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAChC,OAAO,CAAC,IAAI,CAAC;IAuCV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAsCxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,QAAQ,IAAI,OAAO,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IA8BI,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAS5C"}

package/dist/modules/collector/CodeCache.js

@@ -0,0 +1,198 @@
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+import { logger } from '../../utils/logger.js';
+export class CodeCache {
+    cacheDir;
+    maxAge;
+    maxSize;
+    memoryCache = new Map();
+    MAX_MEMORY_CACHE_SIZE = 100;
+    writesSinceCleanup = 0;
+    static CLEANUP_INTERVAL = 20;
+    constructor(options = {}) {
+        this.cacheDir = options.cacheDir || path.join(process.cwd(), '.cache', 'code');
+        this.maxAge = options.maxAge || 24 * 60 * 60 * 1000;
+        this.maxSize = options.maxSize || 100 * 1024 * 1024;
+    }
+    async init() {
+        try {
+            await fs.mkdir(this.cacheDir, { recursive: true });
+            logger.debug(`Cache directory initialized: ${this.cacheDir}`);
+        }
+        catch (error) {
+            logger.error('Failed to initialize cache directory:', error);
+        }
+    }
+    generateKey(url, options) {
+        const data = JSON.stringify({ url, options });
+        return crypto.createHash('md5').update(data).digest('hex');
+    }
+    getCachePath(key) {
+        return path.join(this.cacheDir, `${key}.json`);
+    }
+    isExpired(entry) {
+        return Date.now() - entry.timestamp > this.maxAge;
+    }
+    async get(url, options) {
+        const key = this.generateKey(url, options);
+        if (this.memoryCache.has(key)) {
+            const entry = this.memoryCache.get(key);
+            if (!this.isExpired(entry)) {
+                logger.debug(`Cache hit (memory): ${url}`);
+                return {
+                    files: entry.files,
+                    dependencies: { nodes: [], edges: [] },
+                    totalSize: entry.totalSize,
+                    collectTime: entry.collectTime,
+                };
+            }
+            else {
+                this.memoryCache.delete(key);
+            }
+        }
+        try {
+            const cachePath = this.getCachePath(key);
+            const data = await fs.readFile(cachePath, 'utf-8');
+            const entry = JSON.parse(data);
+            if (this.isExpired(entry)) {
+                logger.debug(`Cache expired: ${url}`);
+                await fs.unlink(cachePath);
+                return null;
+            }
+            this.memoryCache.set(key, entry);
+            logger.debug(`Cache hit (disk): ${url}`);
+            return {
+                files: entry.files,
+                dependencies: { nodes: [], edges: [] },
+                totalSize: entry.totalSize,
+                collectTime: entry.collectTime,
+            };
+        }
+        catch (err) {
+            logger.warn(`Cache read failed for ${url}: ${err instanceof Error ? err.message : String(err)}`);
+            return null;
+        }
+    }
+    async set(url, result, options) {
+        const key = this.generateKey(url, options);
+        const hash = crypto.createHash('md5').update(JSON.stringify(result.files)).digest('hex');
+        const entry = {
+            url,
+            files: result.files,
+            totalSize: result.totalSize,
+            collectTime: result.collectTime,
+            timestamp: Date.now(),
+            hash,
+        };
+        this.memoryCache.set(key, entry);
+        if (this.memoryCache.size > this.MAX_MEMORY_CACHE_SIZE) {
+            const firstKey = this.memoryCache.keys().next().value;
+            if (firstKey) {
+                this.memoryCache.delete(firstKey);
+                logger.debug(`Memory cache evicted: ${firstKey}`);
+            }
+        }
+        try {
+            const cachePath = this.getCachePath(key);
+            await fs.mkdir(this.cacheDir, { recursive: true });
+            await fs.writeFile(cachePath, JSON.stringify(entry, null, 2), 'utf-8');
+            logger.debug(`Cache saved: ${url} (${(result.totalSize / 1024).toFixed(2)} KB)`);
+        }
+        catch (error) {
+            logger.error('Failed to save cache:', error);
+        }
+        if (++this.writesSinceCleanup >= CodeCache.CLEANUP_INTERVAL) {
+            this.writesSinceCleanup = 0;
+            void this.cleanup();
+        }
+    }
+    async cleanup() {
+        try {
+            const files = await fs.readdir(this.cacheDir);
+            let totalSize = 0;
+            const entries = [];
+            for (const file of files) {
+                if (!file.endsWith('.json'))
+                    continue;
+                const filePath = path.join(this.cacheDir, file);
+                const stats = await fs.stat(filePath);
+                totalSize += stats.size;
+                entries.push({
+                    file: filePath,
+                    mtime: stats.mtime,
+                    size: stats.size,
+                });
+            }
+            if (totalSize > this.maxSize) {
+                entries.sort((a, b) => a.mtime.getTime() - b.mtime.getTime());
+                let removedSize = 0;
+                for (const entry of entries) {
+                    if (totalSize - removedSize <= this.maxSize * 0.8)
+                        break;
+                    await fs.unlink(entry.file);
+                    removedSize += entry.size;
+                    logger.debug(`Removed old cache: ${entry.file}`);
+                }
+                logger.info(`Cache cleanup: removed ${removedSize} bytes`);
+            }
+        }
+        catch (error) {
+            logger.error('Failed to cleanup cache:', error);
+        }
+    }
+    async clear() {
+        try {
+            this.memoryCache.clear();
+            const files = await fs.readdir(this.cacheDir);
+            for (const file of files) {
+                if (file.endsWith('.json')) {
+                    await fs.unlink(path.join(this.cacheDir, file));
+                }
+            }
+            logger.info('All cache cleared');
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return;
+            }
+            logger.error('Failed to clear cache:', error);
+        }
+    }
+    async getStats() {
+        try {
+            const files = await fs.readdir(this.cacheDir);
+            let totalSize = 0;
+            let diskEntries = 0;
+            for (const file of files) {
+                if (!file.endsWith('.json'))
+                    continue;
+                const filePath = path.join(this.cacheDir, file);
+                const stats = await fs.stat(filePath);
+                totalSize += stats.size;
+                diskEntries++;
+            }
+            return {
+                memoryEntries: this.memoryCache.size,
+                diskEntries,
+                totalSize,
+            };
+        }
+        catch (error) {
+            logger.error('Failed to get cache stats:', error);
+            return {
+                memoryEntries: this.memoryCache.size,
+                diskEntries: 0,
+                totalSize: 0,
+            };
+        }
+    }
+    async warmup(urls) {
+        logger.info(`Warming up cache for ${urls.length} URLs...`);
+        for (const url of urls) {
+            await this.get(url);
+        }
+        logger.info('Cache warmup completed');
+    }
+}
+//# sourceMappingURL=CodeCache.js.map

package/dist/modules/collector/CodeCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodeCache.js","sourceRoot":"","sources":["../../../src/modules/collector/CodeCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAkBvC,MAAM,OAAO,SAAS;IACZ,QAAQ,CAAS;IACjB,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,WAAW,GAA4B,IAAI,GAAG,EAAE,CAAC;IAExC,qBAAqB,GAAG,GAAG,CAAC;IACrC,kBAAkB,GAAG,CAAC,CAAC;IACvB,MAAM,CAAU,gBAAgB,GAAG,EAAE,CAAC;IAE9C,YAAY,UAAwB,EAAE;QACpC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,OAAiC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IACjD,CAAC;IAEO,SAAS,CAAC,KAAiB;QACjC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,OAAiC;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAE3C,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;gBAC3C,OAAO;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;oBACtC,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;iBAC/B,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;gBACtC,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAEjC,MAAM,CAAC,KAAK,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;YACzC,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACtC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjG,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,GAAW,EACX,MAAyB,EACzB,OAAiC;QAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzF,MAAM,KAAK,GAAe;YACxB,GAAG;YACH,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI;SACL,CAAC;QAEF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEjC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YACtD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAClC,MAAM,CAAC,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;QAGD,IAAI,EAAE,IAAI,CAAC,kBAAkB,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;YAC5D,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;YAC5B,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,OAAO,GAAuD,EAAE,CAAC;YAEvE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAE9D,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,IAAI,SAAS,GAAG,WAAW,IAAI,IAAI,CAAC,OAAO,GAAG,GAAG;wBAAE,MAAM;oBAEzD,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC5B,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC;oBAC1B,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,0BAA0B,WAAW,QAAQ,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3B,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,OAAO;YACT,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QAKZ,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC;gBACxB,WAAW,EAAE,CAAC;YAChB,CAAC;YAED,OAAO;gBACL,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBACpC,WAAW;gBACX,SAAS;aACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YAClD,OAAO;gBACL,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBACpC,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,CAAC;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAc;QACzB,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC;QAE3D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACxC,CAAC"}

package/dist/modules/collector/CodeCollector.d.ts

@@ -0,0 +1,116 @@
+import type { Browser, Page } from 'rebrowser-puppeteer-core';
+import type { CollectCodeOptions, CollectCodeResult, CodeFile, PuppeteerConfig } from '../../types/index.js';
+import { CodeCache } from '../collector/CodeCache.js';
+import { SmartCodeCollector } from '../collector/SmartCodeCollector.js';
+import { CodeCompressor } from '../collector/CodeCompressor.js';
+export declare class CodeCollector {
+    private config;
+    private browser;
+    private collectedUrls;
+    private initPromise;
+    private collectLock;
+    private readonly MAX_COLLECTED_URLS;
+    private readonly MAX_FILES_PER_COLLECT;
+    private readonly MAX_RESPONSE_SIZE;
+    private readonly MAX_SINGLE_FILE_SIZE;
+    private readonly viewport;
+    private readonly userAgent;
+    private collectedFilesCache;
+    private cache;
+    cacheEnabled: boolean;
+    smartCollector: SmartCodeCollector;
+    private compressor;
+    private cdpSession;
+    cdpListeners: {
+        responseReceived?: (params: unknown) => void;
+    };
+    private activePageIndex;
+    private currentHeadless;
+    constructor(config: PuppeteerConfig);
+    setCacheEnabled(enabled: boolean): void;
+    clearFileCache(): Promise<void>;
+    getFileCacheStats(): Promise<{
+        memoryEntries: number;
+        diskEntries: number;
+        totalSize: number;
+    }>;
+    clearAllData(): Promise<void>;
+    getAllStats(): Promise<{
+        cache: {
+            memoryEntries: number;
+            diskEntries: number;
+            totalSize: number;
+        };
+        compression: {
+            cacheSize: number;
+            totalCompressed: number;
+            totalOriginalSize: number;
+            totalCompressedSize: number;
+            averageRatio: number;
+            cacheHits: number;
+            cacheMisses: number;
+            totalTime: number;
+        };
+        collector: {
+            collectedUrls: number;
+            maxCollectedUrls: number;
+        };
+    }>;
+    getCache(): CodeCache;
+    getCompressor(): CodeCompressor;
+    cleanupCollectedUrls(): void;
+    init(headless?: boolean): Promise<void>;
+    private initInner;
+    private resolveExecutablePath;
+    close(): Promise<void>;
+    getActivePage(): Promise<Page>;
+    listPages(): Promise<Array<{
+        index: number;
+        url: string;
+        title: string;
+    }>>;
+    selectPage(index: number): Promise<void>;
+    createPage(url?: string): Promise<Page>;
+    private applyAntiDetection;
+    getStatus(): Promise<{
+        running: boolean;
+        pagesCount: number;
+        version?: string;
+        effectiveHeadless?: boolean;
+    }>;
+    collect(options: CollectCodeOptions): Promise<CollectCodeResult>;
+    private collectInner;
+    shouldCollectUrl(url: string, filterRules?: string[]): boolean;
+    navigateWithRetry(page: Page, url: string, options: NonNullable<Parameters<Page['goto']>[1]>, maxRetries?: number): Promise<void>;
+    getPerformanceMetrics(page: Page): Promise<Record<string, number>>;
+    collectPageMetadata(page: Page): Promise<Record<string, unknown>>;
+    connect(endpoint: string): Promise<void>;
+    getBrowser(): Browser | null;
+    getCollectionStats(): {
+        totalCollected: number;
+        uniqueUrls: number;
+    };
+    clearCache(): void;
+    getCollectedFilesSummary(): Array<{
+        url: string;
+        size: number;
+        type: string;
+        truncated?: boolean;
+        originalSize?: number;
+    }>;
+    getFileByUrl(url: string): CodeFile | null;
+    getFilesByPattern(pattern: string, limit?: number, maxTotalSize?: number): {
+        files: CodeFile[];
+        totalSize: number;
+        matched: number;
+        returned: number;
+        truncated: boolean;
+    };
+    getTopPriorityFiles(topN?: number, maxTotalSize?: number): {
+        files: CodeFile[];
+        totalSize: number;
+        totalFiles: number;
+    };
+    clearCollectedFilesCache(): void;
+}
+//# sourceMappingURL=CodeCollector.d.ts.map

package/dist/modules/collector/CodeCollector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodeCollector.d.ts","sourceRoot":"","sources":["../../../src/modules/collector/CodeCollector.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAc,MAAM,0BAA0B,CAAC;AAC1E,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG9G,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAiBnE,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,WAAW,CAA2C;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,mBAAmB,CAAoC;IAC/D,OAAO,CAAC,KAAK,CAAY;IAClB,YAAY,EAAE,OAAO,CAAQ;IAC7B,cAAc,EAAE,kBAAkB,CAAC;IAC1C,OAAO,CAAC,UAAU,CAAiB;IACnC,OAAO,CAAC,UAAU,CAA2B;IACtC,YAAY,EAAE;QACnB,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;KAC9C,CAAM;IACP,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,eAAe,CAAwB;gBACnC,MAAM,EAAE,eAAe;IAoBnC,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIjC,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAG/B,iBAAiB;;;;;IAGjB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAS7B,WAAW;;;;;;;;;;;;;;;;;;;;;IAeV,QAAQ,IAAI,SAAS;IAGrB,aAAa,IAAI,cAAc;IAG/B,oBAAoB,IAAI,IAAI;IAU7B,IAAI,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAe/B,SAAS;IAsCvB,OAAO,CAAC,qBAAqB;IAoBvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAuB9B,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAgB1E,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWxC,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAgB/B,kBAAkB;IA6B1B,SAAS,IAAI,OAAO,CAAC;QACzB,OAAO,EAAE,OAAO,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,CAAC;IAwBI,OAAO,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAmBxD,YAAY;IAG1B,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO;IAGxD,iBAAiB,CACrB,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACjD,UAAU,SAAI,GACb,OAAO,CAAC,IAAI,CAAC;IAGV,qBAAqB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGlE,mBAAmB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAGjE,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB9C,UAAU,IAAI,OAAO,GAAG,IAAI;IAG5B,kBAAkB,IAAI;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;KACpB;IAMD,UAAU,IAAI,IAAI;IAIlB,wBAAwB,IAAI,KAAK,CAAC;QAChC,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IAaF,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IAS1C,iBAAiB,CACf,OAAO,EAAE,MAAM,EACf,KAAK,GAAE,MAAW,EAClB,YAAY,GAAE,MAA+B,GAC5C;QACD,KAAK,EAAE,QAAQ,EAAE,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,OAAO,CAAC;KACpB;IAqCD,mBAAmB,CACjB,IAAI,GAAE,MAAW,EACjB,YAAY,GAAE,MAA+B,GAC5C;QACD,KAAK,EAAE,QAAQ,EAAE,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB;IA2BD,wBAAwB,IAAI,IAAI;CAKjC"}