@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/queue-manager.js

@@ -0,0 +1,169 @@
+import { RequestQueue, Dataset } from 'crawlee';
+import { generateDocId } from '../util/docs.js';
+import { logger } from '../util/logger.js';
+export class QueueManager {
+    requestQueue = null;
+    websiteId = '';
+    results = [];
+    static BATCH_SIZE = 20;
+    /** Optional path prefix to restrict crawling to URLs under this path */
+    pathPrefix = '';
+    /** The allowed hostname - only URLs with this exact hostname (or its subdomains) are allowed */
+    allowedHostname = '';
+    /** Count of URLs filtered due to path prefix mismatch */
+    filteredByPathCount = 0;
+    /** Count of URLs filtered due to hostname mismatch */
+    filteredByHostnameCount = 0;
+    async initialize(url, pathPrefix) {
+        const parsedUrl = new URL(url);
+        this.websiteId = generateDocId(url, parsedUrl.hostname);
+        this.pathPrefix = pathPrefix || '';
+        this.allowedHostname = parsedUrl.hostname.toLowerCase();
+        this.filteredByPathCount = 0;
+        this.filteredByHostnameCount = 0;
+        logger.info(`[QueueManager] Hostname restriction: only crawling URLs on ${this.allowedHostname} (and its subdomains)`);
+        if (this.pathPrefix) {
+            logger.info(`[QueueManager] Path restriction enabled: only crawling URLs under ${this.pathPrefix}`);
+        }
+        logger.debug(`[QueueManager] Using website ID: ${this.websiteId}`);
+        // Initialize queue
+        this.requestQueue = await RequestQueue.open(this.websiteId);
+        await this.requestQueue.drop();
+        this.requestQueue = await RequestQueue.open(this.websiteId);
+        // Add initial request (strip any hash from URL)
+        const cleanUrl = parsedUrl.origin + parsedUrl.pathname + parsedUrl.search;
+        await this.requestQueue.addRequest({
+            url: cleanUrl,
+            uniqueKey: parsedUrl.pathname + parsedUrl.search,
+        });
+        // Clear existing dataset
+        const dataset = await Dataset.open(this.websiteId);
+        await dataset.drop();
+    }
+    getFilteredByPathCount() {
+        return this.filteredByPathCount;
+    }
+    getFilteredByHostnameCount() {
+        return this.filteredByHostnameCount;
+    }
+    /**
+     * Check if a hostname matches the allowed hostname.
+     * Allows exact match or subdomains of the allowed hostname.
+     * Does NOT allow sibling subdomains or parent domains.
+     *
+     * @example
+     * If allowedHostname is 'docs.example.com':
+     * - 'docs.example.com' → true (exact match)
+     * - 'api.docs.example.com' → true (subdomain)
+     * - 'example.com' → false (parent domain)
+     * - 'python.example.com' → false (sibling subdomain)
+     */
+    isHostnameAllowed(hostname) {
+        const h = hostname.toLowerCase();
+        const allowed = this.allowedHostname;
+        // Exact match
+        if (h === allowed) {
+            return true;
+        }
+        // Subdomain of allowed hostname (e.g., api.docs.example.com when allowed is docs.example.com)
+        if (h.endsWith('.' + allowed)) {
+            return true;
+        }
+        return false;
+    }
+    async handleQueueAndLinks(enqueueLinks, log, rule) {
+        const queueInfo = await this.requestQueue.getInfo();
+        if (queueInfo) {
+            log.info('Queue status:', {
+                pendingCount: queueInfo.pendingRequestCount || 0,
+                handledCount: queueInfo.handledRequestCount || 0,
+                totalCount: queueInfo.totalRequestCount || 0,
+            });
+        }
+        // Capture values for use in transform function closures
+        const pathPrefix = this.pathPrefix;
+        const allowedHostname = this.allowedHostname;
+        const isHostnameAllowed = this.isHostnameAllowed.bind(this);
+        const incrementFilteredByPath = () => {
+            this.filteredByPathCount++;
+        };
+        const incrementFilteredByHostname = () => {
+            this.filteredByHostnameCount++;
+        };
+        const enqueueOptions = {
+            strategy: 'same-domain',
+            transformRequestFunction(req) {
+                const url = new URL(req.url);
+                // Skip URLs with hash fragments (same-page anchors)
+                // These point to sections within a page, not separate pages
+                if (url.hash) {
+                    logger.debug(`[QueueManager] Skipping anchor link: ${req.url}`);
+                    return false;
+                }
+                // Skip URLs with different hostname (stricter than same-domain strategy)
+                // This prevents crawling sibling subdomains (e.g., python.langchain.com when starting from docs.langchain.com)
+                if (!isHostnameAllowed(url.hostname)) {
+                    logger.debug(`[QueueManager] Skipping URL with different hostname: ${url.hostname} (allowed: ${allowedHostname})`);
+                    incrementFilteredByHostname();
+                    return false;
+                }
+                // Skip URLs outside the path prefix (if configured)
+                // Use precise matching: /docs matches /docs and /docs/foo but NOT /documentation
+                if (pathPrefix) {
+                    const isMatch = url.pathname === pathPrefix || url.pathname.startsWith(pathPrefix + '/');
+                    if (!isMatch) {
+                        logger.debug(`[QueueManager] Skipping URL outside path prefix: ${url.pathname} (prefix: ${pathPrefix})`);
+                        incrementFilteredByPath();
+                        return false;
+                    }
+                }
+                // Return the request with a normalized URL (without hash) and unique key
+                return {
+                    ...req,
+                    url: url.origin + url.pathname + url.search, // Strip hash from URL
+                    uniqueKey: url.pathname + url.search,
+                };
+            },
+        };
+        // Add site-specific link selectors if provided
+        if (rule.linkSelectors?.length) {
+            enqueueOptions.selector = rule.linkSelectors.join(', ');
+        }
+        const enqueueResult = await enqueueLinks(enqueueOptions);
+        log.info('Enqueued links:', {
+            processedCount: enqueueResult.processedRequests.length,
+            urls: enqueueResult.processedRequests.map((r) => r.uniqueKey),
+            ...(this.filteredByHostnameCount > 0 ? { filteredByHostname: this.filteredByHostnameCount } : {}),
+            ...(this.pathPrefix ? { pathPrefix: this.pathPrefix, filteredByPath: this.filteredByPathCount } : {}),
+        });
+    }
+    async processBatch() {
+        if (this.results.length === 0)
+            return [];
+        const dataset = await Dataset.open(this.websiteId);
+        const resultsToProcess = [...this.results];
+        this.results = [];
+        // Process in chunks for better memory management
+        for (let i = 0; i < resultsToProcess.length; i += 5) {
+            await dataset.pushData(resultsToProcess.slice(i, i + 5));
+        }
+        return resultsToProcess;
+    }
+    addResult(result) {
+        this.results.push(result);
+    }
+    hasEnoughResults() {
+        return this.results.length >= QueueManager.BATCH_SIZE;
+    }
+    getRequestQueue() {
+        return this.requestQueue;
+    }
+    async cleanup() {
+        this.results = [];
+        if (this.requestQueue) {
+            await this.requestQueue.drop().catch((err) => logger.error('Failed to drop request queue:', err));
+            this.requestQueue = null;
+        }
+    }
+}
+//# sourceMappingURL=queue-manager.js.map

package/build/crawler/queue-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,SAAS,GAAW,EAAE,CAAC;IACvB,OAAO,GAAkB,EAAE,CAAC;IAC5B,MAAM,CAAU,UAAU,GAAG,EAAE,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAW,EAAE,CAAC;IAChC,gGAAgG;IACxF,eAAe,GAAW,EAAE,CAAC;IACrC,yDAAyD;IACjD,mBAAmB,GAAW,CAAC,CAAC;IACxC,sDAAsD;IAC9C,uBAAuB,GAAW,CAAC,CAAC;IAE5C,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAmB;QAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,eAAe,uBAAuB,CAAC,CAAC;QACvH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,qEAAqE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAEnE,mBAAmB;QACnB,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE5D,gDAAgD;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC1E,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YACjC,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,iBAAiB,CAAC,QAAgB;QACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAuG,EACvG,GAAQ,EACR,IAAuB;QAEvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;gBACxB,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,UAAU,EAAE,SAAS,CAAC,iBAAiB,IAAI,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC;QAED,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,uBAAuB,GAAG,GAAG,EAAE;YACnC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC,CAAC;QACF,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACvC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,cAAc,GAAwB;YAC1C,QAAQ,EAAE,aAAgC;YAC1C,wBAAwB,CAAC,GAAG;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE7B,oDAAoD;gBACpD,4DAA4D;gBAC5D,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,wCAAwC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,+GAA+G;gBAC/G,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,wDAAwD,GAAG,CAAC,QAAQ,cAAc,eAAe,GAAG,CAAC,CAAC;oBACnH,2BAA2B,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,oDAAoD;gBACpD,iFAAiF;gBACjF,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,KAAK,UAAU,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;oBACzF,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,MAAM,CAAC,KAAK,CAAC,oDAAoD,GAAG,CAAC,QAAQ,aAAa,UAAU,GAAG,CAAC,CAAC;wBACzG,uBAAuB,EAAE,CAAC;wBAC1B,OAAO,KAAK,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,yEAAyE;gBACzE,OAAO;oBACL,GAAG,GAAG;oBACN,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,sBAAsB;oBACnE,SAAS,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;iBACrC,CAAC;YACJ,CAAC;SACF,CAAC;QAEF,+CAA+C;QAC/C,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;QAEzD,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM;YACtD,IAAI,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACpF,GAAG,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC"}

package/build/crawler/queue-manager.test.d.ts

@@ -0,0 +1 @@
+export {};