@arabold/docs-mcp-server 1.8.0 → 1.9.0

package/README.md

@@ -289,7 +289,7 @@ docs-cli scrape <library> <url> [options]
 - `-v, --version <string>`: The specific version to associate with the scraped documents.
   - Accepts full versions (`1.2.3`), pre-release versions (`1.2.3-beta.1`), or partial versions (`1`, `1.2` which are expanded to `1.0.0`, `1.2.0`).
   - If omitted, the documentation is indexed as **unversioned**.
-- `-p, --max-pages <number>`: Maximum pages to scrape (default: 100).
+- `-p, --max-pages <number>`: Maximum pages to scrape (default: 1000).
 - `-d, --max-depth <number>`: Maximum navigation depth (default: 3).
 - `-c, --max-concurrency <number>`: Maximum concurrent requests (default: 3).
 - `--ignore-errors`: Ignore errors during scraping (default: true).

package/dist/{chunk-ADZQJG2M.js → chunk-A5FW7XVC.js}

@@ -100,6 +100,11 @@ var require_extend = __commonJS({
   }
 });
 
+// src/config.ts
+var DEFAULT_MAX_PAGES = 1e3;
+var DEFAULT_MAX_DEPTH = 3;
+var DEFAULT_MAX_CONCURRENCY = 3;
+
 // src/utils/logger.ts
 var currentLogLevel = 2 /* INFO */;
 function setLogLevel(level) {
@@ -421,9 +426,8 @@ ${cleanedContent}
       );
     }
     const htmlContent = typeof content3.content === "string" ? content3.content : content3.content.toString(content3.encoding || "utf-8");
-    const titleMatch = htmlContent.match(/<title>([^<]+)<\/title>/i);
-    const title = titleMatch?.[1] || "Untitled";
-    const window = new JSDOM(content3.content, { url: content3.source }).window;
+    const window = new JSDOM(htmlContent, { url: content3.source }).window;
+    const title = window.document.title || "Untitled";
     const purify = createDOMPurify(window);
     const purifiedContent = purify.sanitize(htmlContent, {
       WHOLE_DOCUMENT: true,
@@ -629,8 +633,8 @@ var CancellationError = class extends PipelineError {
 };
 
 // src/scraper/strategies/BaseScraperStrategy.ts
-var DEFAULT_MAX_PAGES = 100;
-var DEFAULT_MAX_DEPTH = 3;
+var DEFAULT_MAX_PAGES2 = 100;
+var DEFAULT_MAX_DEPTH2 = 3;
 var DEFAULT_CONCURRENCY = 3;
 var BaseScraperStrategy = class {
   visited = /* @__PURE__ */ new Set();
@@ -651,7 +655,7 @@ var BaseScraperStrategy = class {
         if (signal?.aborted) {
           throw new CancellationError("Scraping cancelled during batch processing");
         }
-        const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
+        const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH2;
         if (item.depth > maxDepth) {
           return [];
         }
@@ -659,7 +663,7 @@ var BaseScraperStrategy = class {
           const result = await this.processItem(item, options, void 0, signal);
           if (result.document) {
             this.pageCount++;
-            const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+            const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES2;
             logger.info(
               `\u{1F310} Scraping page ${this.pageCount}/${maxPages} (depth ${item.depth}/${maxDepth}): ${item.url}`
             );
@@ -711,7 +715,7 @@ var BaseScraperStrategy = class {
     const baseUrl = new URL2(options.url);
     const queue = [{ url: options.url, depth: 0 }];
     this.visited.add(normalizeUrl(options.url, this.options.urlNormalizerOptions));
-    const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+    const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES2;
     const maxConcurrency = options.maxConcurrency ?? DEFAULT_CONCURRENCY;
     while (queue.length > 0 && this.pageCount < maxPages) {
       if (signal?.aborted) {
@@ -1489,9 +1493,9 @@ var ScrapeTool = class {
       version: internalVersion,
       scope: scraperOptions?.scope ?? "subpages",
       followRedirects: scraperOptions?.followRedirects ?? true,
-      maxPages: scraperOptions?.maxPages ?? 100,
-      maxDepth: scraperOptions?.maxDepth ?? 3,
-      // maxConcurrency is handled by the manager itself now
+      maxPages: scraperOptions?.maxPages ?? DEFAULT_MAX_PAGES,
+      maxDepth: scraperOptions?.maxDepth ?? DEFAULT_MAX_DEPTH,
+      maxConcurrency: scraperOptions?.maxConcurrency ?? DEFAULT_MAX_CONCURRENCY,
       ignoreErrors: scraperOptions?.ignoreErrors ?? true
     });
     logger.info(`\u{1F680} Job ${jobId} enqueued for scraping.`);
@@ -11566,6 +11570,9 @@ var DocumentManagementService = class {
 };
 
 export {
+  DEFAULT_MAX_PAGES,
+  DEFAULT_MAX_DEPTH,
+  DEFAULT_MAX_CONCURRENCY,
   setLogLevel,
   logger,
   HttpFetcher,
@@ -11585,4 +11592,4 @@ export {
   SearchTool,
   DocumentManagementService
 };
-//# sourceMappingURL=chunk-ADZQJG2M.js.map
+//# sourceMappingURL=chunk-A5FW7XVC.js.map