@arabold/docs-mcp-server 1.4.5 → 1.5.0

package/dist/{chunk-BD7OFN4H.js → chunk-2YTVPKP5.js}

@@ -210,8 +210,8 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// src/scraper/fetcher/HttpFetcher.ts
-import axios from "axios";
+// src/utils/url.ts
+import psl from "psl";
 
 // src/utils/errors.ts
 var ScraperError = class extends Error {
@@ -231,8 +231,79 @@ var InvalidUrlError = class extends ScraperError {
     super(`Invalid URL: ${url}`, false, cause);
   }
 };
+var RedirectError = class extends ScraperError {
+  constructor(originalUrl, redirectUrl, statusCode) {
+    super(
+      `Redirect detected from ${originalUrl} to ${redirectUrl} (status: ${statusCode})`,
+      false
+    );
+    this.originalUrl = originalUrl;
+    this.redirectUrl = redirectUrl;
+    this.statusCode = statusCode;
+  }
+};
+
+// src/utils/url.ts
+var defaultNormalizerOptions = {
+  ignoreCase: true,
+  removeHash: true,
+  removeTrailingSlash: true,
+  removeQuery: false,
+  removeIndex: true
+};
+function normalizeUrl(url, options = defaultNormalizerOptions) {
+  try {
+    const parsedUrl = new URL(url);
+    const finalOptions = { ...defaultNormalizerOptions, ...options };
+    const normalized = new URL(parsedUrl.origin + parsedUrl.pathname);
+    if (finalOptions.removeIndex) {
+      normalized.pathname = normalized.pathname.replace(
+        /\/index\.(html|htm|asp|php|jsp)$/i,
+        "/"
+      );
+    }
+    if (finalOptions.removeTrailingSlash && normalized.pathname.length > 1) {
+      normalized.pathname = normalized.pathname.replace(/\/+$/, "");
+    }
+    const preservedHash = !finalOptions.removeHash ? parsedUrl.hash : "";
+    const preservedSearch = !finalOptions.removeQuery ? parsedUrl.search : "";
+    let result = normalized.origin + normalized.pathname;
+    if (preservedSearch) {
+      result += preservedSearch;
+    }
+    if (preservedHash) {
+      result += preservedHash;
+    }
+    if (finalOptions.ignoreCase) {
+      result = result.toLowerCase();
+    }
+    return result;
+  } catch {
+    return url;
+  }
+}
+function validateUrl(url) {
+  try {
+    new URL(url);
+  } catch (error) {
+    throw new InvalidUrlError(url, error instanceof Error ? error : void 0);
+  }
+}
+function hasSameHostname(urlA, urlB) {
+  return urlA.hostname.toLowerCase() === urlB.hostname.toLowerCase();
+}
+function hasSameDomain(urlA, urlB) {
+  const domainA = psl.get(urlA.hostname.toLowerCase());
+  const domainB = psl.get(urlB.hostname.toLowerCase());
+  return domainA !== null && domainA === domainB;
+}
+function isSubpath(baseUrl, targetUrl) {
+  const basePath = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : `${baseUrl.pathname}/`;
+  return targetUrl.pathname.startsWith(basePath);
+}
 
 // src/scraper/fetcher/HttpFetcher.ts
+import axios from "axios";
 var HttpFetcher = class {
   MAX_RETRIES = 6;
   BASE_DELAY = 1e3;
@@ -246,16 +317,20 @@ var HttpFetcher = class {
   async fetch(source, options) {
     const maxRetries = options?.maxRetries ?? this.MAX_RETRIES;
     const baseDelay = options?.retryDelay ?? this.BASE_DELAY;
+    const followRedirects = options?.followRedirects ?? true;
     for (let attempt = 0; attempt <= maxRetries; attempt++) {
       try {
-        const response = await axios.get(source, {
+        const config = {
           responseType: "arraybuffer",
           // For handling both text and binary
           headers: options?.headers,
           timeout: options?.timeout,
-          signal: options?.signal
+          signal: options?.signal,
           // Pass signal to axios
-        });
+          // Axios follows redirects by default, we need to explicitly disable it if needed
+          maxRedirects: followRedirects ? 5 : 0
+        };
+        const response = await axios.get(source, config);
         return {
           content: response.data,
           mimeType: response.headers["content-type"] || "application/octet-stream",
@@ -266,6 +341,12 @@ var HttpFetcher = class {
         const axiosError = error;
         const status = axiosError.response?.status;
         const code = axiosError.code;
+        if (!followRedirects && status && status >= 300 && status < 400) {
+          const location = axiosError.response?.headers?.location;
+          if (location) {
+            throw new RedirectError(source, location, status);
+          }
+        }
         if (attempt < maxRetries && (status === void 0 || status >= 500 && status < 600)) {
           const delay = baseDelay * 2 ** attempt;
           logger.warn(
@@ -355,53 +436,6 @@ var CancellationError = class extends PipelineError {
   }
 };
 
-// src/utils/url.ts
-var defaultNormalizerOptions = {
-  ignoreCase: true,
-  removeHash: true,
-  removeTrailingSlash: true,
-  removeQuery: false,
-  removeIndex: true
-};
-function normalizeUrl(url, options = defaultNormalizerOptions) {
-  try {
-    const parsedUrl = new URL(url);
-    const finalOptions = { ...defaultNormalizerOptions, ...options };
-    const normalized = new URL(parsedUrl.origin + parsedUrl.pathname);
-    if (finalOptions.removeIndex) {
-      normalized.pathname = normalized.pathname.replace(
-        /\/index\.(html|htm|asp|php|jsp)$/i,
-        "/"
-      );
-    }
-    if (finalOptions.removeTrailingSlash && normalized.pathname.length > 1) {
-      normalized.pathname = normalized.pathname.replace(/\/+$/, "");
-    }
-    const preservedHash = !finalOptions.removeHash ? parsedUrl.hash : "";
-    const preservedSearch = !finalOptions.removeQuery ? parsedUrl.search : "";
-    let result = normalized.origin + normalized.pathname;
-    if (preservedSearch) {
-      result += preservedSearch;
-    }
-    if (preservedHash) {
-      result += preservedHash;
-    }
-    if (finalOptions.ignoreCase) {
-      result = result.toLowerCase();
-    }
-    return result;
-  } catch {
-    return url;
-  }
-}
-function validateUrl(url) {
-  try {
-    new URL(url);
-  } catch (error) {
-    throw new InvalidUrlError(url, error instanceof Error ? error : void 0);
-  }
-}
-
 // src/scraper/processor/HtmlProcessor.ts
 import createDOMPurify from "dompurify";
 import { JSDOM } from "jsdom";
@@ -736,11 +770,18 @@ var WebScraperStrategy = class extends BaseScraperStrategy {
       return false;
     }
   }
-  isSubpage(baseUrl, targetUrl) {
+  /**
+   * Determines if a target URL should be followed based on the scope setting.
+   */
+  isInScope(baseUrl, targetUrl, scope) {
     try {
-      const basePath = baseUrl.origin + baseUrl.pathname;
-      const targetPath = targetUrl.origin + targetUrl.pathname;
-      return targetPath.startsWith(basePath);
+      if (scope === "domain") {
+        return hasSameDomain(baseUrl, targetUrl);
+      }
+      if (scope === "hostname") {
+        return hasSameHostname(baseUrl, targetUrl);
+      }
+      return hasSameHostname(baseUrl, targetUrl) && isSubpath(baseUrl, targetUrl);
     } catch {
       return false;
     }
@@ -748,17 +789,19 @@ var WebScraperStrategy = class extends BaseScraperStrategy {
   async processItem(item, options, _progressCallback, signal) {
     const { url } = item;
     try {
-      const rawContent = await this.httpFetcher.fetch(url, { signal });
+      const fetchOptions = {
+        signal,
+        followRedirects: options.followRedirects
+      };
+      const rawContent = await this.httpFetcher.fetch(url, fetchOptions);
       const processor = this.getProcessor(rawContent.mimeType);
       const result = await processor.process(rawContent);
       const baseUrl = new URL(options.url);
       const links = result.links.filter((link) => {
         try {
           const targetUrl = new URL(link, baseUrl);
-          if (targetUrl.origin !== baseUrl.origin) {
-            return false;
-          }
-          return (!options.subpagesOnly || this.isSubpage(baseUrl, targetUrl)) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
+          const scope = options.scope || "subpages";
+          return this.isInScope(baseUrl, targetUrl, scope) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
         } catch {
           return false;
         }
@@ -1460,7 +1503,8 @@ var ScrapeTool = class {
       url,
       library,
       version: internalVersion,
-      subpagesOnly: scraperOptions?.subpagesOnly ?? true,
+      scope: scraperOptions?.scope ?? "subpages",
+      followRedirects: scraperOptions?.followRedirects ?? true,
       maxPages: scraperOptions?.maxPages ?? 100,
       maxDepth: scraperOptions?.maxDepth ?? 3,
       // maxConcurrency is handled by the manager itself now
@@ -11524,4 +11568,4 @@ export {
   RemoveTool,
   DocumentManagementService
 };
-//# sourceMappingURL=chunk-BD7OFN4H.js.map
+//# sourceMappingURL=chunk-2YTVPKP5.js.map