@arabold/docs-mcp-server 1.19.0 → 1.20.0

package/dist/index.js

@@ -41,7 +41,7 @@ import "better-sqlite3";
 import "sqlite-vec";
 import { execSync } from "node:child_process";
 import { v4 } from "uuid";
-import psl from "psl";
+import "psl";
 import { minimatch } from "minimatch";
 const LogLevel = {
   ERROR: 0,
@@ -101,7 +101,7 @@ const logger = {
     }
   }
 };
-const version = "1.18.0";
+const version = "1.19.0";
 const packageJson = {
   version
 };
@@ -328,14 +328,43 @@ class HtmlLinkExtractorMiddleware {
       return;
     }
     try {
+      let docBase = context.source;
+      try {
+        const baseEl = $("base[href]").first();
+        const rawBase = baseEl.attr("href");
+        if (rawBase && rawBase.trim() !== "") {
+          try {
+            const trimmed = rawBase.trim();
+            const candidate = new URL(trimmed, context.source);
+            const hasScheme = /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(trimmed);
+            const protocolRelative = trimmed.startsWith("//");
+            const firstSlash = trimmed.indexOf("/");
+            const firstColon = trimmed.indexOf(":");
+            const colonBeforeSlash = firstColon !== -1 && (firstSlash === -1 || firstColon < firstSlash);
+            const suspiciousColon = colonBeforeSlash && !hasScheme && !protocolRelative;
+            if (suspiciousColon || trimmed.startsWith(":")) {
+              logger.debug(
+                `Ignoring suspicious <base href> value (colon misuse): ${rawBase}`
+              );
+            } else {
+              docBase = candidate.href;
+            }
+          } catch {
+            logger.debug(`Ignoring invalid <base href> value: ${rawBase}`);
+          }
+        }
+      } catch {
+      }
       const linkElements = $("a[href]");
-      logger.debug(`Found ${linkElements.length} potential links in ${context.source}`);
+      logger.debug(
+        `Found ${linkElements.length} potential links in ${context.source} (base=${docBase})`
+      );
       const extractedLinks = [];
       linkElements.each((_index, element) => {
         const href = $(element).attr("href");
         if (href && href.trim() !== "") {
           try {
-            const urlObj = new URL(href, context.source);
+            const urlObj = new URL(href, docBase);
             if (!["http:", "https:", "file:"].includes(urlObj.protocol)) {
               logger.debug(`Ignoring link with invalid protocol: ${href}`);
               return;
@@ -2094,12 +2123,18 @@ class HttpFetcher {
         } else {
           content = Buffer.from(response.data);
         }
+        const finalUrl = (
+          // Node follow-redirects style
+          response.request?.res?.responseUrl || // Some adapters may expose directly
+          response.request?.responseUrl || // Fallback to axios recorded config URL
+          response.config?.url || source
+        );
         return {
           content,
           mimeType,
           charset,
           encoding: contentEncoding,
-          source
+          source: finalUrl
         };
       } catch (error) {
         const axiosError = error;
@@ -4459,17 +4494,33 @@ function validateUrl(url) {
     throw new InvalidUrlError(url, error instanceof Error ? error : void 0);
   }
 }
-function hasSameHostname(urlA, urlB) {
-  return urlA.hostname.toLowerCase() === urlB.hostname.toLowerCase();
-}
-function hasSameDomain(urlA, urlB) {
-  const domainA = psl.get(urlA.hostname.toLowerCase());
-  const domainB = psl.get(urlB.hostname.toLowerCase());
-  return domainA !== null && domainA === domainB;
+function computeBaseDirectory(pathname) {
+  if (pathname === "") return "/";
+  if (pathname.endsWith("/")) return pathname;
+  const lastSegment = pathname.split("/").at(-1) || "";
+  const looksLikeFile = lastSegment.includes(".");
+  if (looksLikeFile) {
+    return pathname.replace(/\/[^/]*$/, "/");
+  }
+  return `${pathname}/`;
 }
-function isSubpath(baseUrl, targetUrl) {
-  const basePath = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : `${baseUrl.pathname}/`;
-  return targetUrl.pathname.startsWith(basePath);
+function isInScope(baseUrl, targetUrl, scope) {
+  if (baseUrl.protocol !== targetUrl.protocol) return false;
+  switch (scope) {
+    case "subpages": {
+      if (baseUrl.hostname !== targetUrl.hostname) return false;
+      const baseDir = computeBaseDirectory(baseUrl.pathname);
+      return targetUrl.pathname.startsWith(baseDir);
+    }
+    case "hostname":
+      return baseUrl.hostname === targetUrl.hostname;
+    case "domain": {
+      const getDomain = (host) => host.split(".").slice(-2).join(".");
+      return getDomain(baseUrl.hostname) === getDomain(targetUrl.hostname);
+    }
+    default:
+      return false;
+  }
 }
 function isRegexPattern(pattern) {
   return pattern.length > 2 && pattern.startsWith("/") && pattern.endsWith("/");
@@ -4517,24 +4568,6 @@ function shouldIncludeUrl(url, includePatterns, excludePatterns) {
   if (!includePatterns || includePatterns.length === 0) return true;
   return matchesAnyPattern(normalizedPath, includePatterns) || (basename ? matchesAnyPattern(basename, stripSlash(includePatterns)) : false);
 }
-function isInScope(baseUrl, targetUrl, scope) {
-  if (baseUrl.protocol !== targetUrl.protocol) return false;
-  switch (scope) {
-    case "subpages": {
-      if (baseUrl.hostname !== targetUrl.hostname) return false;
-      const baseDir = baseUrl.pathname.endsWith("/") ? baseUrl.pathname : baseUrl.pathname.replace(/\/[^/]*$/, "/");
-      return targetUrl.pathname.startsWith(baseDir);
-    }
-    case "hostname":
-      return baseUrl.hostname === targetUrl.hostname;
-    case "domain": {
-      const getDomain = (host) => host.split(".").slice(-2).join(".");
-      return getDomain(baseUrl.hostname) === getDomain(targetUrl.hostname);
-    }
-    default:
-      return false;
-  }
-}
 const DEFAULT_MAX_DEPTH = 3;
 const DEFAULT_CONCURRENCY = 3;
 class BaseScraperStrategy {
@@ -4543,6 +4576,8 @@ class BaseScraperStrategy {
   totalDiscovered = 0;
   // Track total URLs discovered (unlimited)
   effectiveTotal = 0;
+  // Track effective total (limited by maxPages)
+  canonicalBaseUrl;
   options;
   constructor(options = {}) {
     this.options = options;
@@ -4554,7 +4589,7 @@ class BaseScraperStrategy {
   shouldProcessUrl(url, options) {
     if (options.scope) {
       try {
-        const base = new URL$1(options.url);
+        const base = this.canonicalBaseUrl ?? new URL$1(options.url);
         const target = new URL$1(url);
         if (!isInScope(base, target, options.scope)) return false;
       } catch {
@@ -4577,6 +4612,23 @@ class BaseScraperStrategy {
         }
         try {
           const result = await this.processItem(item, options, void 0, signal);
+          if (item.depth === 0 && !this.canonicalBaseUrl && result?.finalUrl) {
+            try {
+              const finalUrlStr = result.finalUrl;
+              const original = new URL$1(options.url);
+              const finalUrlObj = new URL$1(finalUrlStr);
+              if (finalUrlObj.href !== original.href && (finalUrlObj.protocol === "http:" || finalUrlObj.protocol === "https:")) {
+                this.canonicalBaseUrl = finalUrlObj;
+                logger.debug(
+                  `Updated scope base after redirect: ${original.href} -> ${finalUrlObj.href}`
+                );
+              } else {
+                this.canonicalBaseUrl = original;
+              }
+            } catch {
+              this.canonicalBaseUrl = new URL$1(options.url);
+            }
+          }
           if (result.document) {
             this.pageCount++;
             logger.info(
@@ -4637,7 +4689,8 @@ class BaseScraperStrategy {
     this.pageCount = 0;
     this.totalDiscovered = 1;
     this.effectiveTotal = 1;
-    const baseUrl = new URL$1(options.url);
+    this.canonicalBaseUrl = new URL$1(options.url);
+    let baseUrl = this.canonicalBaseUrl;
     const queue = [{ url: options.url, depth: 0 }];
     this.visited.add(normalizeUrl(options.url, this.options.urlNormalizerOptions));
     const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
@@ -4658,6 +4711,7 @@ class BaseScraperStrategy {
         queue.length
       );
       const batch = queue.splice(0, batchSize);
+      baseUrl = this.canonicalBaseUrl ?? baseUrl;
       const newUrls = await this.processBatch(
         batch,
         baseUrl,
@@ -4690,22 +4744,7 @@ class WebScraperStrategy extends BaseScraperStrategy {
       return false;
     }
   }
-  /**
-   * Determines if a target URL should be followed based on the scope setting.
-   */
-  isInScope(baseUrl, targetUrl, scope) {
-    try {
-      if (scope === "domain") {
-        return hasSameDomain(baseUrl, targetUrl);
-      }
-      if (scope === "hostname") {
-        return hasSameHostname(baseUrl, targetUrl);
-      }
-      return hasSameHostname(baseUrl, targetUrl) && isSubpath(baseUrl, targetUrl);
-    } catch {
-      return false;
-    }
-  }
+  // Removed custom isInScope logic; using shared scope utility for consistent behavior
   /**
    * Processes a single queue item by fetching its content and processing it through pipelines.
    * @param item - The queue item to process.
@@ -4746,12 +4785,12 @@ class WebScraperStrategy extends BaseScraperStrategy {
         );
         return { document: void 0, links: processed.links };
       }
-      const baseUrl = new URL(options.url);
+      const baseUrl = item.depth === 0 ? new URL(rawContent.source) : this.canonicalBaseUrl ?? new URL(options.url);
       const filteredLinks = processed.links.filter((link) => {
         try {
           const targetUrl = new URL(link);
           const scope = options.scope || "subpages";
-          return this.isInScope(baseUrl, targetUrl, scope) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
+          return isInScope(baseUrl, targetUrl, scope) && (!this.shouldFollowLinkFn || this.shouldFollowLinkFn(baseUrl, targetUrl));
         } catch {
           return false;
         }
@@ -4767,7 +4806,8 @@ class WebScraperStrategy extends BaseScraperStrategy {
             ...processed.metadata
           }
         },
-        links: filteredLinks
+        links: filteredLinks,
+        finalUrl: rawContent.source
       };
     } catch (error) {
       logger.error(`❌ Failed processing page ${url}: ${error}`);