@arabold/docs-mcp-server 1.25.1 → 1.25.3

package/README.md

@@ -254,6 +254,7 @@ You can index documentation from your local filesystem by using a `file://` URL
       -e OPENAI_API_KEY="your-key" \
       -v /absolute/path/to/docs:/docs:ro \
       -v docs-mcp-data:/data \
+      -p 6280:6280 \
       ghcr.io/arabold/docs-mcp-server:latest \
       scrape mylib file:///docs/my-library
     ```
@@ -498,7 +499,11 @@ DOCS_MCP_TELEMETRY=false npx @arabold/docs-mcp-server@latest
 **Option 3: Docker**
 
 ```bash
-docker run -e DOCS_MCP_TELEMETRY=false ghcr.io/arabold/docs-mcp-server:latest
+docker run \
+  -e DOCS_MCP_TELEMETRY=false \
+  -v docs-mcp-data:/data \
+  -p 6280:6280 \
+  ghcr.io/arabold/docs-mcp-server:latest
 ```
 
 For more details about our telemetry practices, see the [Telemetry Guide](docs/telemetry.md).

package/dist/index.js

@@ -9,6 +9,7 @@ import { PostHog } from "posthog-node";
 import { randomUUID } from "node:crypto";
 import fs, { existsSync, readFileSync } from "node:fs";
 import path from "node:path";
+import { fileURLToPath, URL as URL$1 } from "node:url";
 import envPaths from "env-paths";
 import { Option, Command } from "commander";
 import formBody from "@fastify/formbody";
@@ -22,7 +23,6 @@ import { createTRPCProxyClient, httpBatchLink } from "@trpc/client";
 import { v4 } from "uuid";
 import { VirtualConsole, JSDOM } from "jsdom";
 import mime from "mime";
-import { fileURLToPath, URL as URL$1 } from "node:url";
 import psl from "psl";
 import fs$1 from "node:fs/promises";
 import axios from "axios";
@@ -539,6 +539,49 @@ class PostHogClient {
     return this.enabled && !!this.client;
   }
 }
+let projectRoot = null;
+function getProjectRoot() {
+  if (projectRoot) {
+    return projectRoot;
+  }
+  const currentFilePath = fileURLToPath(import.meta.url);
+  let currentDir = path.dirname(currentFilePath);
+  while (true) {
+    const packageJsonPath = path.join(currentDir, "package.json");
+    if (fs.existsSync(packageJsonPath)) {
+      projectRoot = currentDir;
+      return currentDir;
+    }
+    const parentDir = path.dirname(currentDir);
+    if (parentDir === currentDir) {
+      throw new Error("Could not find project root containing package.json.");
+    }
+    currentDir = parentDir;
+  }
+}
+function resolveStorePath(storePath) {
+  let dbDir;
+  if (storePath) {
+    dbDir = storePath;
+  } else {
+    const projectRoot2 = getProjectRoot();
+    const oldDbDir = path.join(projectRoot2, ".store");
+    const oldDbPath = path.join(oldDbDir, "documents.db");
+    const oldDbExists = fs.existsSync(oldDbPath);
+    if (oldDbExists) {
+      dbDir = oldDbDir;
+    } else {
+      const standardPaths = envPaths("docs-mcp-server", { suffix: "" });
+      dbDir = standardPaths.data;
+    }
+  }
+  try {
+    fs.mkdirSync(dbDir, { recursive: true });
+  } catch (error) {
+    console.warn(`⚠️  Failed to create database directory ${dbDir}: ${error}`);
+  }
+  return dbDir;
+}
 class TelemetryConfig {
   static instance;
   enabled = true;
@@ -560,7 +603,7 @@ class TelemetryConfig {
 }
 function generateInstallationId(storePath) {
   try {
-    const dataDir = storePath || envPaths("docs-mcp-server", { suffix: "" }).data;
+    const dataDir = resolveStorePath(storePath);
     const installationIdPath = path.join(dataDir, "installation.id");
     if (fs.existsSync(installationIdPath)) {
       const existingId = fs.readFileSync(installationIdPath, "utf8").trim();
@@ -709,7 +752,7 @@ function extractProtocol(urlOrPath) {
   }
 }
 const name = "@arabold/docs-mcp-server";
-const version = "1.25.0";
+const version = "1.25.2";
 const description = "MCP server for fetching and searching documentation";
 const type = "module";
 const bin = { "docs-mcp-server": "dist/index.js" };
@@ -1582,26 +1625,6 @@ class MimeTypeUtils {
     return mimeToLanguage[mimeType] || "";
   }
 }
-let projectRoot = null;
-function getProjectRoot() {
-  if (projectRoot) {
-    return projectRoot;
-  }
-  const currentFilePath = fileURLToPath(import.meta.url);
-  let currentDir = path.dirname(currentFilePath);
-  while (true) {
-    const packageJsonPath = path.join(currentDir, "package.json");
-    if (fs.existsSync(packageJsonPath)) {
-      projectRoot = currentDir;
-      return projectRoot;
-    }
-    const parentDir = path.dirname(currentDir);
-    if (parentDir === currentDir) {
-      throw new Error("Could not find project root containing package.json.");
-    }
-    currentDir = parentDir;
-  }
-}
 const fullTrim = (str) => {
   return str.replace(/^[\s\r\n\t]+|[\s\r\n\t]+$/g, "");
 };
@@ -4814,6 +4837,98 @@ class MarkdownMetadataExtractorMiddleware {
     await next();
   }
 }
+class HtmlNormalizationMiddleware {
+  async process(context, next) {
+    if (!context.dom) {
+      logger.debug(
+        `Skipping HTML normalization for ${context.source} - no DOM available`
+      );
+      await next();
+      return;
+    }
+    try {
+      logger.debug(`Normalizing HTML URLs and links for ${context.source}`);
+      const $ = context.dom;
+      const baseUrl = context.source;
+      this.normalizeImageUrls($, baseUrl);
+      this.normalizeLinks($, baseUrl);
+      logger.debug(`Successfully normalized HTML content for ${context.source}`);
+    } catch (error) {
+      logger.error(`❌ Failed to normalize HTML for ${context.source}: ${error}`);
+      context.errors.push(
+        error instanceof Error ? error : new Error(`HTML normalization failed: ${String(error)}`)
+      );
+    }
+    await next();
+  }
+  /**
+   * Normalizes image URLs by converting relative URLs to absolute URLs.
+   */
+  normalizeImageUrls($, baseUrl) {
+    $("img").each((_index, element) => {
+      const $img = $(element);
+      const src = $img.attr("src");
+      if (!src) return;
+      try {
+        new URL(src);
+      } catch {
+        try {
+          const absoluteUrl = new URL(src, baseUrl).href;
+          $img.attr("src", absoluteUrl);
+          logger.debug(`Converted relative image URL: ${src} → ${absoluteUrl}`);
+        } catch (error) {
+          logger.debug(`Failed to resolve relative image URL: ${src} - ${error}`);
+        }
+      }
+    });
+  }
+  /**
+   * Normalizes links by:
+   * - Converting relative URLs to absolute URLs
+   * - Unwrapping anchor links (preserving text content)
+   * - Unwrapping non-HTTP links (preserving text content)
+   */
+  normalizeLinks($, baseUrl) {
+    $("a").each((_index, element) => {
+      const $link = $(element);
+      const href = $link.attr("href");
+      if (!href) {
+        this.unwrapElement($, $link);
+        return;
+      }
+      if (href.startsWith("#")) {
+        logger.debug(`Removing anchor link: ${href}`);
+        this.unwrapElement($, $link);
+        return;
+      }
+      try {
+        const url = new URL(href);
+        if (url.protocol !== "http:" && url.protocol !== "https:") {
+          logger.debug(`Removing non-HTTP link: ${href}`);
+          this.unwrapElement($, $link);
+          return;
+        }
+      } catch {
+        try {
+          const absoluteUrl = new URL(href, baseUrl).href;
+          $link.attr("href", absoluteUrl);
+          logger.debug(`Converted relative link URL: ${href} → ${absoluteUrl}`);
+        } catch (error) {
+          logger.debug(`Failed to resolve relative link URL: ${href} - ${error}`);
+          this.unwrapElement($, $link);
+        }
+      }
+    });
+  }
+  /**
+   * Unwraps an element by replacing it with its HTML content.
+   * This preserves the inner HTML (including nested elements) while removing the wrapping tag.
+   */
+  unwrapElement(_$, $element) {
+    const htmlContent = $element.html() || $element.text();
+    $element.replaceWith(htmlContent);
+  }
+}
 function detectCharsetFromHtml(htmlContent) {
   const charsetMatch = htmlContent.match(
     /<meta\s+charset\s*=\s*["']?([^"'>\s]+)["']?[^>]*>/i
@@ -4937,6 +5052,7 @@ class HtmlPipeline extends BasePipeline {
       new HtmlMetadataExtractorMiddleware(),
       new HtmlLinkExtractorMiddleware(),
       new HtmlSanitizerMiddleware(),
+      new HtmlNormalizationMiddleware(),
       new HtmlToMarkdownMiddleware()
     ];
     const semanticSplitter = new SemanticMarkdownSplitter(
@@ -7297,14 +7413,6 @@ function parseHeaders(headerOptions) {
   }
   return headers;
 }
-const CLI_DEFAULTS = {
-  PROTOCOL: DEFAULT_PROTOCOL,
-  HTTP_PORT: DEFAULT_HTTP_PORT,
-  WEB_PORT: DEFAULT_WEB_PORT,
-  HOST: DEFAULT_HOST,
-  MAX_CONCURRENCY: DEFAULT_MAX_CONCURRENCY,
-  TELEMETRY: true
-};
 function parseAuthConfig(options) {
   if (!options.authEnabled) {
     return void 0;
@@ -12447,34 +12555,10 @@ class DocumentManagementService {
   normalizeVersion(version2) {
     return (version2 ?? "").toLowerCase();
   }
-  constructor(embeddingConfig, pipelineConfig, storePath) {
-    let dbPath;
-    let dbDir;
-    if (storePath) {
-      dbDir = storePath;
-      dbPath = path.join(dbDir, "documents.db");
-      logger.debug(`Using database directory from storePath parameter: ${dbDir}`);
-    } else {
-      const projectRoot2 = getProjectRoot();
-      const oldDbDir = path.join(projectRoot2, ".store");
-      const oldDbPath = path.join(oldDbDir, "documents.db");
-      const oldDbExists = fs.existsSync(oldDbPath);
-      if (oldDbExists) {
-        dbPath = oldDbPath;
-        dbDir = oldDbDir;
-        logger.debug(`Using legacy database path: ${dbPath}`);
-      } else {
-        const standardPaths = envPaths("docs-mcp-server", { suffix: "" });
-        dbDir = standardPaths.data;
-        dbPath = path.join(dbDir, "documents.db");
-        logger.debug(`Using standard database directory: ${dbDir}`);
-      }
-    }
-    try {
-      fs.mkdirSync(dbDir, { recursive: true });
-    } catch (error) {
-      logger.error(`⚠️  Failed to create database directory ${dbDir}: ${error}`);
-    }
+  constructor(storePath, embeddingConfig, pipelineConfig) {
+    const dbDir = storePath;
+    const dbPath = path.join(dbDir, "documents.db");
+    logger.debug(`Using database directory: ${dbDir}`);
     this.store = new DocumentStore(dbPath, embeddingConfig);
     this.documentRetriever = new DocumentRetrieverService(this.store);
     this.pipelines = PipelineFactory$1.createStandardPipelines(pipelineConfig);
@@ -12829,16 +12913,19 @@ async function createDocumentManagement(options = {}) {
     await client.initialize();
     return client;
   }
+  if (!options.storePath) {
+    throw new Error("storePath is required when not using a remote server");
+  }
   const service = new DocumentManagementService(
+    options.storePath,
     options.embeddingConfig,
-    void 0,
-    options.storePath
+    void 0
   );
   await service.initialize();
   return service;
 }
-async function createLocalDocumentManagement(embeddingConfig, storePath) {
-  const service = new DocumentManagementService(embeddingConfig, void 0, storePath);
+async function createLocalDocumentManagement(storePath, embeddingConfig) {
+  const service = new DocumentManagementService(storePath, embeddingConfig, void 0);
   await service.initialize();
   return service;
 }
@@ -12846,7 +12933,7 @@ function createDefaultAction(program) {
   return program.addOption(
     new Option("--protocol <protocol>", "Protocol for MCP server").env("DOCS_MCP_PROTOCOL").default("auto").choices(["auto", "stdio", "http"])
   ).addOption(
-    new Option("--port <number>", "Port for the server").env("DOCS_MCP_PORT").env("PORT").default(CLI_DEFAULTS.HTTP_PORT.toString()).argParser((v) => {
+    new Option("--port <number>", "Port for the server").env("DOCS_MCP_PORT").env("PORT").default(DEFAULT_HTTP_PORT.toString()).argParser((v) => {
       const n = Number(v);
       if (!Number.isInteger(n) || n < 1 || n > 65535) {
         throw new Error("Port must be an integer between 1 and 65535");
@@ -12854,7 +12941,7 @@ function createDefaultAction(program) {
       return String(n);
     })
   ).addOption(
-    new Option("--host <host>", "Host to bind the server to").env("DOCS_MCP_HOST").env("HOST").default(CLI_DEFAULTS.HOST).argParser(validateHost)
+    new Option("--host <host>", "Host to bind the server to").env("DOCS_MCP_HOST").env("HOST").default(DEFAULT_HOST).argParser(validateHost)
   ).addOption(
     new Option(
       "--embedding-model <model>",
@@ -12911,12 +12998,12 @@ function createDefaultAction(program) {
         validateAuthConfig(authConfig);
         warnHttpUsage(authConfig, port);
       }
-      const globalOptions = program.parent?.opts() || {};
+      const globalOptions = program.opts();
       ensurePlaywrightBrowsersInstalled();
       const embeddingConfig = resolveEmbeddingContext(options.embeddingModel);
       const docService = await createLocalDocumentManagement(
-        embeddingConfig,
-        globalOptions.storePath
+        globalOptions.storePath,
+        embeddingConfig
       );
       const pipelineOptions = {
         recoverJobs: options.resume || false,
@@ -13067,9 +13154,9 @@ function createListCommand(program) {
 }
 function createMcpCommand(program) {
   return program.command("mcp").description("Start MCP server only").addOption(
-    new Option("--protocol <protocol>", "Protocol for MCP server").env("DOCS_MCP_PROTOCOL").default(CLI_DEFAULTS.PROTOCOL).choices(["auto", "stdio", "http"])
+    new Option("--protocol <protocol>", "Protocol for MCP server").env("DOCS_MCP_PROTOCOL").default(DEFAULT_PROTOCOL).choices(["auto", "stdio", "http"])
   ).addOption(
-    new Option("--port <number>", "Port for the MCP server").env("DOCS_MCP_PORT").env("PORT").default(CLI_DEFAULTS.HTTP_PORT.toString()).argParser((v) => {
+    new Option("--port <number>", "Port for the MCP server").env("DOCS_MCP_PORT").env("PORT").default(DEFAULT_HTTP_PORT.toString()).argParser((v) => {
       const n = Number(v);
       if (!Number.isInteger(n) || n < 1 || n > 65535) {
         throw new Error("Port must be an integer between 1 and 65535");
@@ -13077,7 +13164,7 @@ function createMcpCommand(program) {
       return String(n);
     })
   ).addOption(
-    new Option("--host <host>", "Host to bind the MCP server to").env("DOCS_MCP_HOST").env("HOST").default(CLI_DEFAULTS.HOST).argParser(validateHost)
+    new Option("--host <host>", "Host to bind the MCP server to").env("DOCS_MCP_HOST").env("HOST").default(DEFAULT_HOST).argParser(validateHost)
   ).addOption(
     new Option(
       "--embedding-model <model>",
@@ -13437,7 +13524,7 @@ function createSearchCommand(program) {
 }
 function createWebCommand(program) {
   return program.command("web").description("Start web interface only").addOption(
-    new Option("--port <number>", "Port for the web interface").env("DOCS_MCP_WEB_PORT").env("DOCS_MCP_PORT").env("PORT").default(CLI_DEFAULTS.WEB_PORT.toString()).argParser((v) => {
+    new Option("--port <number>", "Port for the web interface").env("DOCS_MCP_WEB_PORT").env("DOCS_MCP_PORT").env("PORT").default(DEFAULT_WEB_PORT.toString()).argParser((v) => {
       const n = Number(v);
       if (!Number.isInteger(n) || n < 1 || n > 65535) {
         throw new Error("Port must be an integer between 1 and 65535");
@@ -13445,7 +13532,7 @@ function createWebCommand(program) {
       return String(n);
     })
   ).addOption(
-    new Option("--host <host>", "Host to bind the web interface to").env("DOCS_MCP_HOST").env("HOST").default(CLI_DEFAULTS.HOST).argParser(validateHost)
+    new Option("--host <host>", "Host to bind the web interface to").env("DOCS_MCP_HOST").env("HOST").default(DEFAULT_HOST).argParser(validateHost)
   ).addOption(
     new Option(
       "--embedding-model <model>",
@@ -13527,7 +13614,7 @@ function createWorkerCommand(program) {
       return String(n);
     })
   ).addOption(
-    new Option("--host <host>", "Host to bind the worker API to").env("DOCS_MCP_HOST").env("HOST").default(CLI_DEFAULTS.HOST).argParser(validateHost)
+    new Option("--host <host>", "Host to bind the worker API to").env("DOCS_MCP_HOST").env("HOST").default(DEFAULT_HOST).argParser(validateHost)
   ).addOption(
     new Option(
       "--embedding-model <model>",
@@ -13547,11 +13634,15 @@ function createWorkerCommand(program) {
         logger.info(`🚀 Starting external pipeline worker on port ${port}`);
         ensurePlaywrightBrowsersInstalled();
         const embeddingConfig = resolveEmbeddingContext(cmdOptions.embeddingModel);
-        const docService = await createLocalDocumentManagement(embeddingConfig);
+        const globalOptions = program.parent?.opts() || {};
+        const docService = await createLocalDocumentManagement(
+          globalOptions.storePath,
+          embeddingConfig
+        );
         const pipelineOptions = {
           recoverJobs: cmdOptions.resume,
           // Use the resume option
-          concurrency: CLI_DEFAULTS.MAX_CONCURRENCY
+          concurrency: DEFAULT_MAX_CONCURRENCY
         };
         const pipeline = await createPipelineWithCallbacks(docService, pipelineOptions);
         const config = createAppServerConfig({
@@ -13599,10 +13690,12 @@ function createCliProgram() {
   ).enablePositionalOptions().allowExcessArguments(false).showHelpAfterError(true);
   program.hook("preAction", async (thisCommand, actionCommand) => {
     const globalOptions = thisCommand.opts();
+    const resolvedStorePath = resolveStorePath(globalOptions.storePath);
+    globalOptions.storePath = resolvedStorePath;
     setupLogging(globalOptions);
     initTelemetry({
       enabled: globalOptions.telemetry ?? true,
-      storePath: globalOptions.storePath
+      storePath: resolvedStorePath
     });
     if (shouldEnableTelemetry()) {
       if (analytics.isEnabled()) {