@arabold/docs-mcp-server 2.1.0 → 2.2.0

package/db/migrations/013-create-metadata-table.sql

@@ -0,0 +1,7 @@
+-- Migration: Create metadata table for tracking global configuration state.
+-- Used to persist the active embedding model and vector dimension so the system
+-- can detect configuration changes between startups and prevent silent data corruption.
+CREATE TABLE IF NOT EXISTS metadata (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);

package/dist/index.js

@@ -4,8 +4,8 @@ import { s as sanitizeEnvironment, l as logger } from "./logger-CLtABTNb.js";
 process.setSourceMapsEnabled(true);
 sanitizeEnvironment();
 const [{ runCli }, { ensurePlaywrightBrowsersInstalled }] = await Promise.all([
-  import("./main-CmXZnHFM.js"),
-  import("./utils-XwXdQlZ_.js").then((n) => n.u)
+  import("./main-B6iwx1JH.js"),
+  import("./utils-ntayxPjd.js").then((n) => n.u)
 ]);
 ensurePlaywrightBrowsersInstalled();
 runCli().catch((error) => {

package/dist/{main-CmXZnHFM.js → main-B6iwx1JH.js}

@@ -9,7 +9,7 @@ import path from "node:path";
 import { URL as URL$1 } from "node:url";
 import envPaths from "env-paths";
 import { l as logger, n as normalizeEnvValue, g as getLogLevelFromEnv, a as setLogLevel, L as LogLevel } from "./logger-CLtABTNb.js";
-import { t as telemetry, T as TelemetryEvent, P as PipelineJobStatus, g as getProjectRoot, E as EmbeddingConfig, a as EventType, S as ServerEventName, s as shouldEnableTelemetry, r as resolveProtocol, p as parseAuthConfig, v as validateAuthConfig, w as warnHttpUsage, e as ensurePlaywrightBrowsersInstalled, b as getEventBus, c as createAppServerConfig, d as parseHeaders, f as validatePort, h as validateHost, i as resolveStorePath, j as initTelemetry, k as EventBusService, l as TelemetryService } from "./utils-XwXdQlZ_.js";
+import { t as telemetry, T as TelemetryEvent, P as PipelineJobStatus, g as getProjectRoot, E as EmbeddingConfig, a as EventType, S as ServerEventName, s as shouldEnableTelemetry, r as resolveProtocol, p as parseAuthConfig, v as validateAuthConfig, w as warnHttpUsage, e as ensurePlaywrightBrowsersInstalled, b as getEventBus, h as handleEmbeddingModelChange, c as createAppServerConfig, d as parseHeaders, f as validatePort, i as validateHost, j as resolveStorePath, k as initTelemetry, l as EventBusService, m as TelemetryService } from "./utils-ntayxPjd.js";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import yaml from "yaml";
@@ -117,6 +117,23 @@ class DimensionError extends StoreError {
 }
 class ConnectionError extends StoreError {
 }
+class EmbeddingModelChangedError extends StoreError {
+  constructor(previousModel, previousDimension, currentModel, currentDimension) {
+    super(
+      `Embedding model change detected:
+  Previous: ${previousModel} (${previousDimension} dimensions)
+  Current:  ${currentModel} (${currentDimension} dimensions)
+
+All existing vectors are incompatible and must be invalidated.
+To confirm this change, start the server interactively (with a TTY connected)
+and follow the prompts.`
+    );
+    this.previousModel = previousModel;
+    this.previousDimension = previousDimension;
+    this.currentModel = currentModel;
+    this.currentDimension = currentDimension;
+  }
+}
 class MissingCredentialsError extends StoreError {
   constructor(provider, missingCredentials) {
     super(
@@ -456,7 +473,7 @@ const AppConfigSchema = z.object({
     batchChars: z.coerce.number().int().default(DEFAULT_CONFIG.embeddings.batchChars),
     requestTimeoutMs: z.coerce.number().int().default(DEFAULT_CONFIG.embeddings.requestTimeoutMs),
     initTimeoutMs: z.coerce.number().int().default(DEFAULT_CONFIG.embeddings.initTimeoutMs),
-    vectorDimension: z.coerce.number().int().default(DEFAULT_CONFIG.embeddings.vectorDimension)
+    vectorDimension: z.coerce.number().int().min(1, "embedding dimension must be at least 1").default(DEFAULT_CONFIG.embeddings.vectorDimension)
   }).default(DEFAULT_CONFIG.embeddings),
   db: z.object({
     migrationMaxRetries: z.coerce.number().int().default(DEFAULT_CONFIG.db.migrationMaxRetries),
@@ -1963,6 +1980,17 @@ class ChallengeError extends ScraperError {
     this.challengeType = challengeType;
   }
 }
+class TlsCertificateError extends ScraperError {
+  constructor(url, code, cause) {
+    super(
+      `TLS certificate validation failed for ${url}${code ? ` (${code})` : ""}. The remote site may have an incomplete or untrusted certificate chain.`,
+      false,
+      cause
+    );
+    this.url = url;
+    this.code = code;
+  }
+}
 class MimeTypeUtils {
   /**
    * Parses a Content-Type header string into its MIME type and charset.
@@ -2420,6 +2448,9 @@ class MimeTypeUtils {
       "text/x-xml": "xml",
       "text/xml": "xml",
       "application/xml": "xml",
+      "application/xslt+xml": "xml",
+      "application/xml-dtd": "xml",
+      "application/wsdl+xml": "xml",
       "text/x-sql": "sql",
       "text/x-graphql": "graphql",
       // Schema/API definitions
@@ -2688,6 +2719,15 @@ class HttpFetcher {
     "EPERM"
     // Operation not permitted
   ];
+  tlsCertificateErrorCodes = [
+    "CERT_HAS_EXPIRED",
+    "DEPTH_ZERO_SELF_SIGNED_CERT",
+    "ERR_TLS_CERT_ALTNAME_INVALID",
+    "SELF_SIGNED_CERT_IN_CHAIN",
+    "UNABLE_TO_GET_ISSUER_CERT",
+    "UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
+    "UNABLE_TO_VERIFY_LEAF_SIGNATURE"
+  ];
   fingerprintGenerator;
   constructor(scraperConfig) {
     this.maxRetriesDefault = scraperConfig.fetcher.maxRetries;
@@ -2700,6 +2740,9 @@ class HttpFetcher {
   async delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
+  isTlsCertificateError(code) {
+    return code ? this.tlsCertificateErrorCodes.includes(code) : false;
+  }
   async fetch(source, options) {
     const maxRetries = options?.maxRetries ?? this.maxRetriesDefault;
     const baseDelay = options?.retryDelay ?? this.baseDelayDefaultMs;
@@ -2796,6 +2839,7 @@ class HttpFetcher {
         const axiosError = error;
         const status = axiosError.response?.status;
         const code = axiosError.code;
+        const errorCause = error instanceof Error ? error : void 0;
         if (options?.signal?.aborted || code === "ERR_CANCELED") {
           throw new CancellationError("HTTP fetch cancelled");
         }
@@ -2835,6 +2879,9 @@ class HttpFetcher {
             throw new ChallengeError(source, status, "cloudflare");
           }
         }
+        if (this.isTlsCertificateError(code)) {
+          throw new TlsCertificateError(source, code, errorCause);
+        }
         if (attempt < maxRetries && (status === void 0 || this.retryableStatusCodes.includes(status)) && !this.nonRetryableErrorCodes.includes(code ?? "")) {
           const delay = baseDelay * 2 ** attempt;
           logger.warn(
@@ -2846,7 +2893,7 @@ class HttpFetcher {
         throw new ScraperError(
           `Failed to fetch ${source} after ${attempt + 1} attempts: ${axiosError.message ?? "Unknown error"}`,
           true,
-          error instanceof Error ? error : void 0
+          errorCause
         );
       }
     }
@@ -2891,6 +2938,12 @@ class AutoDetectFetcher {
           );
           return this.browserFetcher.fetch(source, options);
         }
+        if (error instanceof TlsCertificateError) {
+          logger.info(
+            `🔄 TLS certificate validation failed for ${source}, falling back to browser fetcher...`
+          );
+          return this.browserFetcher.fetch(source, options);
+        }
         throw error;
       }
     }
@@ -9180,6 +9233,91 @@ class DocumentStore {
     }
     return [...vector, ...new Array(this.dbDimension - vector.length).fill(0)];
   }
+  /**
+   * Reads the stored embedding model and dimension from the metadata table.
+   * Returns null for each value that doesn't exist (e.g., first run or pre-metadata database).
+   */
+  getEmbeddingMetadata() {
+    const modelRow = this.db.prepare("SELECT value FROM metadata WHERE key = ?").get("embedding_model");
+    const dimensionRow = this.db.prepare("SELECT value FROM metadata WHERE key = ?").get("embedding_dimension");
+    return {
+      model: modelRow?.value ?? null,
+      dimension: dimensionRow?.value ?? null
+    };
+  }
+  /**
+   * Persists the active embedding model and dimension to the metadata table.
+   * Uses upsert (INSERT ON CONFLICT UPDATE) so it works for both first-run and subsequent updates.
+   * Uses inline db.prepare() so this method works before prepareStatements() is called.
+   */
+  setEmbeddingMetadata(model, dimension) {
+    const stmt = this.db.prepare(
+      "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
+    );
+    stmt.run("embedding_model", model);
+    stmt.run("embedding_dimension", String(dimension));
+  }
+  /**
+   * Compares the configured embedding model and dimension against stored metadata.
+   * Throws EmbeddingModelChangedError if either has changed since the last run.
+   *
+   * Skipped when:
+   * - No metadata exists (first run / upgrade from pre-metadata DB → silent initialization)
+   * - No embedding model is configured (FTS-only mode)
+   * - Credentials are unavailable for the configured provider (will fall back to FTS-only)
+   */
+  checkEmbeddingModelChange() {
+    if (!this.embeddingConfig) {
+      return;
+    }
+    if (!areCredentialsAvailable(this.embeddingConfig.provider)) {
+      return;
+    }
+    const stored = this.getEmbeddingMetadata();
+    if (stored.model === null) {
+      return;
+    }
+    const currentModel = this.config.app.embeddingModel;
+    const currentDimension = String(this.config.embeddings.vectorDimension);
+    const modelChanged = stored.model !== currentModel;
+    const dimensionChanged = stored.dimension !== null && stored.dimension !== currentDimension;
+    if (modelChanged || dimensionChanged) {
+      throw new EmbeddingModelChangedError(
+        stored.model,
+        stored.dimension ?? "unknown",
+        currentModel,
+        currentDimension
+      );
+    }
+  }
+  /**
+   * Invalidates all existing embedding vectors after a confirmed model/dimension change.
+   * Sets all document embeddings to NULL, drops and recreates the vec table as empty,
+   * and updates the metadata with the new model and dimension.
+   *
+   * After invalidation, FTS search continues working; vector search returns no results
+   * until libraries are re-scraped.
+   */
+  invalidateAllVectors(newModel, newDimension) {
+    logger.warn(
+      `⚠️  Invalidating all embedding vectors due to model/dimension change.
+   All libraries must be re-scraped to restore vector search.
+   Full-text search remains available for all existing documents.`
+    );
+    this.db.exec("UPDATE documents SET embedding = NULL");
+    this.db.exec("DROP TABLE IF EXISTS documents_vec");
+    this.db.exec(`
+      CREATE VIRTUAL TABLE documents_vec USING vec0(
+        library_id INTEGER NOT NULL,
+        version_id INTEGER NOT NULL,
+        embedding FLOAT[${newDimension}]
+      );
+    `);
+    this.setEmbeddingMetadata(newModel, newDimension);
+    logger.info(
+      `✅ Embedding vectors invalidated. Metadata updated to: ${newModel} (${newDimension}d)`
+    );
+  }
   /**
    * Initialize the embeddings client using the provided config.
    * If no embedding config is provided (null or undefined), embeddings will not be initialized.
@@ -9238,6 +9376,9 @@ class DocumentStore {
         }
         EmbeddingConfig.setKnownModelDimensions(config.model, this.modelDimension);
       }
+      if (this.embeddings instanceof FixedDimensionEmbeddings && this.embeddings.allowTruncate) {
+        this.modelDimension = Math.min(this.modelDimension, this.dbDimension);
+      }
       if (this.modelDimension > this.dbDimension) {
         throw new DimensionError(config.modelSpec, this.modelDimension, this.dbDimension);
       }
@@ -9245,6 +9386,7 @@ class DocumentStore {
       logger.debug(
         `Embeddings initialized: ${config.provider}:${config.model} (${this.modelDimension}d)`
       );
+      this.setEmbeddingMetadata(config.modelSpec, this.dbDimension);
     } catch (error) {
       if (error instanceof Error) {
         if (error.message.includes("does not exist") || error.message.includes("MODEL_NOT_FOUND")) {
@@ -9285,14 +9427,14 @@ class DocumentStore {
    * This prevents FTS5 syntax errors while supporting intuitive phrase searches.
    *
    * Query construction:
-   * - Exact match of full input: `("escaped full query")`
-   * - Individual terms: `("term1" AND "term2" AND "phrase")`
-   * - Combined: `("full query") OR ("term1" AND "term2")`
+   * - Exact match of full input: `"escaped full query"`
+   * - Individual terms: `"term1" OR "term2" OR "phrase"`
+   * - Combined: `"full query" OR "term1" OR "term2"`
    *
    * Examples:
-   * - `foo bar` -> `("foo bar") OR ("foo" AND "bar")`
-   * - `"hello world"` -> `("hello world")`
-   * - `test "exact phrase" word` -> `("test exact phrase word") OR ("test" AND "exact phrase" AND "word")`
+   * - `foo bar` -> `"foo bar" OR "foo" OR "bar"`
+   * - `"hello world"` -> `"hello world"`
+   * - `test "exact phrase" word` -> `"test exact phrase word" OR "test" OR "exact phrase" OR "word"`
    */
   escapeFtsQuery(query) {
     const tokens = [];
@@ -9350,6 +9492,8 @@ class DocumentStore {
         maxRetries: this.config.db.migrationMaxRetries,
         retryDelayMs: this.config.db.migrationRetryDelayMs
       });
+      this.checkEmbeddingModelChange();
+      this.ensureVectorTable();
       this.prepareStatements();
       await this.initializeEmbeddings();
     } catch (error) {
@@ -9359,12 +9503,66 @@ class DocumentStore {
       throw new ConnectionError("Failed to initialize database connection", error);
     }
   }
+  /**
+   * Resolves a model change by invalidating all vectors and completing initialization.
+   * Called by the CLI layer after the user confirms a model/dimension change.
+   * Assumes initialize() was previously called and threw EmbeddingModelChangedError,
+   * meaning steps 1-2 (load extensions, apply migrations) completed successfully.
+   * Steps 3+ (ensureVectorTable, prepareStatements, initializeEmbeddings) are
+   * completed here after invalidation.
+   */
+  async resolveModelChange() {
+    const currentModel = this.config.app.embeddingModel;
+    const currentDimension = this.config.embeddings.vectorDimension;
+    this.invalidateAllVectors(currentModel, currentDimension);
+    this.ensureVectorTable();
+    this.prepareStatements();
+    await this.initializeEmbeddings();
+  }
   /**
    * Gracefully closes database connections
    */
   async shutdown() {
     this.db.close();
   }
+  /**
+   * Creates or reconciles the documents_vec virtual table with configurable dimension.
+   * Called after migrations and model change detection. The table is initially created
+   * by migration 003 with a fixed 1536 dimension; this method reconciles it at runtime
+   * if the configured dimension differs.
+   * Idempotent: if the table already exists with the same dimension, no-op; if dimension
+   * changed in config, drops and recreates so any embedding provider (e.g. 1536 or 3584) works.
+   *
+   * Note: No backfill of existing embeddings is performed. Vectors are populated during
+   * scraping, not at startup. Old vectors from a different dimension or model are incompatible
+   * and are handled by the model change detection system (checkEmbeddingModelChange).
+   */
+  ensureVectorTable() {
+    const dim = this.config.embeddings.vectorDimension;
+    if (typeof dim !== "number" || !Number.isInteger(dim) || dim < 1) {
+      throw new StoreError(
+        `Invalid embeddings.vectorDimension: ${dim}. Must be a positive integer.`
+      );
+    }
+    const existingSql = this.db.prepare(
+      "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'documents_vec';"
+    ).get();
+    if (existingSql) {
+      const match = existingSql.sql.match(/embedding\s+FLOAT\s*\[\s*(\d+)\s*]/i);
+      const existingDim = match ? Number(match[1]) : null;
+      if (existingDim === dim) {
+        return;
+      }
+      this.db.exec("DROP TABLE documents_vec;");
+    }
+    this.db.exec(`
+      CREATE VIRTUAL TABLE documents_vec USING vec0(
+        library_id INTEGER NOT NULL,
+        version_id INTEGER NOT NULL,
+        embedding FLOAT[${dim}]
+      );
+    `);
+  }
   /**
    * Resolves a library name and version string to version_id.
    * Creates library and version records if they don't exist.
@@ -10281,6 +10479,13 @@ class DocumentManagementService {
   async initialize() {
     await this.store.initialize();
   }
+  /**
+   * Resolves a confirmed embedding model change by invalidating all vectors
+   * and completing the initialization that was interrupted by EmbeddingModelChangedError.
+   */
+  async resolveModelChange() {
+    await this.store.resolveModelChange();
+  }
   /**
    * Shuts down the underlying document store and cleans up pipeline resources.
    */
@@ -11455,7 +11660,7 @@ const Layout = ({
   children,
   eventClientConfig
 }) => {
-  const versionString = version || "2.1.0";
+  const versionString = version || "2.2.0";
   const versionInitializer = `versionUpdate({ currentVersion: ${`'${versionString}'`} })`;
   return /* @__PURE__ */ jsxs("html", { lang: "en", children: [
     /* @__PURE__ */ jsxs("head", { children: [
@@ -13807,7 +14012,7 @@ class AppServer {
       try {
         if (telemetry.isEnabled()) {
           telemetry.setGlobalContext({
-            appVersion: "2.1.0",
+            appVersion: "2.2.0",
             appPlatform: process.platform,
             appNodeVersion: process.version,
             appServicesEnabled: this.getActiveServicesList(),
@@ -14926,6 +15131,11 @@ const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
   ".html",
   ".htm",
   ".xml",
+  ".xsl",
+  ".xslt",
+  ".xsd",
+  ".dtd",
+  ".wsdl",
   ".xhtml",
   // Stylesheets
   ".css",
@@ -16996,7 +17206,16 @@ function createDefaultAction(cli) {
       }
       ensurePlaywrightBrowsersInstalled();
       const eventBus = getEventBus(argv);
-      const docService = await createLocalDocumentManagement(eventBus, appConfig);
+      const docService = new DocumentManagementService(eventBus, appConfig);
+      try {
+        await docService.initialize();
+      } catch (error) {
+        if (error instanceof EmbeddingModelChangedError) {
+          await handleEmbeddingModelChange(error, docService);
+        } else {
+          throw error;
+        }
+      }
       const pipelineOptions = {
         recoverJobs: argv.resume || false,
         appConfig
@@ -17279,11 +17498,24 @@ function createMcpCommand(cli) {
       try {
         const serverUrl = argv.serverUrl;
         const eventBus = getEventBus(argv);
-        const docService = await createDocumentManagement({
-          serverUrl,
-          eventBus,
-          appConfig
-        });
+        let docService;
+        if (serverUrl) {
+          const client = new DocumentManagementClient(serverUrl);
+          await client.initialize();
+          docService = client;
+        } else {
+          const service = new DocumentManagementService(eventBus, appConfig);
+          try {
+            await service.initialize();
+          } catch (error) {
+            if (error instanceof EmbeddingModelChangedError) {
+              await handleEmbeddingModelChange(error, service);
+            } else {
+              throw error;
+            }
+          }
+          docService = service;
+        }
         const pipelineOptions = {
           recoverJobs: false,
           // MCP command doesn't support job recovery
@@ -17961,7 +18193,7 @@ function createCli(argv) {
   let globalEventBus = null;
   let globalTelemetryService = null;
   const commandStartTimes = /* @__PURE__ */ new Map();
-  const cli = registerGlobalOutputOptions(yargs(hideBin(argv))).scriptName("docs-mcp-server").strict().usage("Usage: $0 <command> [options]").version("2.1.0").option("verbose", {
+  const cli = registerGlobalOutputOptions(yargs(hideBin(argv))).scriptName("docs-mcp-server").strict().usage("Usage: $0 <command> [options]").version("2.2.0").option("verbose", {
     type: "boolean",
     description: "Enable verbose (debug) logging",
     default: false
@@ -18022,7 +18254,7 @@ function createCli(argv) {
     if (shouldEnableTelemetry() && telemetry.isEnabled()) {
       const commandName = argv2._[0]?.toString() || "default";
       telemetry.setGlobalContext({
-        appVersion: "2.1.0",
+        appVersion: "2.2.0",
         appPlatform: process.platform,
         appNodeVersion: process.version,
         appInterface: "cli",
@@ -18168,4 +18400,4 @@ export {
   cleanupCliCommand,
   runCli
 };
-//# sourceMappingURL=main-CmXZnHFM.js.map
+//# sourceMappingURL=main-B6iwx1JH.js.map