@arabold/docs-mcp-server 1.32.0 → 1.33.0

package/dist/index.js

@@ -1472,6 +1472,8 @@ const FETCHER_MAX_CACHE_ITEM_SIZE_BYTES = 500 * 1024;
 const SPLITTER_MIN_CHUNK_SIZE = 500;
 const SPLITTER_PREFERRED_CHUNK_SIZE = 1500;
 const SPLITTER_MAX_CHUNK_SIZE = 5e3;
+const JSON_MAX_NESTING_DEPTH = 5;
+const JSON_MAX_CHUNKS = 1e3;
 const EMBEDDING_BATCH_SIZE = 100;
 const EMBEDDING_BATCH_CHARS = 5e4;
 const MIGRATION_MAX_RETRIES = 5;
@@ -3044,16 +3046,203 @@ class GreedySplitter {
     return common;
   }
 }
+class TextContentSplitter {
+  constructor(options) {
+    this.options = options;
+  }
+  /**
+   * Splits text content into chunks while trying to preserve semantic boundaries.
+   * Prefers paragraph breaks, then line breaks, finally falling back to word boundaries.
+   * Always preserves formatting - trimming should be done by higher-level splitters if needed.
+   */
+  async split(content) {
+    if (content.length <= this.options.chunkSize) {
+      return [content];
+    }
+    const words = content.split(/\s+/);
+    const longestWord = words.reduce(
+      (max, word) => word.length > max.length ? word : max
+    );
+    if (longestWord.length > this.options.chunkSize) {
+      throw new MinimumChunkSizeError(longestWord.length, this.options.chunkSize);
+    }
+    const paragraphChunks = this.splitByParagraphs(content);
+    if (this.areChunksValid(paragraphChunks)) {
+      return paragraphChunks;
+    }
+    const lineChunks = this.splitByLines(content);
+    if (this.areChunksValid(lineChunks)) {
+      return this.mergeChunks(lineChunks, "");
+    }
+    const wordChunks = await this.splitByWords(content);
+    return this.mergeChunks(wordChunks, " ");
+  }
+  /**
+   * Checks if all chunks are within the maximum size limit
+   */
+  areChunksValid(chunks) {
+    return chunks.every((chunk) => chunk.length <= this.options.chunkSize);
+  }
+  /**
+   * Splits text into chunks by paragraph boundaries (double newlines)
+   * Preserves all formatting and whitespace including the paragraph separators
+   */
+  splitByParagraphs(text) {
+    const chunks = [];
+    let startPos = 0;
+    const paragraphRegex = /\n\s*\n/g;
+    let match = paragraphRegex.exec(text);
+    while (match !== null) {
+      const endPos = match.index + match[0].length;
+      const chunk = text.slice(startPos, endPos);
+      if (chunk.length > 2) {
+        chunks.push(chunk);
+      }
+      startPos = endPos;
+      match = paragraphRegex.exec(text);
+    }
+    if (startPos < text.length) {
+      const remainingChunk = text.slice(startPos);
+      if (remainingChunk.length > 2) {
+        chunks.push(remainingChunk);
+      }
+    }
+    return chunks.filter(Boolean);
+  }
+  /**
+   * Splits text into chunks by line boundaries
+   * Preserves all formatting and whitespace, including newlines at the end of each line
+   */
+  splitByLines(text) {
+    const chunks = [];
+    let startPos = 0;
+    for (let i = 0; i < text.length; i++) {
+      if (text[i] === "\n") {
+        const chunk = text.slice(startPos, i + 1);
+        chunks.push(chunk);
+        startPos = i + 1;
+      }
+    }
+    if (startPos < text.length) {
+      chunks.push(text.slice(startPos));
+    }
+    return chunks;
+  }
+  /**
+   * Uses LangChain's recursive splitter for word-based splitting as a last resort
+   */
+  async splitByWords(text) {
+    const splitter = new RecursiveCharacterTextSplitter({
+      chunkSize: this.options.chunkSize,
+      chunkOverlap: 0
+    });
+    const chunks = await splitter.splitText(text);
+    return chunks;
+  }
+  /**
+   * Attempts to merge small chunks with previous chunks to minimize fragmentation.
+   * Only merges if combined size is within maxChunkSize.
+   */
+  mergeChunks(chunks, separator) {
+    const mergedChunks = [];
+    let currentChunk = null;
+    for (const chunk of chunks) {
+      if (currentChunk === null) {
+        currentChunk = chunk;
+        continue;
+      }
+      const currentChunkSize = this.getChunkSize(currentChunk);
+      const nextChunkSize = this.getChunkSize(chunk);
+      if (currentChunkSize + nextChunkSize + separator.length <= this.options.chunkSize) {
+        currentChunk = `${currentChunk}${separator}${chunk}`;
+      } else {
+        mergedChunks.push(currentChunk);
+        currentChunk = chunk;
+      }
+    }
+    if (currentChunk) {
+      mergedChunks.push(currentChunk);
+    }
+    return mergedChunks;
+  }
+  getChunkSize(chunk) {
+    return chunk.length;
+  }
+  wrap(content) {
+    return content;
+  }
+}
+class TextDocumentSplitter {
+  options;
+  textSplitter;
+  constructor(options = {}) {
+    this.options = {
+      maxChunkSize: options.maxChunkSize ?? SPLITTER_MAX_CHUNK_SIZE
+    };
+    this.textSplitter = new TextContentSplitter({
+      chunkSize: this.options.maxChunkSize
+    });
+  }
+  async splitText(content) {
+    if (!content.trim()) {
+      return [];
+    }
+    try {
+      const chunks = await this.textSplitter.split(content);
+      return chunks.map((chunk) => ({
+        types: ["text"],
+        content: chunk,
+        section: {
+          level: 0,
+          path: []
+        }
+      }));
+    } catch (error) {
+      if (!(error instanceof MinimumChunkSizeError) && error instanceof Error) {
+        console.warn(
+          `Unexpected text splitting error: ${error.message}. Forcing character-based split.`
+        );
+      }
+      const chunks = [];
+      let offset = 0;
+      while (offset < content.length) {
+        const chunkContent = content.substring(
+          offset,
+          offset + this.options.maxChunkSize
+        );
+        chunks.push({
+          types: ["text"],
+          content: chunkContent,
+          section: {
+            level: 0,
+            path: []
+          }
+        });
+        offset += this.options.maxChunkSize;
+      }
+      return chunks;
+    }
+  }
+}
 class JsonDocumentSplitter {
   preserveFormatting;
+  maxDepth;
+  maxChunks;
+  textFallbackSplitter;
   constructor(options = {}) {
     this.preserveFormatting = options.preserveFormatting ?? true;
+    this.maxDepth = options.maxDepth ?? JSON_MAX_NESTING_DEPTH;
+    this.maxChunks = options.maxChunks ?? JSON_MAX_CHUNKS;
+    this.textFallbackSplitter = new TextDocumentSplitter();
   }
   async splitText(content, _contentType) {
     try {
       const parsed = JSON.parse(content);
       const chunks = [];
-      this.processValue(parsed, ["root"], 1, 0, chunks, true);
+      await this.processValue(parsed, ["root"], 1, 0, chunks, true);
+      if (chunks.length > this.maxChunks) {
+        return this.textFallbackSplitter.splitText(content);
+      }
       return chunks;
     } catch {
       return [
@@ -3068,16 +3257,20 @@ class JsonDocumentSplitter {
       ];
     }
   }
-  processValue(value, path2, level, indentLevel, chunks, isLastItem) {
+  async processValue(value, path2, level, indentLevel, chunks, isLastItem) {
+    if (level > this.maxDepth) {
+      await this.processValueAsText(value, path2, level, indentLevel, chunks, isLastItem);
+      return;
+    }
     if (Array.isArray(value)) {
-      this.processArray(value, path2, level, indentLevel, chunks, isLastItem);
+      await this.processArray(value, path2, level, indentLevel, chunks, isLastItem);
     } else if (value !== null && typeof value === "object") {
-      this.processObject(value, path2, level, indentLevel, chunks, isLastItem);
+      await this.processObject(value, path2, level, indentLevel, chunks, isLastItem);
     } else {
-      this.processPrimitive(value, path2, level, indentLevel, chunks, isLastItem);
+      await this.processPrimitive(value, path2, level, indentLevel, chunks, isLastItem);
     }
   }
-  processArray(array, path2, level, indentLevel, chunks, isLastItem) {
+  async processArray(array, path2, level, indentLevel, chunks, isLastItem) {
     const indent = this.getIndent(indentLevel);
     const comma = isLastItem ? "" : ",";
     chunks.push({
@@ -3085,18 +3278,19 @@ class JsonDocumentSplitter {
       content: `${indent}[`,
       section: { level, path: [...path2] }
     });
-    array.forEach((item, index) => {
+    for (let index = 0; index < array.length; index++) {
+      const item = array[index];
       const isLast = index === array.length - 1;
       const itemPath = [...path2, `[${index}]`];
-      this.processValue(item, itemPath, level + 1, indentLevel + 1, chunks, isLast);
-    });
+      await this.processValue(item, itemPath, level + 1, indentLevel + 1, chunks, isLast);
+    }
     chunks.push({
       types: ["code"],
       content: `${indent}]${comma}`,
       section: { level, path: [...path2] }
     });
   }
-  processObject(obj, path2, level, indentLevel, chunks, isLastItem) {
+  async processObject(obj, path2, level, indentLevel, chunks, isLastItem) {
     const indent = this.getIndent(indentLevel);
     const comma = isLastItem ? "" : ",";
     const entries = Object.entries(obj);
@@ -3105,10 +3299,11 @@ class JsonDocumentSplitter {
       content: `${indent}{`,
       section: { level, path: [...path2] }
     });
-    entries.forEach(([key, value], index) => {
+    for (let index = 0; index < entries.length; index++) {
+      const [key, value] = entries[index];
       const isLast = index === entries.length - 1;
       const propertyPath = [...path2, key];
-      this.processProperty(
+      await this.processProperty(
         key,
         value,
         propertyPath,
@@ -3117,14 +3312,14 @@ class JsonDocumentSplitter {
         chunks,
         isLast
       );
-    });
+    }
     chunks.push({
       types: ["code"],
       content: `${indent}}${comma}`,
       section: { level, path: [...path2] }
     });
   }
-  processProperty(key, value, path2, level, indentLevel, chunks, isLastProperty) {
+  async processProperty(key, value, path2, level, indentLevel, chunks, isLastProperty) {
     const indent = this.getIndent(indentLevel);
     if (typeof value === "object" && value !== null) {
       chunks.push({
@@ -3132,30 +3327,98 @@ class JsonDocumentSplitter {
         content: `${indent}"${key}": `,
         section: { level, path: path2 }
       });
-      this.processValue(value, path2, level, indentLevel, chunks, isLastProperty);
+      await this.processValue(value, path2, level, indentLevel, chunks, isLastProperty);
     } else {
       const comma = isLastProperty ? "" : ",";
       const formattedValue = JSON.stringify(value);
-      chunks.push({
-        types: ["code"],
-        content: `${indent}"${key}": ${formattedValue}${comma}`,
-        section: { level, path: path2 }
-      });
+      const fullContent = `${indent}"${key}": ${formattedValue}${comma}`;
+      if (fullContent.length > SPLITTER_MAX_CHUNK_SIZE) {
+        const textChunks = await this.textFallbackSplitter.splitText(formattedValue);
+        chunks.push({
+          types: ["code"],
+          content: `${indent}"${key}": `,
+          section: { level, path: path2 }
+        });
+        textChunks.forEach((textChunk, index) => {
+          const isLastChunk = index === textChunks.length - 1;
+          const content = `${textChunk.content}${isLastChunk ? comma : ""}`;
+          chunks.push({
+            types: ["code"],
+            content,
+            section: { level, path: path2 }
+          });
+        });
+      } else {
+        chunks.push({
+          types: ["code"],
+          content: fullContent,
+          section: { level, path: path2 }
+        });
+      }
     }
   }
-  processPrimitive(value, path2, level, indentLevel, chunks, isLastItem) {
+  async processPrimitive(value, path2, level, indentLevel, chunks, isLastItem) {
     const indent = this.getIndent(indentLevel);
     const comma = isLastItem ? "" : ",";
     const formattedValue = JSON.stringify(value);
-    chunks.push({
-      types: ["code"],
-      content: `${indent}${formattedValue}${comma}`,
-      section: { level, path: path2 }
-    });
+    const fullContent = `${indent}${formattedValue}${comma}`;
+    if (fullContent.length > SPLITTER_MAX_CHUNK_SIZE) {
+      const textChunks = await this.textFallbackSplitter.splitText(formattedValue);
+      textChunks.forEach((textChunk, index) => {
+        const isFirstChunk = index === 0;
+        const isLastChunk = index === textChunks.length - 1;
+        const valueContent = isFirstChunk ? `${indent}${textChunk.content}` : textChunk.content;
+        const content = `${valueContent}${isLastChunk ? comma : ""}`;
+        chunks.push({
+          types: ["code"],
+          content,
+          section: { level, path: [...path2] }
+        });
+      });
+    } else {
+      chunks.push({
+        types: ["code"],
+        content: fullContent,
+        section: { level, path: path2 }
+      });
+    }
   }
   getIndent(level) {
     return this.preserveFormatting ? "  ".repeat(level) : "";
   }
+  /**
+   * Process a value that has exceeded the maximum depth limit by serializing it as text.
+   * This prevents excessive chunking of deeply nested structures.
+   * If the serialized value is too large, splits it using the text fallback splitter.
+   */
+  async processValueAsText(value, path2, level, indentLevel, chunks, isLastItem) {
+    const indent = this.getIndent(indentLevel);
+    const comma = isLastItem ? "" : ",";
+    let serialized;
+    if (this.preserveFormatting) {
+      const lines = JSON.stringify(value, null, 2).split("\n");
+      serialized = lines.map((line, idx) => idx === 0 ? line : `${indent}${line}`).join("\n");
+    } else {
+      serialized = JSON.stringify(value);
+    }
+    const fullContent = `${indent}${serialized}${comma}`;
+    if (fullContent.length > SPLITTER_MAX_CHUNK_SIZE) {
+      const textChunks = await this.textFallbackSplitter.splitText(serialized);
+      for (const textChunk of textChunks) {
+        chunks.push({
+          types: ["code"],
+          content: textChunk.content,
+          section: { level, path: [...path2] }
+        });
+      }
+    } else {
+      chunks.push({
+        types: ["code"],
+        content: fullContent,
+        section: { level, path: [...path2] }
+      });
+    }
+  }
 }
 class CodeContentSplitter {
   constructor(options) {
@@ -3255,132 +3518,6 @@ class TableContentSplitter {
     return separator.includes("|") && /^\|?[\s-|]+\|?$/.test(separator);
   }
 }
-class TextContentSplitter {
-  constructor(options) {
-    this.options = options;
-  }
-  /**
-   * Splits text content into chunks while trying to preserve semantic boundaries.
-   * Prefers paragraph breaks, then line breaks, finally falling back to word boundaries.
-   * Always preserves formatting - trimming should be done by higher-level splitters if needed.
-   */
-  async split(content) {
-    if (content.length <= this.options.chunkSize) {
-      return [content];
-    }
-    const words = content.split(/\s+/);
-    const longestWord = words.reduce(
-      (max, word) => word.length > max.length ? word : max
-    );
-    if (longestWord.length > this.options.chunkSize) {
-      throw new MinimumChunkSizeError(longestWord.length, this.options.chunkSize);
-    }
-    const paragraphChunks = this.splitByParagraphs(content);
-    if (this.areChunksValid(paragraphChunks)) {
-      return paragraphChunks;
-    }
-    const lineChunks = this.splitByLines(content);
-    if (this.areChunksValid(lineChunks)) {
-      return this.mergeChunks(lineChunks, "");
-    }
-    const wordChunks = await this.splitByWords(content);
-    return this.mergeChunks(wordChunks, " ");
-  }
-  /**
-   * Checks if all chunks are within the maximum size limit
-   */
-  areChunksValid(chunks) {
-    return chunks.every((chunk) => chunk.length <= this.options.chunkSize);
-  }
-  /**
-   * Splits text into chunks by paragraph boundaries (double newlines)
-   * Preserves all formatting and whitespace including the paragraph separators
-   */
-  splitByParagraphs(text) {
-    const chunks = [];
-    let startPos = 0;
-    const paragraphRegex = /\n\s*\n/g;
-    let match = paragraphRegex.exec(text);
-    while (match !== null) {
-      const endPos = match.index + match[0].length;
-      const chunk = text.slice(startPos, endPos);
-      if (chunk.length > 2) {
-        chunks.push(chunk);
-      }
-      startPos = endPos;
-      match = paragraphRegex.exec(text);
-    }
-    if (startPos < text.length) {
-      const remainingChunk = text.slice(startPos);
-      if (remainingChunk.length > 2) {
-        chunks.push(remainingChunk);
-      }
-    }
-    return chunks.filter(Boolean);
-  }
-  /**
-   * Splits text into chunks by line boundaries
-   * Preserves all formatting and whitespace, including newlines at the end of each line
-   */
-  splitByLines(text) {
-    const chunks = [];
-    let startPos = 0;
-    for (let i = 0; i < text.length; i++) {
-      if (text[i] === "\n") {
-        const chunk = text.slice(startPos, i + 1);
-        chunks.push(chunk);
-        startPos = i + 1;
-      }
-    }
-    if (startPos < text.length) {
-      chunks.push(text.slice(startPos));
-    }
-    return chunks;
-  }
-  /**
-   * Uses LangChain's recursive splitter for word-based splitting as a last resort
-   */
-  async splitByWords(text) {
-    const splitter = new RecursiveCharacterTextSplitter({
-      chunkSize: this.options.chunkSize,
-      chunkOverlap: 0
-    });
-    const chunks = await splitter.splitText(text);
-    return chunks;
-  }
-  /**
-   * Attempts to merge small chunks with previous chunks to minimize fragmentation.
-   * Only merges if combined size is within maxChunkSize.
-   */
-  mergeChunks(chunks, separator) {
-    const mergedChunks = [];
-    let currentChunk = null;
-    for (const chunk of chunks) {
-      if (currentChunk === null) {
-        currentChunk = chunk;
-        continue;
-      }
-      const currentChunkSize = this.getChunkSize(currentChunk);
-      const nextChunkSize = this.getChunkSize(chunk);
-      if (currentChunkSize + nextChunkSize + separator.length <= this.options.chunkSize) {
-        currentChunk = `${currentChunk}${separator}${chunk}`;
-      } else {
-        mergedChunks.push(currentChunk);
-        currentChunk = chunk;
-      }
-    }
-    if (currentChunk) {
-      mergedChunks.push(currentChunk);
-    }
-    return mergedChunks;
-  }
-  getChunkSize(chunk) {
-    return chunk.length;
-  }
-  wrap(content) {
-    return content;
-  }
-}
 class SemanticMarkdownSplitter {
   constructor(preferredChunkSize, maxChunkSize) {
     this.preferredChunkSize = preferredChunkSize;
@@ -6452,45 +6589,6 @@ class MarkdownPipeline extends BasePipeline {
     };
   }
 }
-class TextDocumentSplitter {
-  options;
-  textSplitter;
-  constructor(options = {}) {
-    this.options = {
-      maxChunkSize: options.maxChunkSize ?? SPLITTER_MAX_CHUNK_SIZE
-    };
-    this.textSplitter = new TextContentSplitter({
-      chunkSize: this.options.maxChunkSize
-    });
-  }
-  async splitText(content) {
-    if (!content.trim()) {
-      return [];
-    }
-    try {
-      const chunks = await this.textSplitter.split(content);
-      return chunks.map((chunk) => ({
-        types: ["text"],
-        content: chunk,
-        section: {
-          level: 0,
-          path: []
-        }
-      }));
-    } catch {
-      return [
-        {
-          types: ["text"],
-          content,
-          section: {
-            level: 0,
-            path: []
-          }
-        }
-      ];
-    }
-  }
-}
 class TextPipeline extends BasePipeline {
   middleware;
   splitter;
@@ -8973,9 +9071,10 @@ class DocumentStore {
    * - Single texts that are too large are truncated and retried once
    *
    * @param texts Array of texts to embed
+   * @param isRetry Internal flag to prevent duplicate warning logs
    * @returns Array of embedding vectors
    */
-  async embedDocumentsWithRetry(texts) {
+  async embedDocumentsWithRetry(texts, isRetry = false) {
     if (texts.length === 0) {
       return [];
     }
@@ -8987,26 +9086,27 @@ class DocumentStore {
           const midpoint = Math.floor(texts.length / 2);
           const firstHalf = texts.slice(0, midpoint);
           const secondHalf = texts.slice(midpoint);
-          logger.warn(
-            `⚠️  Batch of ${texts.length} texts exceeded size limit, splitting into ${firstHalf.length} + ${secondHalf.length}`
-          );
+          if (!isRetry) {
+            logger.warn(
+              `⚠️  Batch of ${texts.length} texts exceeded size limit, splitting into ${firstHalf.length} + ${secondHalf.length}`
+            );
+          }
           const [firstEmbeddings, secondEmbeddings] = await Promise.all([
-            this.embedDocumentsWithRetry(firstHalf),
-            this.embedDocumentsWithRetry(secondHalf)
+            this.embedDocumentsWithRetry(firstHalf, true),
+            this.embedDocumentsWithRetry(secondHalf, true)
           ]);
           return [...firstEmbeddings, ...secondEmbeddings];
         } else {
           const text = texts[0];
           const midpoint = Math.floor(text.length / 2);
           const firstHalf = text.substring(0, midpoint);
-          logger.warn(
-            `⚠️  Single text exceeded embedding size limit (${text.length} chars). Truncating at ${firstHalf.length} chars.`
-          );
-          try {
-            const embedding = await this.embedDocumentsWithRetry([firstHalf]);
-            logger.info(
-              `✓ Using embedding from first half of split text (${firstHalf.length} chars)`
+          if (!isRetry) {
+            logger.warn(
+              `⚠️  Single text exceeded embedding size limit (${text.length} chars).`
             );
+          }
+          try {
+            const embedding = await this.embedDocumentsWithRetry([firstHalf], true);
             return embedding;
           } catch (retryError) {
             logger.error(
@@ -9130,8 +9230,8 @@ class DocumentStore {
           const rowId = result2.lastInsertRowid;
           if (this.isVectorSearchEnabled && paddedEmbeddings.length > 0) {
             this.statements.insertEmbedding.run(
-              BigInt(rowId),
-              JSON.stringify(paddedEmbeddings[docIndex])
+              JSON.stringify(paddedEmbeddings[docIndex]),
+              BigInt(rowId)
             );
           }
           docIndex++;
@@ -10770,7 +10870,7 @@ const Layout = ({
   children,
   eventClientConfig
 }) => {
-  const versionString = version || "1.32.0";
+  const versionString = version || "1.33.0";
   const versionInitializer = `versionUpdate({ currentVersion: ${`'${versionString}'`} })`;
   return /* @__PURE__ */ jsxs("html", { lang: "en", children: [
     /* @__PURE__ */ jsxs("head", { children: [
@@ -13082,7 +13182,7 @@ class AppServer {
       try {
         if (telemetry.isEnabled()) {
           telemetry.setGlobalContext({
-            appVersion: "1.32.0",
+            appVersion: "1.33.0",
             appPlatform: process.platform,
             appNodeVersion: process.version,
             appServicesEnabled: this.getActiveServicesList(),
@@ -16569,7 +16669,7 @@ function createCliProgram() {
   const commandStartTimes = /* @__PURE__ */ new Map();
   let globalEventBus = null;
   let globalTelemetryService = null;
-  program.name("docs-mcp-server").description("Unified CLI, MCP Server, and Web Interface for Docs MCP Server.").version("1.32.0").addOption(
+  program.name("docs-mcp-server").description("Unified CLI, MCP Server, and Web Interface for Docs MCP Server.").version("1.33.0").addOption(
     new Option("--verbose", "Enable verbose (debug) logging").conflicts("silent")
   ).addOption(new Option("--silent", "Disable all logging except errors")).addOption(
     new Option("--telemetry", "Enable telemetry collection").env("DOCS_MCP_TELEMETRY").argParser((value) => {
@@ -16603,7 +16703,7 @@ function createCliProgram() {
     if (shouldEnableTelemetry()) {
       if (telemetry.isEnabled()) {
         telemetry.setGlobalContext({
-          appVersion: "1.32.0",
+          appVersion: "1.33.0",
           appPlatform: process.platform,
           appNodeVersion: process.version,
           appInterface: "cli",