mcp-docs-service 0.5.2 → 7.2.0

package/dist/handlers/documents.js

@@ -7,6 +7,16 @@ import fs from "fs/promises";
 import path from "path";
 import { glob } from "glob";
 import { createTwoFilesPatch } from "diff";
+import { writeFileSync } from "fs";
+// Log debug info to file
+function debugLog(message) {
+    try {
+        writeFileSync("docs/debug_log.txt", message + "\n", { flag: "a" });
+    }
+    catch (error) {
+        // Silently fail if we can't write to the file
+    }
+}
 // File editing and diffing utilities
 function normalizeLineEndings(text) {
     return text.replace(/\r\n/g, "\n");
@@ -54,8 +64,12 @@ export function parseFrontmatter(content) {
 }
 export class DocumentHandler {
     docsDir;
-    constructor(docsDir) {
+    singleDocPath;
+    useSingleDoc;
+    constructor(docsDir, useSingleDoc = false) {
         this.docsDir = docsDir;
+        this.useSingleDoc = useSingleDoc;
+        this.singleDocPath = path.join(docsDir, "single_doc.md");
     }
     /**
      * Validates that a path is within the docs directory
@@ -77,13 +91,87 @@ export class DocumentHandler {
      */
     async readDocument(docPath) {
         try {
+            // If single-doc mode is enabled, redirect to the single doc
+            if (this.useSingleDoc) {
+                const singleDocPath = path.join(this.docsDir, "single_doc.md");
+                // Check if the requested path is a specific part
+                if (docPath.startsWith("single_doc_part") && docPath.endsWith(".md")) {
+                    const requestedPartPath = path.join(this.docsDir, docPath);
+                    // Check if the requested part exists
+                    try {
+                        await fs.access(requestedPartPath);
+                        const content = await fs.readFile(requestedPartPath, "utf-8");
+                        return {
+                            content: [{ type: "text", text: content }],
+                            metadata: {
+                                path: docPath,
+                                ...this.parseFrontmatter(content).frontmatter,
+                            },
+                        };
+                    }
+                    catch (error) {
+                        // If the part doesn't exist, fall back to the index
+                        if (await this.fileExists(singleDocPath)) {
+                            const content = await fs.readFile(singleDocPath, "utf-8");
+                            return {
+                                content: [{ type: "text", text: content }],
+                                metadata: {
+                                    path: "single_doc.md",
+                                    ...this.parseFrontmatter(content).frontmatter,
+                                },
+                            };
+                        }
+                    }
+                }
+                // For all other requested documents in single-doc mode, serve the single document
+                if (await this.fileExists(singleDocPath)) {
+                    const content = await fs.readFile(singleDocPath, "utf-8");
+                    // Check if this is an index file (indicating multi-part doc)
+                    const { frontmatter } = this.parseFrontmatter(content);
+                    if (content.includes("DOCUMENTATION INDEX") ||
+                        content.includes("Available Parts")) {
+                        return {
+                            content: [{ type: "text", text: content }],
+                            metadata: {
+                                path: "single_doc.md",
+                                ...frontmatter,
+                                isIndex: true,
+                            },
+                        };
+                    }
+                    return {
+                        content: [{ type: "text", text: content }],
+                        metadata: {
+                            path: "single_doc.md",
+                            ...frontmatter,
+                        },
+                    };
+                }
+                else {
+                    // If single-doc doesn't exist yet, try to generate it
+                    await this.refreshSingleDoc();
+                    // Check if it was created successfully
+                    if (await this.fileExists(singleDocPath)) {
+                        const content = await fs.readFile(singleDocPath, "utf-8");
+                        return {
+                            content: [{ type: "text", text: content }],
+                            metadata: {
+                                path: "single_doc.md",
+                                ...this.parseFrontmatter(content).frontmatter,
+                            },
+                        };
+                    }
+                    // If it still doesn't exist, fall back to the requested path
+                }
+            }
+            // Normal mode or fallback: read the actual requested document
             const validPath = await this.validatePath(docPath);
             const content = await fs.readFile(validPath, "utf-8");
             return {
                 content: [{ type: "text", text: content }],
                 metadata: {
                     path: docPath,
-                    ...parseFrontmatter(content).frontmatter,
+                    ...this.parseFrontmatter(content).frontmatter,
                 },
             };
         }
@@ -97,6 +185,18 @@ export class DocumentHandler {
             };
         }
     }
+    /**
+     * Helper method to check if a file exists
+     */
+    async fileExists(filePath) {
+        try {
+            await fs.access(filePath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
     /**
      * Write a document to the docs directory
      */
@@ -108,7 +208,31 @@ export class DocumentHandler {
                 const dirPath = path.dirname(validPath);
                 await fs.mkdir(dirPath, { recursive: true });
             }
+            // Write the document
             await fs.writeFile(validPath, content, "utf-8");
+            // If in single-doc mode, refresh the single doc
+            if (this.useSingleDoc && !docPath.startsWith("single_doc")) {
+                try {
+                    // Check if we have multi-part docs
+                    const parts = await glob(path.join(this.docsDir, "single_doc_part*.md"));
+                    if (parts.length > 0) {
+                        // If we have multi-part docs, refresh all parts
+                        await this.refreshSingleDoc();
+                    }
+                    else {
+                        // Check if single_doc.md exists
+                        const singleDocPath = path.join(this.docsDir, "single_doc.md");
+                        if (await this.fileExists(singleDocPath)) {
+                            // Refresh the single doc
+                            await this.refreshSingleDoc();
+                        }
+                    }
+                }
+                catch (refreshError) {
+                    // Log but don't fail if refreshing fails
+                    console.error("Error refreshing single doc:", refreshError);
+                }
+            }
             return {
                 content: [{ type: "text", text: `Successfully wrote to ${docPath}` }],
             };
@@ -280,256 +404,311 @@ export class DocumentHandler {
         }
     }
     /**
-     * Create a new folder in the docs directory
+     * Estimate the number of tokens in a text string
+     * This is a simple estimation: ~4 characters per token for English text
      */
-    async createFolder(folderPath, createReadme = true) {
-        try {
-            const validPath = await this.validatePath(folderPath);
-            // Create the directory
-            await fs.mkdir(validPath, { recursive: true });
-            // Create a README.md file if requested
-            if (createReadme) {
-                const readmePath = path.join(validPath, "README.md");
-                const folderName = path.basename(validPath);
-                const content = `---
-title: ${folderName}
-description: Documentation for ${folderName}
-date: ${new Date().toISOString()}
-status: draft
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Splits content into multiple documents based on token limits
+     * Returns the number of chunks created
+     */
+    async createDocumentChunks(sections, template, maxTokens) {
+        // Create a base template for each chunk
+        const projectName = path.basename(path.resolve(this.docsDir, ".."));
+        const footer = "\n---\n\n_This document is automatically maintained and optimized for LLM context. This is part {part} of a multi-part document._\n";
+        // Token budgeting
+        const footerBaseTokens = this.estimateTokens(footer.replace("{part}", "X"));
+        const templateTokens = this.estimateTokens(template);
+        const availableTokensPerChunk = maxTokens - templateTokens - footerBaseTokens;
+        // Track total tokens
+        let currentChunk = 1;
+        let currentTokens = 0;
+        let chunkContent = "";
+        for (const [section, content] of sections) {
+            // Skip empty sections
+            if (content.trim() === `## ${section}`.trim()) {
+                continue;
+            }
+            const sectionTokens = this.estimateTokens(content);
+            // Check if we need to start a new chunk
+            if (currentTokens + sectionTokens > availableTokensPerChunk &&
+                currentTokens > 0) {
+                // Write the current chunk to file
+                const chunkPath = path.join(this.docsDir, `single_doc_part${currentChunk}.md`);
+                const finalChunkContent = template +
+                    chunkContent +
+                    footer.replace("{part}", `${currentChunk}`) +
+                    `\n<!-- Token usage: ${currentTokens + templateTokens + footerBaseTokens}/${maxTokens} -->`;
+                await fs.writeFile(chunkPath, finalChunkContent, "utf-8");
+                // Reset for next chunk
+                currentChunk++;
+                currentTokens = 0;
+                chunkContent = "";
+            }
+            // Add section to current chunk
+            chunkContent += content + "\n";
+            currentTokens += sectionTokens;
+        }
+        // Write the last chunk if there's any content
+        if (chunkContent) {
+            const chunkPath = path.join(this.docsDir, `single_doc_part${currentChunk}.md`);
+            const finalChunkContent = template +
+                chunkContent +
+                footer.replace("{part}", `${currentChunk} of ${currentChunk}`) +
+                `\n<!-- Token usage: ${currentTokens + templateTokens + footerBaseTokens}/${maxTokens} -->`;
+            await fs.writeFile(chunkPath, finalChunkContent, "utf-8");
+        }
+        // Create an index file that lists all parts
+        if (currentChunk > 1) {
+            const indexPath = path.join(this.docsDir, "single_doc.md");
+            let indexContent = `---
+title: ${projectName} Documentation Index
+description: Index of documentation parts optimized for LLM context
+author: Documentation System
+date: "${new Date().toISOString()}"
+tags:
+  - documentation
+  - llm-optimized
+status: published
+version: 1.0.0
 ---
 
-# ${folderName}
+# ${projectName.toUpperCase()}: DOCUMENTATION INDEX
+
+_This documentation has been split into multiple parts due to its size. Each part is optimized for LLM context._
+
+## Available Parts
 
-This is the documentation for ${folderName}.
 `;
-                await fs.writeFile(readmePath, content, "utf-8");
+            for (let i = 1; i <= currentChunk; i++) {
+                indexContent += `- [Part ${i}](single_doc_part${i}.md)\n`;
             }
-            return {
-                content: [
-                    { type: "text", text: `Successfully created folder: ${folderPath}` },
-                ],
-                metadata: {
-                    path: folderPath,
-                    readme: createReadme ? path.join(folderPath, "README.md") : null,
-                },
-            };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error creating folder: ${errorMessage}` },
-                ],
-                isError: true,
-            };
+            indexContent +=
+                "\n---\n\n_This index was automatically generated to manage documentation that exceeds token limits._\n";
+            await fs.writeFile(indexPath, indexContent, "utf-8");
         }
+        return currentChunk;
     }
     /**
-     * Move a document to a new location
+     * Refresh the single document by recompiling all markdown files
      */
-    async moveDocument(sourcePath, destinationPath, updateReferences = true) {
+    async refreshSingleDoc() {
+        const singleDocPath = path.join(this.docsDir, "single_doc.md");
+        const MAX_TOKENS = 100000; // Limit to 100k tokens to fit within LLM context windows
+        const MULTI_DOC_THRESHOLD = 9000; // Lower threshold for testing - if estimated tokens exceed this, create multiple docs
         try {
-            const validSourcePath = await this.validatePath(sourcePath);
-            const validDestPath = await this.validatePath(destinationPath);
-            // Check if source exists
-            try {
-                await fs.access(validSourcePath);
-            }
-            catch {
-                throw new Error(`Source file does not exist: ${sourcePath}`);
-            }
-            // Create destination directory if it doesn't exist
-            const destDir = path.dirname(validDestPath);
-            await fs.mkdir(destDir, { recursive: true });
-            // Read the source file
-            const content = await fs.readFile(validSourcePath, "utf-8");
-            // Write to destination
-            await fs.writeFile(validDestPath, content, "utf-8");
-            // Delete the source file
-            await fs.unlink(validSourcePath);
-            // Update references if requested
-            let referencesUpdated = 0;
-            if (updateReferences) {
-                referencesUpdated = await this.updateReferences(sourcePath, destinationPath);
-            }
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: `Successfully moved document from ${sourcePath} to ${destinationPath}` +
-                            (referencesUpdated > 0
-                                ? `. Updated ${referencesUpdated} references.`
-                                : ""),
-                    },
-                ],
-                metadata: {
-                    sourcePath,
-                    destinationPath,
-                    referencesUpdated,
-                },
+            // Get all markdown files
+            const files = await glob(path.join(this.docsDir, "**/*.md"));
+            // Filter out the single_doc.md itself and any single_doc_part*.md files
+            const docsToProcess = files.filter((file) => !path.basename(file).startsWith("single_doc"));
+            debugLog(`Processing ${docsToProcess.length} documents`);
+            // Create a template for the single doc
+            const projectName = path.basename(path.resolve(this.docsDir, ".."));
+            const template = `---
+title: ${projectName} Documentation
+description: Complete documentation optimized for LLM context
+author: Documentation System
+date: "${new Date().toISOString()}"
+tags:
+  - documentation
+  - llm-optimized
+status: published
+version: 1.0.0
+---
+
+# ${projectName.toUpperCase()}: COMPLETE REFERENCE
+
+_This document is automatically generated and optimized for LLM context. It contains the complete reference for ${projectName}._
+
+## TABLE OF CONTENTS
+
+1. [Overview](#overview)
+2. [Core Concepts](#core-concepts)
+3. [Features](#features)
+4. [API Reference](#api-reference)
+5. [Usage Examples](#usage-examples)
+6. [Integration](#integration)
+7. [Best Practices](#best-practices)
+8. [Troubleshooting](#troubleshooting)
+9. [Additional Information](#additional-information)
+
+`;
+            // Initialize sections
+            const sections = {
+                OVERVIEW: "## OVERVIEW\n\n",
+                "CORE CONCEPTS": "## CORE CONCEPTS\n\n",
+                FEATURES: "## FEATURES\n\n",
+                "API REFERENCE": "## API REFERENCE\n\n",
+                "USAGE EXAMPLES": "## USAGE EXAMPLES\n\n",
+                INTEGRATION: "## INTEGRATION\n\n",
+                "BEST PRACTICES": "## BEST PRACTICES\n\n",
+                TROUBLESHOOTING: "## TROUBLESHOOTING\n\n",
+                "ADDITIONAL INFORMATION": "## ADDITIONAL INFORMATION\n\n",
             };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error moving document: ${errorMessage}` },
-                ],
-                isError: true,
+            // Define section priorities (1 = highest priority)
+            const sectionPriorities = {
+                OVERVIEW: 1,
+                FEATURES: 2,
+                "API REFERENCE": 3,
+                "USAGE EXAMPLES": 4,
+                "CORE CONCEPTS": 5,
+                INTEGRATION: 6,
+                "BEST PRACTICES": 7,
+                TROUBLESHOOTING: 8,
+                "ADDITIONAL INFORMATION": 9,
             };
-        }
-    }
-    /**
-     * Rename a document
-     */
-    async renameDocument(docPath, newName, updateReferences = true) {
-        try {
-            const validPath = await this.validatePath(docPath);
-            // Get directory and extension
-            const dir = path.dirname(validPath);
-            const ext = path.extname(validPath);
-            // Create new path
-            const newPath = path.join(dir, newName + ext);
-            const validNewPath = await this.validatePath(newPath);
-            // Check if source exists
-            try {
-                await fs.access(validPath);
-            }
-            catch {
-                throw new Error(`Source file does not exist: ${docPath}`);
-            }
-            // Check if destination already exists
-            try {
-                await fs.access(validNewPath);
-                throw new Error(`Destination file already exists: ${newPath}`);
-            }
-            catch (error) {
-                // If error is "file doesn't exist", that's good
-                if (!(error instanceof Error &&
-                    error.message.includes("Destination file already exists"))) {
-                    // Continue with rename
+            // Read each document and add to appropriate section
+            for (const file of docsToProcess) {
+                const relativePath = path.relative(this.docsDir, file);
+                const content = await fs.readFile(file, "utf-8");
+                // Parse frontmatter
+                const { frontmatter, content: docContent } = this.parseFrontmatter(content);
+                // Skip empty documents
+                if (!docContent.trim())
+                    continue;
+                // Determine which section to add to
+                let targetSection = "ADDITIONAL INFORMATION";
+                if (file.includes("api") ||
+                    (frontmatter.tags && frontmatter.tags.includes("api"))) {
+                    targetSection = "API REFERENCE";
                 }
-                else {
-                    throw error;
+                else if (file.includes("concept") ||
+                    (frontmatter.tags && frontmatter.tags.includes("concept"))) {
+                    targetSection = "CORE CONCEPTS";
                 }
-            }
-            // Read the source file
-            const content = await fs.readFile(validPath, "utf-8");
-            // Parse frontmatter
-            const { frontmatter, content: docContent } = parseFrontmatter(content);
-            // Update title in frontmatter if it exists
-            if (frontmatter.title) {
-                frontmatter.title = newName;
-            }
-            // Reconstruct content with updated frontmatter
-            let frontmatterStr = "---\n";
-            for (const [key, value] of Object.entries(frontmatter)) {
-                if (Array.isArray(value)) {
-                    frontmatterStr += `${key}:\n`;
-                    for (const item of value) {
-                        frontmatterStr += `  - ${item}\n`;
-                    }
+                else if (file.includes("feature") ||
+                    (frontmatter.tags && frontmatter.tags.includes("feature"))) {
+                    targetSection = "FEATURES";
                 }
-                else {
-                    frontmatterStr += `${key}: ${value}\n`;
+                else if (file.includes("example") ||
+                    (frontmatter.tags && frontmatter.tags.includes("example"))) {
+                    targetSection = "USAGE EXAMPLES";
                 }
+                else if (file.includes("integration") ||
+                    (frontmatter.tags && frontmatter.tags.includes("integration"))) {
+                    targetSection = "INTEGRATION";
+                }
+                else if (file.includes("best-practice") ||
+                    (frontmatter.tags && frontmatter.tags.includes("best-practice"))) {
+                    targetSection = "BEST PRACTICES";
+                }
+                else if (file.includes("troubleshoot") ||
+                    (frontmatter.tags && frontmatter.tags.includes("troubleshoot"))) {
+                    targetSection = "TROUBLESHOOTING";
+                }
+                else if (file.includes("overview") ||
+                    (frontmatter.tags && frontmatter.tags.includes("overview")) ||
+                    file.endsWith("README.md")) {
+                    targetSection = "OVERVIEW";
+                }
+                // Add document to the section
+                const title = frontmatter.title || path.basename(file, ".md");
+                const formattedContent = `### ${title}\n\n_Source: ${relativePath}_\n\n${docContent}\n\n`;
+                sections[targetSection] += formattedContent;
             }
-            frontmatterStr += "---\n\n";
-            const updatedContent = frontmatterStr + docContent;
-            // Write to new path
-            await fs.writeFile(validNewPath, updatedContent, "utf-8");
-            // Delete the source file
-            await fs.unlink(validPath);
-            // Update references if requested
-            let referencesUpdated = 0;
-            if (updateReferences) {
-                const relativeSrcPath = path.relative(this.docsDir, validPath);
-                const relativeDestPath = path.relative(this.docsDir, validNewPath);
-                referencesUpdated = await this.updateReferences(relativeSrcPath, relativeDestPath);
-            }
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: `Successfully renamed document from ${docPath} to ${newName}${ext}` +
-                            (referencesUpdated > 0
-                                ? `. Updated ${referencesUpdated} references.`
-                                : ""),
-                    },
-                ],
-                metadata: {
-                    originalPath: docPath,
-                    newPath: path.relative(this.docsDir, validNewPath),
-                    referencesUpdated,
-                },
-            };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error renaming document: ${errorMessage}` },
-                ],
-                isError: true,
-            };
-        }
-    }
-    /**
-     * Update navigation order for a document
-     */
-    async updateNavigationOrder(docPath, order) {
-        try {
-            const validPath = await this.validatePath(docPath);
-            // Check if file exists
-            try {
-                await fs.access(validPath);
-            }
-            catch {
-                throw new Error(`File does not exist: ${docPath}`);
+            // Calculate estimated total size
+            let estimatedTotalTokens = this.estimateTokens(template);
+            debugLog(`Template tokens: ${estimatedTotalTokens}`);
+            Object.entries(sections).forEach(([section, content]) => {
+                const sectionTokens = this.estimateTokens(content);
+                debugLog(`Section "${section}" tokens: ${sectionTokens}`);
+                estimatedTotalTokens += sectionTokens;
+            });
+            debugLog(`Total estimated tokens: ${estimatedTotalTokens}`);
+            debugLog(`Multi-doc threshold: ${MULTI_DOC_THRESHOLD}`);
+            // Check if we need to split into multiple documents
+            if (estimatedTotalTokens > MULTI_DOC_THRESHOLD) {
+                debugLog("Threshold exceeded, creating multiple documents");
+                // Delete any existing single_doc_part*.md files
+                const existingParts = await glob(path.join(this.docsDir, "single_doc_part*.md"));
+                for (const part of existingParts) {
+                    await fs.unlink(part);
+                }
+                // Create multiple document parts
+                const prioritizedSections = Object.entries(sections).sort(([sectionA], [sectionB]) => sectionPriorities[sectionA] - sectionPriorities[sectionB]);
+                const numChunks = await this.createDocumentChunks(prioritizedSections, template, MAX_TOKENS);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Documentation exceeded token limits (${estimatedTotalTokens} estimated tokens). Created ${numChunks} document parts.`,
+                        },
+                    ],
+                };
             }
-            // Read the file
-            const content = await fs.readFile(validPath, "utf-8");
-            // Parse frontmatter
-            const { frontmatter, content: docContent } = parseFrontmatter(content);
-            // Update order in frontmatter
-            frontmatter.order = order;
-            // Reconstruct content with updated frontmatter
-            let frontmatterStr = "---\n";
-            for (const [key, value] of Object.entries(frontmatter)) {
-                if (Array.isArray(value)) {
-                    frontmatterStr += `${key}:\n`;
-                    for (const item of value) {
-                        frontmatterStr += `  - ${item}\n`;
-                    }
+            // If not exceeding the multi-doc threshold, continue with standard approach
+            let totalTokens = this.estimateTokens(template);
+            let includedDocuments = 0;
+            let truncatedSections = 0;
+            let omittedSections = 0;
+            // Sort sections by priority
+            const prioritizedSections = Object.entries(sections).sort(([sectionA], [sectionB]) => sectionPriorities[sectionA] - sectionPriorities[sectionB]);
+            // Compile the final document with token limits
+            let finalContent = template;
+            const footer = "\n---\n\n_This document is automatically maintained and optimized for LLM context._\n";
+            const footerTokens = this.estimateTokens(footer);
+            // Reserve tokens for the footer
+            const availableTokens = MAX_TOKENS - totalTokens - footerTokens;
+            for (const [section, content] of prioritizedSections) {
+                // Skip empty sections
+                if (content.trim() === `## ${section}`.trim()) {
+                    continue;
+                }
+                const sectionTokens = this.estimateTokens(content);
+                if (totalTokens + sectionTokens <= MAX_TOKENS - footerTokens) {
+                    // Full section fits
+                    finalContent += content + "\n";
+                    totalTokens += sectionTokens;
+                    // Count documents in this section
+                    const docMatches = content.match(/### .+/g);
+                    includedDocuments += docMatches ? docMatches.length : 0;
                 }
                 else {
-                    frontmatterStr += `${key}: ${value}\n`;
+                    // Section doesn't fit entirely - try to include a truncated version
+                    omittedSections++;
+                    // Add a note about truncation
+                    const truncationNote = `## ${section} (Content Limit Reached)\n\nSome content in this section was omitted to fit within token limits.\n\n`;
+                    const truncationTokens = this.estimateTokens(truncationNote);
+                    if (totalTokens + truncationTokens <= MAX_TOKENS - footerTokens) {
+                        finalContent += truncationNote;
+                        totalTokens += truncationTokens;
+                        truncatedSections++;
+                    }
                 }
             }
-            frontmatterStr += "---\n\n";
-            const updatedContent = frontmatterStr + docContent;
-            // Write updated content
-            await fs.writeFile(validPath, updatedContent, "utf-8");
+            // Add footer with token usage information
+            finalContent += footer;
+            // Add token usage metadata
+            const tokenUsageInfo = `\n<!-- Token usage: ${totalTokens}/${MAX_TOKENS} (${Math.round((totalTokens / MAX_TOKENS) * 100)}%) -->\n`;
+            finalContent += tokenUsageInfo;
+            // Delete any existing part files since we're creating a single file
+            const existingParts = await glob(path.join(this.docsDir, "single_doc_part*.md"));
+            for (const part of existingParts) {
+                await fs.unlink(part);
+            }
+            // Write the file
+            await fs.writeFile(singleDocPath, finalContent, "utf-8");
             return {
                 content: [
                     {
                         type: "text",
-                        text: `Successfully updated navigation order for ${docPath} to ${order}`,
+                        text: `Successfully refreshed single_doc.md from ${docsToProcess.length} documents. Included ${includedDocuments} documents using ${totalTokens} tokens (${Math.round((totalTokens / MAX_TOKENS) * 100)}% of limit).${truncatedSections > 0
+                            ? ` ${truncatedSections} sections were truncated.`
+                            : ""}${omittedSections > 0
+                            ? ` ${omittedSections} sections were omitted due to token limits.`
+                            : ""}`,
                     },
                 ],
-                metadata: {
-                    path: docPath,
-                    order,
-                },
             };
         }
         catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
             return {
                 content: [
                     {
                         type: "text",
-                        text: `Error updating navigation order: ${errorMessage}`,
+                        text: `Error refreshing single_doc.md: ${error instanceof Error ? error.message : String(error)}`,
                     },
                 ],
                 isError: true,
@@ -537,215 +716,109 @@ This is the documentation for ${folderName}.
         }
     }
     /**
-     * Create a new navigation section
+     * Parse frontmatter from a markdown document
      */
-    async createSection(title, sectionPath, order) {
+    parseFrontmatter(content) {
+        // If no content, return empty
+        if (!content || content.trim() === "") {
+            return { frontmatter: {}, content: "" };
+        }
+        // Check if the content has frontmatter (starts with ---)
+        if (!content.startsWith("---")) {
+            return { frontmatter: {}, content };
+        }
         try {
-            // Create the directory for the section
-            const validPath = await this.validatePath(sectionPath);
-            await fs.mkdir(validPath, { recursive: true });
-            // Create an index.md file for the section
-            const indexPath = path.join(validPath, "index.md");
-            const validIndexPath = await this.validatePath(indexPath);
-            // Create content with frontmatter
-            let content = "---\n";
-            content += `title: ${title}\n`;
-            content += `description: ${title} section\n`;
-            content += `date: ${new Date().toISOString()}\n`;
-            content += `status: published\n`;
-            if (order !== undefined) {
-                content += `order: ${order}\n`;
+            // Find the end of the frontmatter
+            const endIndex = content.indexOf("---", 3);
+            if (endIndex === -1) {
+                return { frontmatter: {}, content };
             }
-            content += "---\n\n";
-            content += `# ${title}\n\n`;
-            content += `Welcome to the ${title} section.\n`;
-            // Write the index file
-            await fs.writeFile(validIndexPath, content, "utf-8");
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: `Successfully created section: ${title} at ${sectionPath}`,
-                    },
-                ],
-                metadata: {
-                    title,
-                    path: sectionPath,
-                    indexPath: path.join(sectionPath, "index.md"),
-                    order,
-                },
-            };
+            // Extract frontmatter and parse as YAML
+            const frontmatterText = content.substring(3, endIndex).trim();
+            const frontmatter = this.parseFrontmatterLines(frontmatterText);
+            // Extract the content after frontmatter
+            const contentWithoutFrontmatter = content.substring(endIndex + 3).trim();
+            return { frontmatter, content: contentWithoutFrontmatter };
         }
         catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error creating section: ${errorMessage}` },
-                ],
-                isError: true,
-            };
+            console.error("Error parsing frontmatter:", error);
+            return { frontmatter: {}, content };
         }
     }
     /**
-     * Update references to a moved or renamed document
-     * @private
+     * Parse frontmatter lines into an object
      */
-    async updateReferences(oldPath, newPath) {
-        // Normalize paths for comparison
-        const normalizedOldPath = oldPath.replace(/\\/g, "/");
-        const normalizedNewPath = newPath.replace(/\\/g, "/");
-        // Find all markdown files
-        const files = await glob("**/*.md", { cwd: this.docsDir });
-        let updatedCount = 0;
-        for (const file of files) {
-            const filePath = path.join(this.docsDir, file);
-            const content = await fs.readFile(filePath, "utf-8");
-            // Look for references to the old path
-            // Match markdown links: [text](path)
-            const linkRegex = new RegExp(`\\[([^\\]]+)\\]\\(${normalizedOldPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\)`, "g");
-            // Match direct path references
-            const pathRegex = new RegExp(normalizedOldPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g");
-            // Replace references
-            let updatedContent = content.replace(linkRegex, `[$1](${normalizedNewPath})`);
-            updatedContent = updatedContent.replace(pathRegex, normalizedNewPath);
-            // If content changed, write the updated file
-            if (updatedContent !== content) {
-                await fs.writeFile(filePath, updatedContent, "utf-8");
-                updatedCount++;
-            }
-        }
-        return updatedCount;
-    }
-    /**
-     * Validate links in documentation
-     */
-    async validateLinks(basePath = "", recursive = true) {
-        try {
-            const validBasePath = await this.validatePath(basePath || this.docsDir);
-            // Find all markdown files
-            const pattern = recursive ? "**/*.md" : "*.md";
-            const files = await glob(pattern, { cwd: validBasePath });
-            const brokenLinks = [];
-            // Check each file for links
-            for (const file of files) {
-                const filePath = path.join(validBasePath, file);
-                const content = await fs.readFile(filePath, "utf-8");
-                const lines = content.split("\n");
-                // Find markdown links: [text](path)
-                const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
-                for (let i = 0; i < lines.length; i++) {
-                    const line = lines[i];
-                    let match;
-                    while ((match = linkRegex.exec(line)) !== null) {
-                        const [, , linkPath] = match;
-                        // Skip external links and anchors
-                        if (linkPath.startsWith("http://") ||
-                            linkPath.startsWith("https://") ||
-                            linkPath.startsWith("#") ||
-                            linkPath.startsWith("mailto:")) {
-                            continue;
-                        }
-                        // Resolve the link path relative to the current file
-                        const fileDir = path.dirname(filePath);
-                        const resolvedPath = path.resolve(fileDir, linkPath);
-                        // Check if the link target exists
-                        try {
-                            await fs.access(resolvedPath);
-                        }
-                        catch {
-                            brokenLinks.push({
-                                file: path.relative(this.docsDir, filePath),
-                                link: linkPath,
-                                lineNumber: i + 1,
-                            });
-                        }
+    parseFrontmatterLines(frontmatterText) {
+        const frontmatter = {};
+        const lines = frontmatterText.split("\n");
+        let currentKey = null;
+        let isMultilineValue = false;
+        let multilineValue = "";
+        let isArray = false;
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            if (trimmedLine === "")
+                continue;
+            if (isMultilineValue) {
+                if (trimmedLine.startsWith("  -")) {
+                    // Array item
+                    const value = trimmedLine.substring(3).trim();
+                    if (!Array.isArray(frontmatter[currentKey])) {
+                        frontmatter[currentKey] = [];
+                    }
+                    frontmatter[currentKey].push(value);
+                }
+                else if (trimmedLine.startsWith("  ")) {
+                    // Continuation of multiline value
+                    multilineValue += "\n" + trimmedLine.substring(2);
+                }
+                else {
+                    // End of multiline value
+                    if (multilineValue && !isArray) {
+                        frontmatter[currentKey] = multilineValue.trim();
+                        multilineValue = "";
                     }
+                    isMultilineValue = false;
+                    isArray = false;
+                    currentKey = null;
                 }
             }
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: brokenLinks.length > 0
-                            ? `Found ${brokenLinks.length} broken links in ${files.length} files`
-                            : `No broken links found in ${files.length} files`,
-                    },
-                ],
-                metadata: {
-                    brokenLinks,
-                    filesChecked: files.length,
-                    basePath: path.relative(this.docsDir, validBasePath),
-                },
-            };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error validating links: ${errorMessage}` },
-                ],
-                isError: true,
-            };
-        }
-    }
-    /**
-     * Validate metadata in documentation
-     */
-    async validateMetadata(basePath = "", requiredFields) {
-        try {
-            const validBasePath = await this.validatePath(basePath || this.docsDir);
-            // Default required fields if not specified
-            const fields = requiredFields || ["title", "description", "status"];
-            // Find all markdown files
-            const files = await glob("**/*.md", { cwd: validBasePath });
-            const missingMetadata = [];
-            // Check each file for metadata
-            for (const file of files) {
-                const filePath = path.join(validBasePath, file);
-                const content = await fs.readFile(filePath, "utf-8");
-                // Parse frontmatter
-                const { frontmatter } = parseFrontmatter(content);
-                // Check for required fields
-                const missing = fields.filter((field) => !frontmatter[field]);
-                if (missing.length > 0) {
-                    missingMetadata.push({
-                        file: path.relative(this.docsDir, filePath),
-                        missingFields: missing,
-                    });
+            if (!isMultilineValue) {
+                const colonIndex = trimmedLine.indexOf(":");
+                if (colonIndex > 0) {
+                    currentKey = trimmedLine.substring(0, colonIndex).trim();
+                    const value = trimmedLine.substring(colonIndex + 1).trim();
+                    if (value === "" || value === "|" || value === ">") {
+                        // Multiline value
+                        isMultilineValue = true;
+                        multilineValue = "";
+                        isArray = false;
+                    }
+                    else if (value === "[]" ||
+                        (value.startsWith("[") && value.endsWith("]"))) {
+                        // Array
+                        frontmatter[currentKey] =
+                            value === "[]"
+                                ? []
+                                : value
+                                    .substring(1, value.length - 1)
+                                    .split(",")
+                                    .map((v) => v.trim());
+                    }
+                    else {
+                        // Simple value
+                        if (value === "true")
+                            frontmatter[currentKey] = true;
+                        else if (value === "false")
+                            frontmatter[currentKey] = false;
+                        else if (!isNaN(Number(value)))
+                            frontmatter[currentKey] = Number(value);
+                        else
+                            frontmatter[currentKey] = value.replace(/^["'](.*)["']$/, "$1");
+                    }
                 }
             }
-            // Calculate completeness percentage
-            const totalFields = files.length * fields.length;
-            const missingFields = missingMetadata.reduce((sum, item) => sum + item.missingFields.length, 0);
-            const completenessPercentage = totalFields > 0
-                ? Math.round(((totalFields - missingFields) / totalFields) * 100)
-                : 100;
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: missingMetadata.length > 0
-                            ? `Found ${missingMetadata.length} files with missing metadata. Completeness: ${completenessPercentage}%`
-                            : `All ${files.length} files have complete metadata. Completeness: 100%`,
-                    },
-                ],
-                metadata: {
-                    missingMetadata,
-                    filesChecked: files.length,
-                    requiredFields: fields,
-                    completenessPercentage,
-                    basePath: path.relative(this.docsDir, validBasePath),
-                },
-            };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                content: [
-                    { type: "text", text: `Error validating metadata: ${errorMessage}` },
-                ],
-                isError: true,
-            };
         }
+        return frontmatter;
     }
 }