@cosmocoder/mcp-web-docs 1.2.0 → 1.4.0

package/build/index.js

@@ -23,7 +23,7 @@ import { AuthManager } from './crawler/auth.js';
 import { fetchFavicon } from './util/favicon.js';
 import { generateDocId } from './util/docs.js';
 import { logger } from './util/logger.js';
-import { StorageStateSchema, safeJsonParse, validateToolArgs, sanitizeErrorMessage, detectPromptInjection, wrapExternalContent, addInjectionWarnings, SessionExpiredError, AddDocumentationArgsSchema, AuthenticateArgsSchema, ClearAuthArgsSchema, SearchDocumentationArgsSchema, ReindexDocumentationArgsSchema, DeleteDocumentationArgsSchema, SetTagsArgsSchema, } from './util/security.js';
+import { StorageStateSchema, safeJsonParse, validateToolArgs, sanitizeErrorMessage, detectPromptInjection, wrapExternalContent, addInjectionWarnings, SessionExpiredError, AddDocumentationArgsSchema, AuthenticateArgsSchema, ClearAuthArgsSchema, SearchDocumentationArgsSchema, ReindexDocumentationArgsSchema, DeleteDocumentationArgsSchema, SetTagsArgsSchema, CreateCollectionArgsSchema, DeleteCollectionArgsSchema, UpdateCollectionArgsSchema, GetCollectionArgsSchema, AddToCollectionArgsSchema, RemoveFromCollectionArgsSchema, SearchCollectionArgsSchema, } from './util/security.js';
 class WebDocsServer {
     server;
     config;
@@ -128,7 +128,17 @@ class WebDocsServer {
                     name: 'add_documentation',
                     description: `Add new documentation site for indexing. Supports authenticated sites via the auth options.
 
-IMPORTANT: Before calling this tool, ask the user if they want to restrict crawling to a specific path prefix. For example, if indexing https://docs.example.com/api/v2/overview, the user might want to restrict to '/api/v2' to avoid crawling unrelated sections of the site.`,
+IMPORTANT: Before calling this tool, ask the user if they want to restrict crawling to a specific path prefix. For example, if indexing https://docs.example.com/api/v2/overview, the user might want to restrict to '/api/v2' to avoid crawling unrelated sections of the site.
+
+VERSIONING: If the user is indexing documentation for a versioned software package/library (e.g., React, Vue, Python, a database, an SDK), ask what version they want to associate with this documentation. Many packages have multiple versions with different APIs.
+
+Do NOT ask about versioning for:
+- Internal company documentation (wikis, best practices, runbooks)
+- Single-version products or services
+- Documentation the user indicates should always reflect "latest"
+
+Examples where version matters: "React 18", "Python 3.11", "PostgreSQL 15", "Next.js 14"
+Examples where version doesn't matter: "Company engineering handbook", "AWS console docs", "Confluence spaces"`,
                     inputSchema: {
                         type: 'object',
                         properties: {
@@ -153,6 +163,10 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
                                 items: { type: 'string' },
                                 description: 'Optional tags to categorize the documentation (e.g., ["frontend", "mycompany"]). Tags help filter search results across multiple documentation sites.',
                             },
+                            version: {
+                                type: 'string',
+                                description: 'Optional version identifier for versioned package documentation (e.g., "18", "v6.4", "3.11", "latest"). Helps distinguish between multiple versions of the same package.',
+                            },
                             auth: {
                                 type: 'object',
                                 description: 'Authentication options for protected documentation sites',
@@ -361,6 +375,143 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
                         properties: {},
                     },
                 },
+                // ============ Collection Tools ============
+                {
+                    name: 'create_collection',
+                    description: 'Create a new collection to group related documentation sites. Collections help organize docs by project or context (e.g., "My React Project" with React + Next.js + TypeScript docs).',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Unique name for the collection (e.g., "My React Project", "Backend APIs")',
+                            },
+                            description: {
+                                type: 'string',
+                                description: 'Optional description of what this collection contains',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'delete_collection',
+                    description: 'Delete a collection. The documentation sites in the collection are NOT deleted, only the collection grouping.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection to delete',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'update_collection',
+                    description: "Update a collection's name or description.",
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Current name of the collection',
+                            },
+                            newName: {
+                                type: 'string',
+                                description: 'Optional new name for the collection',
+                            },
+                            description: {
+                                type: 'string',
+                                description: 'Optional new description for the collection',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'list_collections',
+                    description: 'List all collections with their document counts. Use this to see available collections for context switching.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {},
+                    },
+                },
+                {
+                    name: 'get_collection',
+                    description: 'Get details of a specific collection including all its documentation sites.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'add_to_collection',
+                    description: 'Add one or more documentation sites to a collection. Sites must already be indexed.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                            urls: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'URLs of indexed documentation sites to add (max 50)',
+                            },
+                        },
+                        required: ['name', 'urls'],
+                    },
+                },
+                {
+                    name: 'remove_from_collection',
+                    description: 'Remove one or more documentation sites from a collection. The sites remain indexed, just removed from the collection.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                            urls: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'URLs of documentation sites to remove from the collection',
+                            },
+                        },
+                        required: ['name', 'urls'],
+                    },
+                },
+                {
+                    name: 'search_collection',
+                    description: 'Search for documentation within a specific collection. This is useful for focused searches within a project context. Uses the same hybrid search (full-text + semantic) as search_documentation.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection to search in',
+                            },
+                            query: {
+                                type: 'string',
+                                description: 'Search query - be specific and include unique terms',
+                            },
+                            limit: {
+                                type: 'number',
+                                description: 'Maximum number of results (default: 10)',
+                            },
+                        },
+                        required: ['name', 'query'],
+                    },
+                },
             ],
         }));
         // Handle tool calls
@@ -390,6 +541,23 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
                     return this.handleSetTags(request.params.arguments);
                 case 'list_tags':
                     return this.handleListTags();
+                // Collection handlers
+                case 'create_collection':
+                    return this.handleCreateCollection(request.params.arguments);
+                case 'delete_collection':
+                    return this.handleDeleteCollection(request.params.arguments);
+                case 'update_collection':
+                    return this.handleUpdateCollection(request.params.arguments);
+                case 'list_collections':
+                    return this.handleListCollections();
+                case 'get_collection':
+                    return this.handleGetCollection(request.params.arguments);
+                case 'add_to_collection':
+                    return this.handleAddToCollection(request.params.arguments);
+                case 'remove_from_collection':
+                    return this.handleRemoveFromCollection(request.params.arguments);
+                case 'search_collection':
+                    return this.handleSearchCollection(request.params.arguments);
                 default:
                     throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${request.params.name}`);
             }
@@ -404,7 +572,7 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
         catch (error) {
             throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
         }
-        const { url, title, id, pathPrefix, tags, auth: authOptions } = validatedArgs;
+        const { url, title, id, pathPrefix, tags, version, auth: authOptions } = validatedArgs;
         // Additional SSRF protection check
         if (!isValidPublicUrl(url)) {
             throw new McpError(ErrorCode.InvalidParams, 'Access to private networks is blocked');
@@ -472,7 +640,7 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
         // Start indexing process
         this.statusTracker.startIndexing(docId, normalizedUrl, docTitle);
         // Start indexing in the background with abort support
-        const operationPromise = this.indexAndAdd(docId, normalizedUrl, docTitle, false, controller.signal, pathPrefix, authInfo, tags)
+        const operationPromise = this.indexAndAdd(docId, normalizedUrl, docTitle, false, controller.signal, pathPrefix, authInfo, tags, version)
             .catch((error) => {
             const err = error;
             if (err?.name !== 'AbortError') {
@@ -615,8 +783,9 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
                 authDomain: doc.authDomain || new URL(normalizedUrl).hostname,
             }
             : undefined;
-        // Preserve existing tags during reindex
+        // Preserve existing tags and version during reindex
         const existingTags = doc.tags;
+        const existingVersion = doc.version;
         // Cancel any existing operation for this URL
         const wasCancelled = this.indexingQueue.isIndexing(normalizedUrl);
         const controller = await this.indexingQueue.startOperation(normalizedUrl);
@@ -627,8 +796,8 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
             logger.info(`[Progress] Registered token for ${docId}: ${progressToken}`);
         }
         this.statusTracker.startIndexing(docId, normalizedUrl, doc.title);
-        // Start reindexing in the background with abort support (preserving existing tags)
-        const operationPromise = this.indexAndAdd(docId, normalizedUrl, doc.title, true, controller.signal, undefined, authInfo, existingTags)
+        // Start reindexing in the background with abort support (preserving existing tags and version)
+        const operationPromise = this.indexAndAdd(docId, normalizedUrl, doc.title, true, controller.signal, undefined, authInfo, existingTags, existingVersion)
             .catch((error) => {
             const err = error;
             if (err?.name !== 'AbortError') {
@@ -941,7 +1110,319 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
             ],
         };
     }
-    async indexAndAdd(id, url, title, reIndex = false, signal, pathPrefix, authInfo, tags) {
+    // ============ Collection Handlers ============
+    /**
+     * Handle creating a new collection
+     */
+    async handleCreateCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, CreateCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, description } = validatedArgs;
+        try {
+            await this.store.createCollection(name, description);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection "${name}" created successfully`,
+                            collection: {
+                                name,
+                                description,
+                            },
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('already exists')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to create collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle deleting a collection
+     */
+    async handleDeleteCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, DeleteCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name } = validatedArgs;
+        try {
+            await this.store.deleteCollection(name);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection "${name}" deleted. Documentation sites remain indexed.`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to delete collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle updating a collection's metadata
+     */
+    async handleUpdateCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, UpdateCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, newName, description } = validatedArgs;
+        // Must provide at least one field to update
+        if (newName === undefined && description === undefined) {
+            throw new McpError(ErrorCode.InvalidParams, 'Must provide newName or description to update');
+        }
+        try {
+            await this.store.updateCollection(name, { newName, description });
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection updated successfully`,
+                            collection: {
+                                name: newName ?? name,
+                                description,
+                            },
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found') || safeMessage.includes('already exists')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to update collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle listing all collections
+     */
+    async handleListCollections() {
+        const collections = await this.store.listCollections();
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        collections,
+                        total: collections.length,
+                    }, null, 2),
+                },
+            ],
+        };
+    }
+    /**
+     * Handle getting a specific collection with its documents
+     */
+    async handleGetCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, GetCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name } = validatedArgs;
+        const collection = await this.store.getCollection(name);
+        if (!collection) {
+            throw new McpError(ErrorCode.InvalidParams, `Collection "${name}" not found`);
+        }
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(collection, null, 2),
+                },
+            ],
+        };
+    }
+    /**
+     * Handle adding documents to a collection
+     */
+    async handleAddToCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, AddToCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, urls } = validatedArgs;
+        // Normalize URLs
+        const normalizedUrls = urls.map((url) => normalizeUrl(url));
+        try {
+            const result = await this.store.addToCollection(name, normalizedUrls);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Added ${result.added.length} document(s) to collection "${name}"`,
+                            ...result,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to add to collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle removing documents from a collection
+     */
+    async handleRemoveFromCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, RemoveFromCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, urls } = validatedArgs;
+        // Normalize URLs
+        const normalizedUrls = urls.map((url) => normalizeUrl(url));
+        try {
+            const result = await this.store.removeFromCollection(name, normalizedUrls);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Removed ${result.removed.length} document(s) from collection "${name}"`,
+                            ...result,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to remove from collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle searching within a collection
+     */
+    async handleSearchCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, SearchCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, query, limit = 10 } = validatedArgs;
+        // Get URLs in the collection
+        const collectionUrls = await this.store.getCollectionUrls(name);
+        if (collectionUrls.length === 0) {
+            // Check if collection exists but is empty
+            const collection = await this.store.getCollection(name);
+            if (!collection) {
+                throw new McpError(ErrorCode.InvalidParams, `Collection "${name}" not found`);
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            results: [],
+                            message: `Collection "${name}" is empty. Add documentation sites to search.`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        // Search using the existing searchByText but filter by collection URLs
+        // We'll use a custom approach: search all then filter
+        // This is less efficient than a native filter, but works with existing infrastructure
+        const allResults = await this.store.searchByText(query, { limit: limit * 3 }); // Get more results to filter
+        // Filter to only include results from collection URLs
+        const collectionUrlSet = new Set(collectionUrls);
+        let filteredResults = allResults.filter((result) => collectionUrlSet.has(result.url));
+        // Apply limit
+        filteredResults = filteredResults.slice(0, limit);
+        // Apply prompt injection detection and filter/process results (same as handleSearchDocumentation)
+        let blockedCount = 0;
+        const safeResults = filteredResults
+            .map((result) => {
+            const injectionResult = detectPromptInjection(result.content);
+            if (injectionResult.maxSeverity === 'high') {
+                blockedCount++;
+                logger.debug(`[Security] Blocked search result from ${result.url} due to high-severity injection pattern: ${injectionResult.detections[0]?.description}`);
+                return null;
+            }
+            let safeContent = addInjectionWarnings(result.content, injectionResult);
+            safeContent = wrapExternalContent(safeContent, result.url);
+            return {
+                ...result,
+                content: safeContent,
+                security: {
+                    isExternalContent: true,
+                    injectionDetected: injectionResult.hasInjection,
+                    injectionSeverity: injectionResult.maxSeverity,
+                    detectionCount: injectionResult.detections.length,
+                },
+            };
+        })
+            .filter((result) => result !== null);
+        const response = {
+            results: safeResults,
+            collection: name,
+        };
+        if (blockedCount > 0) {
+            response.securityNotice = `${blockedCount} result(s) were blocked due to high-severity prompt injection patterns.`;
+        }
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(response, null, 2),
+                },
+            ],
+        };
+    }
+    async indexAndAdd(id, url, title, reIndex = false, signal, pathPrefix, authInfo, tags, version) {
         // Helper to check if operation was cancelled
         const checkCancelled = () => {
             if (signal?.aborted) {
@@ -1092,6 +1573,7 @@ IMPORTANT: Before calling this tool, ask the user if they want to restrict crawl
                     lastIndexed: new Date(),
                     requiresAuth: authInfo?.requiresAuth,
                     authDomain: authInfo?.authDomain,
+                    version,
                 },
                 chunks: chunks.map((chunk, i) => ({
                     ...chunk,