@cosmocoder/mcp-web-docs 1.3.0 → 1.4.1

package/README.md

@@ -38,6 +38,7 @@ AI assistants struggle with documentation:
 
 - **🌐 Universal Crawler** - Works with any documentation site, not just GitHub
 - **🔍 Hybrid Search** - Combines full-text search (FTS) with semantic vector search
+- **📂 Collections** - Group related docs into named collections for project-based organization
 - **🏷️ Tags & Categories** - Organize docs with tags and filter searches by project, team, or category
 - **📦 Version Support** - Index multiple versions of the same package (e.g., React 18 and 19)
 - **🔐 Authentication Support** - Crawl private/protected docs with interactive browser login (auto-detects your default browser)
@@ -390,6 +391,95 @@ Delete an indexed documentation site and all its data.
 
 Clear saved authentication session for a domain.
 
+### Collection Tools
+
+Collections let you group related documentation sites together for project-based organization. Unlike tags (which categorize individual docs), collections create named workspaces like "My React Project" containing React + Next.js + TypeScript docs.
+
+#### `create_collection`
+
+Create a new collection to group documentation sites.
+
+```typescript
+create_collection({
+  name: "My React Project",
+  description: "React, Next.js, and TypeScript docs for my project"  // Optional
+})
+```
+
+#### `add_to_collection`
+
+Add indexed documentation sites to a collection.
+
+```typescript
+add_to_collection({
+  name: "My React Project",
+  urls: [
+    "https://react.dev/",
+    "https://nextjs.org/docs/",
+    "https://www.typescriptlang.org/docs/"
+  ]
+})
+```
+
+#### `search_collection`
+
+Search within a specific collection. Uses the same hybrid search as `search_documentation` but limited to docs in the collection.
+
+```typescript
+search_collection({
+  name: "My React Project",
+  query: "server components data fetching",
+  limit: 10  // Optional
+})
+```
+
+#### `list_collections`
+
+List all collections with their document counts.
+
+#### `get_collection`
+
+Get details of a specific collection including all its documentation sites.
+
+```typescript
+get_collection({
+  name: "My React Project"
+})
+```
+
+#### `update_collection`
+
+Rename a collection or update its description.
+
+```typescript
+update_collection({
+  name: "My React Project",
+  newName: "Frontend Stack",           // Optional
+  description: "Updated description"   // Optional
+})
+```
+
+#### `remove_from_collection`
+
+Remove documentation sites from a collection. The sites remain indexed, just removed from the collection.
+
+```typescript
+remove_from_collection({
+  name: "My React Project",
+  urls: ["https://old-library.dev/docs/"]
+})
+```
+
+#### `delete_collection`
+
+Delete a collection. The documentation sites in the collection are **not** deleted, only the collection grouping.
+
+```typescript
+delete_collection({
+  name: "Old Project"
+})
+```
+
 ---
 
 ## 💡 Tips
@@ -471,6 +561,48 @@ search_documentation({
 
 You can also add tags to existing documentation with `set_tags`.
 
+### Using Collections for Project Organization
+
+Collections provide a higher-level grouping than tags — they let you organize documentation by project or context, making it easy to switch between different work contexts.
+
+**Create a collection for your project:**
+```typescript
+create_collection({
+  name: "E-commerce Backend",
+  description: "All docs for the backend rewrite project"
+})
+```
+
+**Add relevant documentation:**
+```typescript
+add_to_collection({
+  name: "E-commerce Backend",
+  urls: [
+    "https://fastapi.tiangolo.com/",
+    "https://docs.sqlalchemy.org/",
+    "https://redis.io/docs/"
+  ]
+})
+```
+
+**Search within your project context:**
+```typescript
+search_collection({
+  name: "E-commerce Backend",
+  query: "connection pooling best practices"
+})
+```
+
+**Collections vs Tags:**
+| Feature | Collections | Tags |
+|---------|-------------|------|
+| Purpose | Group docs as a project/workspace | Categorize individual docs |
+| Structure | Named container with multiple docs | Labels on individual docs |
+| Use case | "My React Project" with React + Next.js + TS | "This doc is about React" |
+| Searching | `search_collection` for focused results | `tags` filter in `search_documentation` |
+
+You can use both together — a document can have tags AND belong to multiple collections.
+
 ### Versioning Package Documentation
 
 When indexing documentation for versioned packages (React, Vue, Python libraries, etc.), you can specify the version to track which version you've indexed:

package/build/index.js

@@ -23,7 +23,7 @@ import { AuthManager } from './crawler/auth.js';
 import { fetchFavicon } from './util/favicon.js';
 import { generateDocId } from './util/docs.js';
 import { logger } from './util/logger.js';
-import { StorageStateSchema, safeJsonParse, validateToolArgs, sanitizeErrorMessage, detectPromptInjection, wrapExternalContent, addInjectionWarnings, SessionExpiredError, AddDocumentationArgsSchema, AuthenticateArgsSchema, ClearAuthArgsSchema, SearchDocumentationArgsSchema, ReindexDocumentationArgsSchema, DeleteDocumentationArgsSchema, SetTagsArgsSchema, } from './util/security.js';
+import { StorageStateSchema, safeJsonParse, validateToolArgs, sanitizeErrorMessage, detectPromptInjection, wrapExternalContent, addInjectionWarnings, SessionExpiredError, AddDocumentationArgsSchema, AuthenticateArgsSchema, ClearAuthArgsSchema, SearchDocumentationArgsSchema, ReindexDocumentationArgsSchema, DeleteDocumentationArgsSchema, SetTagsArgsSchema, CreateCollectionArgsSchema, DeleteCollectionArgsSchema, UpdateCollectionArgsSchema, GetCollectionArgsSchema, AddToCollectionArgsSchema, RemoveFromCollectionArgsSchema, SearchCollectionArgsSchema, } from './util/security.js';
 class WebDocsServer {
     server;
     config;
@@ -375,6 +375,143 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
                         properties: {},
                     },
                 },
+                // ============ Collection Tools ============
+                {
+                    name: 'create_collection',
+                    description: 'Create a new collection to group related documentation sites. Collections help organize docs by project or context (e.g., "My React Project" with React + Next.js + TypeScript docs).',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Unique name for the collection (e.g., "My React Project", "Backend APIs")',
+                            },
+                            description: {
+                                type: 'string',
+                                description: 'Optional description of what this collection contains',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'delete_collection',
+                    description: 'Delete a collection. The documentation sites in the collection are NOT deleted, only the collection grouping.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection to delete',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'update_collection',
+                    description: "Update a collection's name or description.",
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Current name of the collection',
+                            },
+                            newName: {
+                                type: 'string',
+                                description: 'Optional new name for the collection',
+                            },
+                            description: {
+                                type: 'string',
+                                description: 'Optional new description for the collection',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'list_collections',
+                    description: 'List all collections with their document counts. Use this to see available collections for context switching.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {},
+                    },
+                },
+                {
+                    name: 'get_collection',
+                    description: 'Get details of a specific collection including all its documentation sites.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'add_to_collection',
+                    description: 'Add one or more documentation sites to a collection. Sites must already be indexed.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                            urls: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'URLs of indexed documentation sites to add (max 50)',
+                            },
+                        },
+                        required: ['name', 'urls'],
+                    },
+                },
+                {
+                    name: 'remove_from_collection',
+                    description: 'Remove one or more documentation sites from a collection. The sites remain indexed, just removed from the collection.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection',
+                            },
+                            urls: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'URLs of documentation sites to remove from the collection',
+                            },
+                        },
+                        required: ['name', 'urls'],
+                    },
+                },
+                {
+                    name: 'search_collection',
+                    description: 'Search for documentation within a specific collection. This is useful for focused searches within a project context. Uses the same hybrid search (full-text + semantic) as search_documentation.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: {
+                                type: 'string',
+                                description: 'Name of the collection to search in',
+                            },
+                            query: {
+                                type: 'string',
+                                description: 'Search query - be specific and include unique terms',
+                            },
+                            limit: {
+                                type: 'number',
+                                description: 'Maximum number of results (default: 10)',
+                            },
+                        },
+                        required: ['name', 'query'],
+                    },
+                },
             ],
         }));
         // Handle tool calls
@@ -404,6 +541,23 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
                     return this.handleSetTags(request.params.arguments);
                 case 'list_tags':
                     return this.handleListTags();
+                // Collection handlers
+                case 'create_collection':
+                    return this.handleCreateCollection(request.params.arguments);
+                case 'delete_collection':
+                    return this.handleDeleteCollection(request.params.arguments);
+                case 'update_collection':
+                    return this.handleUpdateCollection(request.params.arguments);
+                case 'list_collections':
+                    return this.handleListCollections();
+                case 'get_collection':
+                    return this.handleGetCollection(request.params.arguments);
+                case 'add_to_collection':
+                    return this.handleAddToCollection(request.params.arguments);
+                case 'remove_from_collection':
+                    return this.handleRemoveFromCollection(request.params.arguments);
+                case 'search_collection':
+                    return this.handleSearchCollection(request.params.arguments);
                 default:
                     throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${request.params.name}`);
             }
@@ -956,6 +1110,318 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
             ],
         };
     }
+    // ============ Collection Handlers ============
+    /**
+     * Handle creating a new collection
+     */
+    async handleCreateCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, CreateCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, description } = validatedArgs;
+        try {
+            await this.store.createCollection(name, description);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection "${name}" created successfully`,
+                            collection: {
+                                name,
+                                description,
+                            },
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('already exists')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to create collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle deleting a collection
+     */
+    async handleDeleteCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, DeleteCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name } = validatedArgs;
+        try {
+            await this.store.deleteCollection(name);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection "${name}" deleted. Documentation sites remain indexed.`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to delete collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle updating a collection's metadata
+     */
+    async handleUpdateCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, UpdateCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, newName, description } = validatedArgs;
+        // Must provide at least one field to update
+        if (newName === undefined && description === undefined) {
+            throw new McpError(ErrorCode.InvalidParams, 'Must provide newName or description to update');
+        }
+        try {
+            await this.store.updateCollection(name, { newName, description });
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Collection updated successfully`,
+                            collection: {
+                                name: newName ?? name,
+                                description,
+                            },
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found') || safeMessage.includes('already exists')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to update collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle listing all collections
+     */
+    async handleListCollections() {
+        const collections = await this.store.listCollections();
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        collections,
+                        total: collections.length,
+                    }, null, 2),
+                },
+            ],
+        };
+    }
+    /**
+     * Handle getting a specific collection with its documents
+     */
+    async handleGetCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, GetCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name } = validatedArgs;
+        const collection = await this.store.getCollection(name);
+        if (!collection) {
+            throw new McpError(ErrorCode.InvalidParams, `Collection "${name}" not found`);
+        }
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(collection, null, 2),
+                },
+            ],
+        };
+    }
+    /**
+     * Handle adding documents to a collection
+     */
+    async handleAddToCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, AddToCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, urls } = validatedArgs;
+        // Normalize URLs
+        const normalizedUrls = urls.map((url) => normalizeUrl(url));
+        try {
+            const result = await this.store.addToCollection(name, normalizedUrls);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Added ${result.added.length} document(s) to collection "${name}"`,
+                            ...result,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to add to collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle removing documents from a collection
+     */
+    async handleRemoveFromCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, RemoveFromCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, urls } = validatedArgs;
+        // Normalize URLs
+        const normalizedUrls = urls.map((url) => normalizeUrl(url));
+        try {
+            const result = await this.store.removeFromCollection(name, normalizedUrls);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            status: 'success',
+                            message: `Removed ${result.removed.length} document(s) from collection "${name}"`,
+                            ...result,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const safeMessage = sanitizeErrorMessage(error);
+            if (safeMessage.includes('not found')) {
+                throw new McpError(ErrorCode.InvalidParams, safeMessage);
+            }
+            throw new McpError(ErrorCode.InternalError, `Failed to remove from collection: ${safeMessage}`);
+        }
+    }
+    /**
+     * Handle searching within a collection
+     */
+    async handleSearchCollection(args) {
+        let validatedArgs;
+        try {
+            validatedArgs = validateToolArgs(args, SearchCollectionArgsSchema);
+        }
+        catch (error) {
+            throw new McpError(ErrorCode.InvalidParams, sanitizeErrorMessage(error));
+        }
+        const { name, query, limit = 10 } = validatedArgs;
+        // Get URLs in the collection
+        const collectionUrls = await this.store.getCollectionUrls(name);
+        if (collectionUrls.length === 0) {
+            // Check if collection exists but is empty
+            const collection = await this.store.getCollection(name);
+            if (!collection) {
+                throw new McpError(ErrorCode.InvalidParams, `Collection "${name}" not found`);
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            results: [],
+                            message: `Collection "${name}" is empty. Add documentation sites to search.`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        // Search using the existing searchByText but filter by collection URLs
+        // We'll use a custom approach: search all then filter
+        // This is less efficient than a native filter, but works with existing infrastructure
+        const allResults = await this.store.searchByText(query, { limit: limit * 3 }); // Get more results to filter
+        // Filter to only include results from collection URLs
+        const collectionUrlSet = new Set(collectionUrls);
+        let filteredResults = allResults.filter((result) => collectionUrlSet.has(result.url));
+        // Apply limit
+        filteredResults = filteredResults.slice(0, limit);
+        // Apply prompt injection detection and filter/process results (same as handleSearchDocumentation)
+        let blockedCount = 0;
+        const safeResults = filteredResults
+            .map((result) => {
+            const injectionResult = detectPromptInjection(result.content);
+            if (injectionResult.maxSeverity === 'high') {
+                blockedCount++;
+                logger.debug(`[Security] Blocked search result from ${result.url} due to high-severity injection pattern: ${injectionResult.detections[0]?.description}`);
+                return null;
+            }
+            let safeContent = addInjectionWarnings(result.content, injectionResult);
+            safeContent = wrapExternalContent(safeContent, result.url);
+            return {
+                ...result,
+                content: safeContent,
+                security: {
+                    isExternalContent: true,
+                    injectionDetected: injectionResult.hasInjection,
+                    injectionSeverity: injectionResult.maxSeverity,
+                    detectionCount: injectionResult.detections.length,
+                },
+            };
+        })
+            .filter((result) => result !== null);
+        const response = {
+            results: safeResults,
+            collection: name,
+        };
+        if (blockedCount > 0) {
+            response.securityNotice = `${blockedCount} result(s) were blocked due to high-severity prompt injection patterns.`;
+        }
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(response, null, 2),
+                },
+            ],
+        };
+    }
     async indexAndAdd(id, url, title, reIndex = false, signal, pathPrefix, authInfo, tags, version) {
         // Helper to check if operation was cancelled
         const checkCancelled = () => {