@cosmocoder/mcp-web-docs 2.0.14 → 2.0.16

package/build/storage/storage.test.js

@@ -5,6 +5,7 @@ import { join } from 'path';
 import { mkdtemp, rm } from 'node:fs/promises';
 import { DocumentStore } from './storage.js';
 import { createMockEmbeddings } from '../__mocks__/embeddings.js';
+import { logger } from '../util/logger.js';
 import { setImmediate as nextTurn } from 'node:timers/promises';
 function deferred() {
     let resolve;
@@ -1005,14 +1006,13 @@ describe('DocumentStore', () => {
         it('should clear search cache for deleted document', async () => {
             const url = 'https://example.com/cache-test';
             await store.addDocument(createTestDocument(url, 'Cache Test'));
-            // Search to populate cache
-            await store.searchByText('cache test');
+            const cachedResults = await store.searchByText('cache test');
+            expect(cachedResults.some((result) => result.url === url)).toBe(true);
             // Delete document
             await store.deleteDocument(url);
             // Search again - should not find the document
-            const results = await store.searchByText('cache test');
-            const hasDeletedDoc = results.some((r) => r.url === url);
-            expect(hasDeletedDoc).toBe(false);
+            const refreshedResults = await store.searchByText('cache test');
+            expect(refreshedResults.some((result) => result.url === url)).toBe(false);
         });
     });
     describe('searchDocuments (vector search)', () => {
@@ -1067,6 +1067,25 @@ describe('DocumentStore', () => {
                 expect(result.metadata.type).toBe('api');
             });
         });
+        it('should prefilter pure vector search to exact URLs before applying the limit', async () => {
+            const queryVector = await mockEmbeddings.embed('exact scope query');
+            const globalUrl = 'https://example.com/global-best';
+            const scopedUrl = 'https://example.com/scoped-lower';
+            const globalDoc = createTestDocument(globalUrl, 'Global Best');
+            const scopedDoc = createTestDocument(scopedUrl, 'Scoped Lower');
+            globalDoc.chunks[0].vector = queryVector;
+            scopedDoc.chunks[0].vector = queryVector.map((value) => -value);
+            await store.addDocument(globalDoc);
+            await store.addDocument(scopedDoc);
+            const globalResults = await store.searchDocuments(queryVector, { limit: 1 });
+            const scopedResults = await store.searchDocuments(queryVector, { limit: 1, filterUrls: [scopedUrl] });
+            expect(globalResults[0].url).toBe(globalUrl);
+            expect(scopedResults.map((result) => result.url)).toEqual([scopedUrl]);
+        });
+        it('should return no vector results for an explicit empty URL scope', async () => {
+            const queryVector = await mockEmbeddings.embed('guide');
+            await expect(store.searchDocuments(queryVector, { filterUrls: [] })).resolves.toEqual([]);
+        });
         it('should return empty array for empty query vector without text query', async () => {
             const results = await store.searchDocuments([], { limit: 5 });
             expect(results).toEqual([]);
@@ -1084,10 +1103,12 @@ describe('DocumentStore', () => {
         });
         it('should use caching for repeated queries', async () => {
             const query = 'unique cache test query';
+            const embedSpy = vi.spyOn(mockEmbeddings, 'embed');
             const results1 = await store.searchByText(query);
             const results2 = await store.searchByText(query);
             // Results should be identical (from cache)
             expect(results1).toEqual(results2);
+            expect(embedSpy).toHaveBeenCalledTimes(1);
         });
         it('should respect limit option', async () => {
             const results = await store.searchByText('guide', { limit: 1 });
@@ -1101,9 +1122,60 @@ describe('DocumentStore', () => {
                 expect(result.url.startsWith('https://example.com/react')).toBe(true);
             });
         });
-        it('should handle quoted phrases', async () => {
-            const results = await store.searchByText('"React Hooks"');
-            expect(Array.isArray(results)).toBe(true);
+        it('should scope both hybrid search legs before ranking', async () => {
+            const query = 'adversarial hybrid scope';
+            const queryVector = await mockEmbeddings.embed(query);
+            const globalUrl = 'https://example.com/hybrid-global-best';
+            const scopedUrl = 'https://example.com/hybrid-scoped-lower';
+            const globalDoc = createDocumentWithContent(globalUrl, 'Global Hybrid', query);
+            const scopedDoc = createDocumentWithContent(scopedUrl, 'Scoped Hybrid', query);
+            globalDoc.chunks[0].vector = queryVector;
+            scopedDoc.chunks[0].vector = queryVector.map((value) => -value);
+            await store.addDocument(globalDoc);
+            await store.addDocument(scopedDoc);
+            expect(store.ftsIndexCreated).toBe(true);
+            const results = await store.searchByText(query, { limit: 2, filterUrls: [scopedUrl] });
+            expect(results.map((result) => result.url)).toEqual([scopedUrl]);
+            expect(results[0].score).toBeGreaterThan(0);
+        });
+        it('should preserve exact URL scope in the pure-vector text fallback', async () => {
+            const query = 'fallback scope query';
+            const queryVector = await mockEmbeddings.embed(query);
+            const globalUrl = 'https://example.com/fallback-global-best';
+            const scopedUrl = 'https://example.com/fallback-scoped-lower';
+            const globalDoc = createTestDocument(globalUrl, 'Fallback Global Best');
+            const scopedDoc = createTestDocument(scopedUrl, 'Fallback Scoped Lower');
+            globalDoc.chunks[0].vector = queryVector;
+            scopedDoc.chunks[0].vector = queryVector.map((value) => -value);
+            await store.addDocument(globalDoc);
+            await store.addDocument(scopedDoc);
+            store.ftsIndexCreated = false;
+            const globalResults = await store.searchByText(query, { limit: 1 });
+            const scopedResults = await store.searchByText(query, { limit: 1, filterUrls: [scopedUrl] });
+            expect(globalResults[0].url).toBe(globalUrl);
+            expect(scopedResults.map((result) => result.url)).toEqual([scopedUrl]);
+        });
+        it('should return no text results for an explicit empty URL scope', async () => {
+            await expect(store.searchByText('guide', { filterUrls: [] })).resolves.toEqual([]);
+        });
+        it('should scope both phrase search legs before ranking', async () => {
+            const phrase = 'adversarial phrase scope';
+            const query = `"${phrase}"`;
+            const queryVector = await mockEmbeddings.embed(query);
+            const globalUrl = 'https://example.com/phrase-global-best';
+            const scopedUrl = 'https://example.com/phrase-scoped-lower';
+            const globalDoc = createDocumentWithContent(globalUrl, 'Global Phrase', phrase);
+            const scopedDoc = createDocumentWithContent(scopedUrl, 'Scoped Phrase', phrase);
+            globalDoc.chunks[0].vector = queryVector;
+            scopedDoc.chunks[0].vector = queryVector.map((value) => -value);
+            await store.addDocument(globalDoc);
+            await store.addDocument(scopedDoc);
+            const internals = replacementInternals();
+            internals.ftsIndexCreated = false;
+            await internals.createFTSIndex();
+            const results = await store.searchByText(query, { limit: 2, filterUrls: [scopedUrl] });
+            expect(logger.debug).toHaveBeenCalledWith(expect.stringMatching(/Phrase-based FTS returned [1-9]/));
+            expect(results.map((result) => result.url)).toEqual([scopedUrl]);
         });
         it('should handle empty query gracefully', async () => {
             // Generate embedding for empty query
@@ -1336,6 +1408,13 @@ describe('DocumentStore', () => {
             // Should not throw and should return empty (no match)
             expect(Array.isArray(results)).toBe(true);
         });
+        it('should safely filter by exact URLs with special characters', async () => {
+            const url = "https://example.com/team's-guide";
+            await store.addDocument(createTestDocument(url, 'Special URL Guide'));
+            const results = await store.searchByText('guide', { filterUrls: [url] });
+            expect(results.length).toBeGreaterThan(0);
+            expect(results.every((result) => result.url === url)).toBe(true);
+        });
     });
     describe('document tags', () => {
         it('should set and retrieve tags for a document', async () => {
@@ -1499,6 +1578,15 @@ describe('DocumentStore', () => {
                 expect(result.url.startsWith('https://example.com/react')).toBe(true);
             });
         });
+        it('should intersect exact URL, prefix URL, and tag filters', async () => {
+            const results = await store.searchByText('guide', {
+                filterByTags: ['frontend'],
+                filterUrl: 'https://example.com/react',
+                filterUrls: ['https://example.com/react-hooks', 'https://example.com/express-api'],
+            });
+            expect(results.length).toBeGreaterThan(0);
+            expect(results.every((result) => result.url === 'https://example.com/react-hooks')).toBe(true);
+        });
     });
     describe('document version', () => {
         it('should store and retrieve version for a document', async () => {
@@ -1605,15 +1693,23 @@ describe('DocumentStore', () => {
             expect(typeof result.compacted).toBe('boolean');
             expect(typeof result.cleanedUp).toBe('boolean');
         });
-        it('should clear search cache after optimization', async () => {
-            // Add document and search to populate cache
-            await store.addDocument(createTestDocument('https://example.com/cache', 'Cache Test'));
-            await store.searchByText('cache test');
-            // Run optimization (which should clear cache)
+        it('should not cache a search that started before optimization', async () => {
+            const query = 'in-flight optimization cache';
+            const embed = mockEmbeddings.embed.bind(mockEmbeddings);
+            const searchStarted = Promise.withResolvers();
+            const releaseSearch = Promise.withResolvers();
+            const embedSpy = vi.spyOn(mockEmbeddings, 'embed').mockImplementationOnce(async (text) => {
+                searchStarted.resolve();
+                await releaseSearch.promise;
+                return embed(text);
+            });
+            const inFlightSearch = store.searchByText(query);
+            await searchStarted.promise;
             await store.optimize();
-            // Search should still work (but cache was cleared)
-            const results = await store.searchByText('cache test');
-            expect(Array.isArray(results)).toBe(true);
+            releaseSearch.resolve();
+            await inFlightSearch;
+            await store.searchByText(query);
+            expect(embedSpy).toHaveBeenCalledTimes(2);
         });
         it('should preserve data after optimization', async () => {
             // Add documents