@cosmocoder/mcp-web-docs 2.0.20 → 2.0.22

package/build/indexing/workflow.js

@@ -0,0 +1,223 @@
+import { setTimeout as delay } from 'node:timers/promises';
+import { detectPromptInjection, safeJsonParse, sanitizeErrorMessage, SessionExpiredError, StorageStateSchema, } from '../util/security.js';
+import { logger } from '../util/logger.js';
+export class IndexingWorkflow {
+    dependencies;
+    constructor(dependencies) {
+        this.dependencies = dependencies;
+    }
+    async run(request, signal) {
+        const { operationId, url, title, pathPrefix, authInfo, tags, version } = request;
+        const reIndex = request.reIndex ?? false;
+        const { store, processor, statusTracker, authManager } = this.dependencies;
+        const checkCancelled = () => {
+            if (signal.aborted) {
+                logger.info(`[IndexingWorkflow] Operation cancelled for ${url}`);
+                if (statusTracker.getStatus(operationId)?.status !== 'cancelled') {
+                    statusTracker.cancelIndexing(operationId);
+                }
+                const error = new Error('Operation cancelled');
+                error.name = 'AbortError';
+                throw error;
+            }
+        };
+        try {
+            logger.info(`[IndexingWorkflow] Starting indexing for ${url} (reIndex: ${reIndex})`);
+            checkCancelled();
+            logger.debug(`[IndexingWorkflow] Checking if document exists: ${url}`);
+            const existingDoc = await store.getDocument(url);
+            checkCancelled();
+            if (existingDoc) {
+                logger.debug(`[IndexingWorkflow] Document exists: ${url}`);
+                if (!reIndex) {
+                    logger.info(`[IndexingWorkflow] Document ${url} already indexed and reIndex=false`);
+                    statusTracker.completeIndexing(operationId);
+                    return;
+                }
+                logger.info(`[IndexingWorkflow] Will reindex existing document: ${url}`);
+            }
+            else {
+                logger.debug(`[IndexingWorkflow] Document does not exist: ${url}`);
+            }
+            checkCancelled();
+            statusTracker.updateProgress(operationId, 0, 'Finding subpages');
+            logger.info(`[IndexingWorkflow] Starting crawl${pathPrefix ? ` with pathPrefix=${pathPrefix}` : ''}`);
+            const crawler = this.dependencies.createCrawler();
+            if (pathPrefix) {
+                crawler.setPathPrefix(pathPrefix);
+            }
+            const savedSession = await authManager.loadSession(url);
+            checkCancelled();
+            if (savedSession) {
+                try {
+                    const validatedState = safeJsonParse(savedSession, StorageStateSchema);
+                    crawler.setStorageState(validatedState);
+                    logger.info(`[IndexingWorkflow] Using validated authentication session for ${url}`);
+                }
+                catch (error) {
+                    logger.warn(`[IndexingWorkflow] Failed to parse or validate saved session:`, error);
+                }
+            }
+            const pages = [];
+            let processedPages = 0;
+            let estimatedProgress = 0;
+            logger.info(`[IndexingWorkflow] Starting page crawl for ${url}`);
+            const abortCrawler = () => crawler.abort();
+            signal.addEventListener('abort', abortCrawler, { once: true });
+            if (signal.aborted) {
+                abortCrawler();
+            }
+            try {
+                for await (const page of crawler.crawl(url)) {
+                    checkCancelled();
+                    logger.debug(`[IndexingWorkflow] Found page ${processedPages + 1}: ${page.path}`);
+                    processedPages++;
+                    estimatedProgress += 1 / 2 ** processedPages;
+                    statusTracker.updateProgress(operationId, 0.15 * estimatedProgress + Math.min(0.35, (0.35 * processedPages) / 500), `Finding subpages (${page.path})`);
+                    statusTracker.updateStats(operationId, { pagesFound: processedPages });
+                    pages.push(page);
+                    await delay(50, undefined, { signal });
+                    checkCancelled();
+                }
+            }
+            catch (error) {
+                if (signal.aborted) {
+                    checkCancelled();
+                }
+                throw error;
+            }
+            finally {
+                signal.removeEventListener('abort', abortCrawler);
+            }
+            checkCancelled();
+            if (pages.length === 0) {
+                logger.warn('[IndexingWorkflow] No pages found during crawl');
+                throw new Error('No pages found to index');
+            }
+            logger.info(`[IndexingWorkflow] Found ${pages.length} pages to process`);
+            logger.info('[IndexingWorkflow] Starting content processing and embedding generation');
+            statusTracker.updateStats(operationId, { pagesFound: pages.length });
+            checkCancelled();
+            const chunks = [];
+            for (let i = 0; i < pages.length; i++) {
+                checkCancelled();
+                const page = pages[i];
+                logger.debug(`[IndexingWorkflow] Processing page ${i + 1}/${pages.length}: ${page.path}`);
+                statusTracker.updateProgress(operationId, 0.5 + 0.3 * (i / pages.length), `Creating embeddings (${i + 1}/${pages.length})`);
+                try {
+                    const processed = await processor.process(page);
+                    logger.debug(`[IndexingWorkflow] Created ${processed.chunks.length} chunks for ${page.path}`);
+                    chunks.push(...processed.chunks);
+                    statusTracker.updateStats(operationId, {
+                        pagesProcessed: i + 1,
+                        chunksCreated: chunks.length,
+                    });
+                }
+                catch (error) {
+                    if (signal.aborted) {
+                        checkCancelled();
+                    }
+                    logger.error(`[IndexingWorkflow] Error processing page ${page.path}:`, error);
+                    throw new Error(sanitizeErrorMessage(`Failed to process ${page.path}: ${sanitizeErrorMessage(error)}`));
+                }
+                checkCancelled();
+                await delay(20, undefined, { signal });
+                checkCancelled();
+            }
+            logger.info(`[IndexingWorkflow] Total chunks created: ${chunks.length}`);
+            let injectionWarnings = 0;
+            for (const chunk of chunks) {
+                const injectionResult = detectPromptInjection(chunk.content);
+                if (injectionResult.hasInjection) {
+                    injectionWarnings++;
+                    if (injectionResult.maxSeverity === 'high') {
+                        logger.debug(`[Security] Prompt injection pattern detected in ${chunk.path || 'unknown'}: ${injectionResult.detections[0]?.description}`);
+                    }
+                }
+            }
+            if (injectionWarnings > 0) {
+                logger.debug(`[Security] Detected ${injectionWarnings} chunks with potential prompt injection patterns in ${url}. Content will be marked when returned in search results.`);
+            }
+            checkCancelled();
+            if (chunks.length === 0) {
+                logger.warn(`[IndexingWorkflow] No content was extracted from ${url}`);
+                logger.warn(`[IndexingWorkflow] Pages found: ${pages.length}`);
+                logger.warn(`[IndexingWorkflow] Chunks created: ${chunks.length}`);
+                statusTracker.failIndexing(operationId, 'No content was extracted from the pages');
+                return;
+            }
+            checkCancelled();
+            const favicon = await this.dependencies.fetchFavicon(new URL(url));
+            checkCancelled();
+            statusTracker.updateProgress(operationId, 0.9, `Storing ${chunks.length} chunks`);
+            await this.addDocumentWithRetry({
+                metadata: {
+                    url,
+                    title,
+                    favicon: favicon ?? undefined,
+                    lastIndexed: new Date(),
+                    requiresAuth: authInfo?.requiresAuth,
+                    authDomain: authInfo?.authDomain,
+                    version,
+                    pathPrefix,
+                },
+                chunks,
+            }, signal, tags);
+            checkCancelled();
+            if (tags && tags.length > 0) {
+                logger.info(`[IndexingWorkflow] Tags set for ${url}:`, tags);
+            }
+            else {
+                logger.debug(`[IndexingWorkflow] Tags cleared for ${url}`);
+            }
+            logger.info(`[IndexingWorkflow] Successfully indexed ${url}`);
+            logger.info(`[IndexingWorkflow] Pages processed: ${pages.length}`);
+            logger.info(`[IndexingWorkflow] Chunks stored: ${chunks.length}`);
+            statusTracker.updateStats(operationId, { chunksCreated: chunks.length });
+            statusTracker.completeIndexing(operationId);
+            store.optimize().catch((error) => {
+                logger.warn('[IndexingWorkflow] Background optimization failed:', error);
+            });
+        }
+        catch (error) {
+            if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) {
+                if (signal.aborted && statusTracker.getStatus(operationId)?.status !== 'cancelled') {
+                    statusTracker.cancelIndexing(operationId);
+                }
+                logger.info(`[IndexingWorkflow] Indexing cancelled for ${url}`);
+                return;
+            }
+            if (error instanceof SessionExpiredError) {
+                logger.warn(`[IndexingWorkflow] Session expired during crawl of ${url}: ${error.message}`);
+                logger.warn(`[IndexingWorkflow] Expected URL: ${error.expectedUrl}, Detected URL: ${error.detectedUrl}`);
+                await authManager.clearSession(url);
+                checkCancelled();
+                logger.info(`[IndexingWorkflow] Cleared expired session for ${url}`);
+                statusTracker.failIndexing(operationId, `Authentication session has expired. The crawler was redirected to a login page. Please use the 'authenticate' tool to log in again before re-indexing.`);
+                return;
+            }
+            logger.error('[IndexingWorkflow] Error during indexing:', error);
+            logger.error('[IndexingWorkflow] Error details:', error instanceof Error ? error.stack : error);
+            statusTracker.failIndexing(operationId, error instanceof Error ? error.message : 'Unknown error');
+        }
+    }
+    async addDocumentWithRetry(doc, signal, tags, maxRetries = 3) {
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            signal.throwIfAborted();
+            try {
+                await this.dependencies.store.addDocument(doc, { signal, tags: tags ?? [] });
+                return;
+            }
+            catch (error) {
+                const isRetryable = error instanceof Error && (error.message.includes('Commit conflict') || error.message.startsWith('Replacement lease lost for '));
+                if (isRetryable && attempt < maxRetries) {
+                    logger.warn(`[IndexingWorkflow] Storage conflict, retrying (${attempt}/${maxRetries})...`);
+                    await delay(1000 * attempt, undefined, { signal });
+                    continue;
+                }
+                throw error;
+            }
+        }
+    }
+}
+//# sourceMappingURL=workflow.js.map

package/build/indexing/workflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow.js","sourceRoot":"","sources":["../../src/indexing/workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAO3D,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,GAEnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAgC3C,MAAM,OAAO,gBAAgB;IACE;IAA7B,YAA6B,YAA0C;QAA1C,iBAAY,GAAZ,YAAY,CAA8B;IAAG,CAAC;IAE3E,KAAK,CAAC,GAAG,CAAC,OAAwB,EAAE,MAAmB;QACrD,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QACjF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC;QAE3E,MAAM,cAAc,GAAG,GAAG,EAAE;YAC1B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,8CAA8C,GAAG,EAAE,CAAC,CAAC;gBACjE,IAAI,aAAa,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;oBACjE,aAAa,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC5C,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAC/C,KAAK,CAAC,IAAI,GAAG,YAAY,CAAC;gBAC1B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,4CAA4C,GAAG,cAAc,OAAO,GAAG,CAAC,CAAC;YACrF,cAAc,EAAE,CAAC;YAEjB,MAAM,CAAC,KAAK,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACjD,cAAc,EAAE,CAAC;YAEjB,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,oCAAoC,CAAC,CAAC;oBACpF,aAAa,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;oBAC5C,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;YAC3E,CAAC;iBACI,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,cAAc,EAAE,CAAC;YAEjB,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,EAAE,kBAAkB,CAAC,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,oCAAoC,UAAU,CAAC,CAAC,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtG,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;YAElD,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACpC,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACxD,cAAc,EAAE,CAAC;YACjB,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,CAAC;oBACH,MAAM,cAAc,GAA0B,aAAa,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;oBAC9F,OAAO,CAAC,eAAe,CAAC,cAA8B,CAAC,CAAC;oBACxD,MAAM,CAAC,IAAI,CAAC,iEAAiE,GAAG,EAAE,CAAC,CAAC;gBACtF,CAAC;gBACD,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CAAC,+DAA+D,EAAE,KAAK,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;YAED,MAAM,KAAK,GAAG,EAAE,CAAC;YACjB,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,MAAM,CAAC,IAAI,CAAC,8CAA8C,GAAG,EAAE,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,GAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,YAAY,EAAE,CAAC;YACjB,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5C,cAAc,EAAE,CAAC;oBAEjB,MAAM,CAAC,KAAK,CAAC,iCAAiC,cAAc,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBAClF,cAAc,EAAE,CAAC;oBACjB,iBAAiB,IAAI,CAAC,GAAG,CAAC,IAAI,cAAc,CAAC;oBAE7C,aAAa,CAAC,cAAc,CAC1B,WAAW,EACX,IAAI,GAAG,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,EACxE,qBAAqB,IAAI,CAAC,IAAI,GAAG,CAClC,CAAC;oBACF,aAAa,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,CAAC;oBACvE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAEjB,MAAM,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;oBACvC,cAAc,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,cAAc,EAAE,CAAC;gBACnB,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;oBACO,CAAC;gBACP,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YACpD,CAAC;YACD,cAAc,EAAE,CAAC;YAEjB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,4BAA4B,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;YACvF,aAAa,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,cAAc,EAAE,CAAC;YAEjB,MAAM,MAAM,GAAoB,EAAE,CAAC;YAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,cAAc,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1F,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,wBAAwB,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;gBAE5H,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChD,MAAM,CAAC,KAAK,CAAC,8BAA8B,SAAS,CAAC,MAAM,CAAC,MAAM,eAAe,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC9F,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;oBACjC,aAAa,CAAC,WAAW,CAAC,WAAW,EAAE;wBACrC,cAAc,EAAE,CAAC,GAAG,CAAC;wBACrB,aAAa,EAAE,MAAM,CAAC,MAAM;qBAC7B,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,KAAK,EAAE,CAAC;oBACb,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,cAAc,EAAE,CAAC;oBACnB,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,4CAA4C,IAAI,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;oBAC9E,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,qBAAqB,IAAI,CAAC,IAAI,KAAK,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1G,CAAC;gBAED,cAAc,EAAE,CAAC;gBACjB,MAAM,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;gBACvC,cAAc,EAAE,CAAC;YACnB,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,4CAA4C,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAEzE,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAC1B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,eAAe,GAAG,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7D,IAAI,eAAe,CAAC,YAAY,EAAE,CAAC;oBACjC,iBAAiB,EAAE,CAAC;oBACpB,IAAI,eAAe,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;wBAC3C,MAAM,CAAC,KAAK,CACV,mDAAmD,KAAK,CAAC,IAAI,IAAI,SAAS,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAC5H,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,KAAK,CACV,uBAAuB,iBAAiB,uDAAuD,GAAG,2DAA2D,CAC9J,CAAC;YACJ,CAAC;YAED,cAAc,EAAE,CAAC;YAEjB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC/D,MAAM,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACnE,aAAa,CAAC,YAAY,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;gBACnF,OAAO;YACT,CAAC;YAED,cAAc,EAAE,CAAC;YAEjB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,cAAc,EAAE,CAAC;YACjB,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,WAAW,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,oBAAoB,CAC7B;gBACE,QAAQ,EAAE;oBACR,GAAG;oBACH,KAAK;oBACL,OAAO,EAAE,OAAO,IAAI,SAAS;oBAC7B,WAAW,EAAE,IAAI,IAAI,EAAE;oBACvB,YAAY,EAAE,QAAQ,EAAE,YAAY;oBACpC,UAAU,EAAE,QAAQ,EAAE,UAAU;oBAChC,OAAO;oBACP,UAAU;iBACX;gBACD,MAAM;aACP,EACD,MAAM,EACN,IAAI,CACL,CAAC;YACF,cAAc,EAAE,CAAC;YAEjB,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,mCAAmC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;YAC/D,CAAC;iBACI,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,uCAAuC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAClE,aAAa,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YACzE,aAAa,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAE5C,KAAK,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/B,MAAM,CAAC,IAAI,CAAC,oDAAoD,EAAE,KAAK,CAAC,CAAC;YAC3E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;gBAC9E,IAAI,MAAM,CAAC,OAAO,IAAI,aAAa,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;oBACnF,aAAa,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC5C,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,6CAA6C,GAAG,EAAE,CAAC,CAAC;gBAChE,OAAO;YACT,CAAC;YAED,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC,sDAAsD,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3F,MAAM,CAAC,IAAI,CAAC,oCAAoC,KAAK,CAAC,WAAW,mBAAmB,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;gBACzG,MAAM,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;gBACpC,cAAc,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,kDAAkD,GAAG,EAAE,CAAC,CAAC;gBACrE,aAAa,CAAC,YAAY,CACxB,WAAW,EACX,wJAAwJ,CACzJ,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAChG,aAAa,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,GAAgD,EAChD,MAAmB,EACnB,IAAe,EACf,UAAU,GAAG,CAAC;QAEd,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC7E,OAAO;YACT,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,WAAW,GACf,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC,CAAC;gBACnI,IAAI,WAAW,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;oBACxC,MAAM,CAAC,IAAI,CAAC,kDAAkD,OAAO,IAAI,UAAU,MAAM,CAAC,CAAC;oBAC3F,MAAM,KAAK,CAAC,IAAI,GAAG,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;oBACnD,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/build/indexing/workflow.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/indexing/workflow.test.js

@@ -0,0 +1,218 @@
+import { SessionExpiredError } from '../util/security.js';
+import { DocsCrawler } from '../crawler/docs-crawler.js';
+import { IndexingWorkflow } from './workflow.js';
+const request = {
+    operationId: 'operation-1',
+    url: 'https://docs.example.com',
+    title: 'Example Docs',
+};
+const page = {
+    url: request.url,
+    path: '/',
+    content: '<h1>Example</h1>',
+    contentFormat: 'text',
+    title: request.title,
+};
+const chunk = {
+    content: 'Example content',
+    url: request.url,
+    title: request.title,
+    path: '/',
+    startLine: 0,
+    endLine: 0,
+    vector: [0.1, 0.2],
+    metadata: { type: 'overview' },
+};
+function createHarness(options = {}) {
+    const addDocument = vi.fn().mockResolvedValue(undefined);
+    const store = {
+        addDocument,
+        getDocument: vi.fn().mockResolvedValue(options.existingDocument ?? null),
+        optimize: vi.fn().mockResolvedValue({ compacted: false, cleanedUp: false }),
+    };
+    const statusTracker = {
+        cancelIndexing: vi.fn(),
+        completeIndexing: vi.fn(),
+        failIndexing: vi.fn(),
+        getStatus: vi.fn(),
+        updateProgress: vi.fn(),
+        updateStats: vi.fn(),
+    };
+    const authManager = {
+        clearSession: vi.fn().mockResolvedValue(undefined),
+        loadSession: vi.fn().mockResolvedValue(options.savedSession ?? null),
+    };
+    const crawler = {
+        abort: vi.fn(),
+        crawl: options.crawl ??
+            async function* () {
+                yield page;
+            },
+        setPathPrefix: vi.fn(),
+        setStorageState: vi.fn(),
+    };
+    const process = vi.fn(options.process ??
+        (async () => ({
+            metadata: { url: page.url, title: page.title, lastIndexed: new Date() },
+            chunks: [chunk],
+        })));
+    const processor = { process };
+    const createCrawler = vi.fn(options.createCrawler ?? (() => crawler));
+    const fetchFavicon = vi.fn().mockResolvedValue('data:image/x-icon;base64,AA==');
+    const workflow = new IndexingWorkflow({ store, processor, statusTracker, authManager, createCrawler, fetchFavicon });
+    return { workflow, store, statusTracker, authManager, crawler, createCrawler, fetchFavicon, addDocument, process };
+}
+const runWorkflow = (workflow, indexingRequest, signal = new AbortController().signal) => workflow.run(indexingRequest, signal);
+describe('IndexingWorkflow', () => {
+    it('crawls, processes, stores, tags, completes, and optimizes a document', async () => {
+        const harness = createHarness();
+        await runWorkflow(harness.workflow, { ...request, pathPrefix: '/guide', tags: ['guide'], version: '2' });
+        expect(harness.createCrawler).toHaveBeenCalledWith();
+        expect(harness.crawler.setPathPrefix).toHaveBeenCalledWith('/guide');
+        expect(harness.addDocument).toHaveBeenCalledWith({
+            metadata: expect.objectContaining({
+                url: request.url,
+                title: request.title,
+                favicon: 'data:image/x-icon;base64,AA==',
+                version: '2',
+                pathPrefix: '/guide',
+            }),
+            chunks: [chunk],
+        }, { signal: expect.any(AbortSignal), tags: ['guide'] });
+        expect(harness.statusTracker.completeIndexing).toHaveBeenCalledWith(request.operationId);
+        expect(harness.store.optimize).toHaveBeenCalledOnce();
+        const progressStages = harness.statusTracker.updateProgress.mock.calls.map((call) => call[2]);
+        expect(progressStages).toEqual(expect.arrayContaining(['Finding subpages', 'Finding subpages (/)', 'Creating embeddings (1/1)', 'Storing 1 chunks']));
+        expect(progressStages.indexOf('Finding subpages')).toBeLessThan(progressStages.indexOf('Creating embeddings (1/1)'));
+        expect(progressStages.indexOf('Creating embeddings (1/1)')).toBeLessThan(progressStages.indexOf('Storing 1 chunks'));
+        expect(harness.statusTracker.updateStats).toHaveBeenCalledWith(request.operationId, { pagesFound: 1 });
+        expect(harness.statusTracker.updateStats).toHaveBeenCalledWith(request.operationId, { pagesProcessed: 1, chunksCreated: 1 });
+        expect(harness.statusTracker.updateStats).toHaveBeenCalledWith(request.operationId, { chunksCreated: 1 });
+    });
+    it('reindexes only after processing and finishes metadata after the replacement write', async () => {
+        const harness = createHarness({
+            existingDocument: { url: request.url, title: request.title, lastIndexed: new Date() },
+        });
+        await runWorkflow(harness.workflow, { ...request, reIndex: true, tags: ['existing'] });
+        const processOrder = harness.process.mock.invocationCallOrder[0];
+        const addOrder = harness.addDocument.mock.invocationCallOrder[0];
+        expect(processOrder).toBeLessThan(addOrder);
+        expect(addOrder).toBeLessThan(harness.statusTracker.completeIndexing.mock.invocationCallOrder[0]);
+        const progressStages = harness.statusTracker.updateProgress.mock.calls.map((call) => call[2]);
+        expect(progressStages.indexOf('Creating embeddings (1/1)')).toBeLessThan(progressStages.indexOf('Storing 1 chunks'));
+        expect(harness.statusTracker.updateStats).toHaveBeenCalledWith(request.operationId, {
+            pagesProcessed: 1,
+            chunksCreated: 1,
+        });
+        const failedWrite = createHarness({
+            existingDocument: { url: request.url, title: request.title, lastIndexed: new Date() },
+        });
+        failedWrite.addDocument.mockRejectedValue(new Error('write failed'));
+        await runWorkflow(failedWrite.workflow, { ...request, reIndex: true, tags: ['existing'] });
+        expect(failedWrite.addDocument).toHaveBeenCalledOnce();
+        expect(failedWrite.statusTracker.completeIndexing).not.toHaveBeenCalled();
+        expect(failedWrite.statusTracker.failIndexing).toHaveBeenCalledWith(request.operationId, 'write failed');
+    });
+    it('completes an existing add without crawling or writing', async () => {
+        const harness = createHarness({
+            existingDocument: { url: request.url, title: request.title, lastIndexed: new Date() },
+        });
+        await runWorkflow(harness.workflow, request);
+        expect(harness.statusTracker.completeIndexing).toHaveBeenCalledWith(request.operationId);
+        expect(harness.createCrawler).not.toHaveBeenCalled();
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it('fails without writing when processing produces no chunks', async () => {
+        const harness = createHarness({
+            existingDocument: { url: request.url, title: request.title, lastIndexed: new Date() },
+            process: vi.fn().mockResolvedValue({
+                metadata: { url: page.url, title: page.title, lastIndexed: new Date() },
+                chunks: [],
+            }),
+        });
+        await runWorkflow(harness.workflow, { ...request, reIndex: true });
+        expect(harness.statusTracker.failIndexing).toHaveBeenCalledWith(request.operationId, 'No content was extracted from the pages');
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it.each([
+        ['valid', JSON.stringify({ cookies: [] }), { cookies: [] }],
+        ['malformed', '{not json', undefined],
+    ])('continues crawling with a %s saved session', async (_label, savedSession, expectedState) => {
+        const harness = createHarness({ savedSession });
+        await runWorkflow(harness.workflow, request);
+        if (expectedState) {
+            expect(harness.crawler.setStorageState).toHaveBeenCalledWith(expectedState);
+        }
+        else {
+            expect(harness.crawler.setStorageState).not.toHaveBeenCalled();
+        }
+        expect(harness.addDocument).toHaveBeenCalledOnce();
+        expect(harness.statusTracker.completeIndexing).toHaveBeenCalledWith(request.operationId);
+    });
+    it('aborts an in-progress crawl without writing', async () => {
+        const controller = new AbortController();
+        const harness = createHarness({
+            crawl: async function* () {
+                yield page;
+                controller.abort();
+                yield { ...page, path: '/second' };
+            },
+        });
+        await runWorkflow(harness.workflow, request, controller.signal);
+        expect(harness.crawler.abort).toHaveBeenCalledOnce();
+        expect(harness.statusTracker.cancelIndexing).toHaveBeenCalledWith(request.operationId);
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it('clears an expired session and reports the friendly authentication failure', async () => {
+        const harness = createHarness({
+            crawl: async function* () {
+                yield* [];
+                throw new SessionExpiredError('redirected', request.url, 'https://docs.example.com/login', {
+                    isLoginPage: true,
+                    confidence: 1,
+                    reasons: ['login URL'],
+                });
+            },
+        });
+        await runWorkflow(harness.workflow, request);
+        expect(harness.authManager.clearSession).toHaveBeenCalledWith(request.url);
+        expect(harness.statusTracker.failIndexing).toHaveBeenCalledWith(request.operationId, expect.stringContaining("Please use the 'authenticate' tool"));
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it('consumes generic failures and marks the operation failed', async () => {
+        const harness = createHarness({
+            crawl: async function* () {
+                yield* [];
+                throw new Error('crawl failed');
+            },
+        });
+        await expect(runWorkflow(harness.workflow, request)).resolves.toBeUndefined();
+        expect(harness.statusTracker.failIndexing).toHaveBeenCalledWith(request.operationId, 'crawl failed');
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it('surfaces unsupported GitHub blob URLs through indexing status', async () => {
+        const harness = createHarness({ createCrawler: () => new DocsCrawler() });
+        const blobUrl = 'https://github.com/owner/repo/blob/main/README.md';
+        await runWorkflow(harness.workflow, { ...request, url: blobUrl });
+        expect(harness.statusTracker.failIndexing).toHaveBeenCalledWith(request.operationId, `Unsupported GitHub URL: ${blobUrl}. Use a repository root or /tree/<branch>[/path] URL.`);
+        expect(fetchMock).not.toHaveBeenCalled();
+        expect(harness.fetchFavicon).not.toHaveBeenCalled();
+        expect(harness.addDocument).not.toHaveBeenCalled();
+    });
+    it('retries transient commit conflicts before succeeding', async () => {
+        vi.useFakeTimers();
+        const harness = createHarness();
+        harness.addDocument.mockRejectedValueOnce(new Error('Commit conflict')).mockResolvedValueOnce(undefined);
+        const run = runWorkflow(harness.workflow, request);
+        await vi.runAllTimersAsync();
+        await run;
+        expect(harness.addDocument).toHaveBeenCalledTimes(2);
+        expect(harness.addDocument).toHaveBeenLastCalledWith(expect.any(Object), {
+            signal: expect.any(AbortSignal),
+            tags: [],
+        });
+        expect(harness.statusTracker.completeIndexing).toHaveBeenCalledWith(request.operationId);
+        vi.useRealTimers();
+    });
+});
+//# sourceMappingURL=workflow.test.js.map

package/build/indexing/workflow.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow.test.js","sourceRoot":"","sources":["../../src/indexing/workflow.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAwB,MAAM,eAAe,CAAC;AAEvE,MAAM,OAAO,GAAoB;IAC/B,WAAW,EAAE,aAAa;IAC1B,GAAG,EAAE,0BAA0B;IAC/B,KAAK,EAAE,cAAc;CACtB,CAAC;AAEF,MAAM,IAAI,GAAgB;IACxB,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,IAAI,EAAE,GAAG;IACT,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,MAAM;IACrB,KAAK,EAAE,OAAO,CAAC,KAAK;CACrB,CAAC;AAEF,MAAM,KAAK,GAAkB;IAC3B,OAAO,EAAE,iBAAiB;IAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,KAAK,EAAE,OAAO,CAAC,KAAK;IACpB,IAAI,EAAE,GAAG;IACT,SAAS,EAAE,CAAC;IACZ,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;CAC/B,CAAC;AAEF,SAAS,aAAa,CACpB,UAMI,EAAE;IAEN,MAAM,WAAW,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG;QACZ,WAAW;QACX,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;QACxE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,aAAa,GAAG;QACpB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;QACvB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;QACzB,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;QAClB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;QACvB,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE;KACrB,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAClD,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;KACrE,CAAC;IACF,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;QACd,KAAK,EACH,OAAO,CAAC,KAAK;YACb,KAAK,SAAS,CAAC;gBACb,MAAM,IAAI,CAAC;YACb,CAAC;QACH,aAAa,EAAE,EAAE,CAAC,EAAE,EAAE;QACtB,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;KACzB,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,CACnB,OAAO,CAAC,OAAO;QACb,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACZ,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;YACvE,MAAM,EAAE,CAAC,KAAK,CAAC;SAChB,CAAC,CAAC,CACN,CAAC;IACF,MAAM,SAAS,GAAG,EAAE,OAAO,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAG,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,MAAM,YAAY,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,+BAA+B,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC;IAErH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;AACrH,CAAC;AAED,MAAM,WAAW,GAAG,CAAC,QAA0B,EAAE,eAAgC,EAAE,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1H,QAAQ,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;AAExC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACpF,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;QAEhC,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QAEzG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,oBAAoB,EAAE,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAC9C;YACE,QAAQ,EAAE,MAAM,CAAC,gBAAgB,CAAC;gBAChC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,+BAA+B;gBACxC,OAAO,EAAE,GAAG;gBACZ,UAAU,EAAE,QAAQ;aACrB,CAAC;YACF,MAAM,EAAE,CAAC,KAAK,CAAC;SAChB,EACD,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CACrD,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACzF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAEtD,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,CAC5B,MAAM,CAAC,eAAe,CAAC,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,kBAAkB,CAAC,CAAC,CACtH,CAAC;QACF,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC;QACrH,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACrH,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QACvG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7H,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;QACjG,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;SACtF,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAEvF,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;QAElG,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACrH,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE;YAClF,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,aAAa,CAAC;YAChC,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;SACtF,CAAC,CAAC;QACH,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;QAErE,MAAM,WAAW,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAE3F,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,oBAAoB,EAAE,CAAC;QACvD,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC1E,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC3G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;SACtF,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE7C,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACzF,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;YACrF,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE;gBACvE,MAAM,EAAE,EAAE;aACX,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAEnE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;QAChI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;QACN,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC3D,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC;KACtC,CAAC,CAAC,4CAA4C,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,EAAE;QAC7F,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;QAEhD,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE7C,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC9E,CAAC;aACI,CAAC;YACJ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACjE,CAAC;QACD,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,KAAK,EAAE,KAAK,SAAS,CAAC;gBACpB,MAAM,IAAI,CAAC;gBACX,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;YACrC,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAEhE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,oBAAoB,EAAE,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvF,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;QACzF,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,KAAK,EAAE,KAAK,SAAS,CAAC;gBACpB,KAAK,CAAC,CAAC,EAAmB,CAAC;gBAC3B,MAAM,IAAI,mBAAmB,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,EAAE,gCAAgC,EAAE;oBACzF,WAAW,EAAE,IAAI;oBACjB,UAAU,EAAE,CAAC;oBACb,OAAO,EAAE,CAAC,WAAW,CAAC;iBACvB,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE7C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3E,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAC7D,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAC9D,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,OAAO,GAAG,aAAa,CAAC;YAC5B,KAAK,EAAE,KAAK,SAAS,CAAC;gBACpB,KAAK,CAAC,CAAC,EAAmB,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;YAClC,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAE9E,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QACrG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,WAAW,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,mDAAmD,CAAC;QAEpE,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QAElE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAC7D,OAAO,CAAC,WAAW,EACnB,2BAA2B,OAAO,uDAAuD,CAC1F,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACpD,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEzG,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,EAAE,CAAC,iBAAiB,EAAE,CAAC;QAC7B,MAAM,GAAG,CAAC;QAEV,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACvE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAC/B,IAAI,EAAE,EAAE;SACT,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACzF,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/server.d.ts

@@ -0,0 +1,88 @@
+export declare class WebDocsServer {
+    private server;
+    private config;
+    private store;
+    private processor;
+    private statusTracker;
+    private indexingQueue;
+    private authManager;
+    private indexingWorkflow;
+    private runPromise?;
+    private closePromise?;
+    private activeToolCalls;
+    /** Maps operation ID to progress token for MCP notifications */
+    private progressTokens;
+    /** Tracks last notified progress to throttle notifications */
+    private lastNotifiedProgress;
+    constructor();
+    /**
+     * Send MCP progress notification to client.
+     * Only sends if the client provided a progressToken in the original request.
+     * Throttled to avoid flooding - sends on 5% increments or status changes.
+     */
+    private sendProgressNotification;
+    private initialize;
+    private setupToolHandlers;
+    private handleAddDocumentation;
+    private handleListDocumentation;
+    private handleSearchDocumentation;
+    private handleReindexDocumentation;
+    private handleGetIndexingStatus;
+    /**
+     * Handle interactive authentication request.
+     * Opens a visible browser for the user to login manually.
+     */
+    private handleAuthenticate;
+    /**
+     * Handle clearing saved authentication for a domain
+     */
+    private handleClearAuth;
+    /**
+     * Handle deleting an indexed documentation site and all its data
+     */
+    private handleDeleteDocumentation;
+    /**
+     * Handle setting tags for a documentation site
+     */
+    private handleSetTags;
+    /**
+     * Handle listing all tags with usage counts
+     */
+    private handleListTags;
+    /**
+     * Handle creating a new collection
+     */
+    private handleCreateCollection;
+    /**
+     * Handle deleting a collection
+     */
+    private handleDeleteCollection;
+    /**
+     * Handle updating a collection's metadata
+     */
+    private handleUpdateCollection;
+    /**
+     * Handle listing all collections
+     */
+    private handleListCollections;
+    /**
+     * Handle getting a specific collection with its documents
+     */
+    private handleGetCollection;
+    /**
+     * Handle adding documents to a collection
+     */
+    private handleAddToCollection;
+    /**
+     * Handle removing documents from a collection
+     */
+    private handleRemoveFromCollection;
+    /**
+     * Handle searching within a collection
+     */
+    private handleSearchCollection;
+    run(): Promise<void>;
+    private start;
+    close(): Promise<void>;
+    private closeResources;
+}