mcp-docs-scraper 0.1.0

package/dist/services/search-index.js

@@ -0,0 +1,210 @@
+/**
+ * Search index service - Full-text search using MiniSearch.
+ *
+ * Indexes documentation content for fast searching.
+ * Index is built during docs indexing and stored in cache.
+ */
+import MiniSearch from "minisearch";
+// Current index version
+const INDEX_VERSION = 1;
+// Snippet configuration
+const SNIPPET_LENGTH = 150;
+const SNIPPET_CONTEXT = 50;
+/**
+ * Creates and manages a full-text search index for documentation.
+ */
+export class SearchIndex {
+    miniSearch;
+    documents;
+    constructor() {
+        this.miniSearch = new MiniSearch({
+            fields: ["title", "headings", "content"],
+            storeFields: ["title"],
+            searchOptions: {
+                boost: { title: 3, headings: 2, content: 1 },
+                fuzzy: 0.2,
+                prefix: true,
+            },
+        });
+        this.documents = new Map();
+    }
+    /**
+     * Adds a document to the index.
+     */
+    addDocument(doc) {
+        // Store document for snippet generation
+        this.documents.set(doc.id, {
+            title: doc.title,
+            content: doc.content,
+        });
+        // Add to search index
+        this.miniSearch.add(doc);
+    }
+    /**
+     * Adds multiple documents to the index.
+     */
+    addDocuments(docs) {
+        for (const doc of docs) {
+            this.addDocument(doc);
+        }
+    }
+    /**
+     * Searches the index and returns results with snippets.
+     */
+    search(query, limit = 10) {
+        if (!query.trim()) {
+            return [];
+        }
+        // MiniSearch returns all results, we slice them to the limit
+        const results = this.miniSearch.search(query);
+        return results
+            .slice(0, limit)
+            .map((result) => this.toSearchResult(result, query));
+    }
+    /**
+     * Converts a MiniSearch result to our SearchResult format.
+     */
+    toSearchResult(result, query) {
+        const doc = this.documents.get(result.id);
+        const title = doc?.title || result.id;
+        const content = doc?.content || "";
+        return {
+            path: result.id,
+            title,
+            snippet: this.generateSnippet(content, query),
+            score: result.score,
+        };
+    }
+    /**
+     * Generates a snippet with the matching text highlighted.
+     */
+    generateSnippet(content, query) {
+        if (!content) {
+            return "";
+        }
+        // Find the best position to start the snippet
+        const queryTerms = query.toLowerCase().split(/\s+/);
+        const contentLower = content.toLowerCase();
+        let bestPosition = 0;
+        let bestScore = 0;
+        // Search for query terms and find the position with most matches
+        for (const term of queryTerms) {
+            if (term.length < 2)
+                continue;
+            const pos = contentLower.indexOf(term);
+            if (pos !== -1) {
+                // Score based on how early the term appears
+                const score = 1 / (pos + 1);
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestPosition = pos;
+                }
+            }
+        }
+        // Calculate snippet boundaries
+        const start = Math.max(0, bestPosition - SNIPPET_CONTEXT);
+        const end = Math.min(content.length, start + SNIPPET_LENGTH);
+        // Extract snippet
+        let snippet = content.slice(start, end);
+        // Clean up snippet
+        snippet = snippet
+            .replace(/\n+/g, " ") // Replace newlines with spaces
+            .replace(/\s+/g, " ") // Normalize whitespace
+            .trim();
+        // Add ellipsis if needed
+        if (start > 0) {
+            snippet = "..." + snippet;
+        }
+        if (end < content.length) {
+            snippet = snippet + "...";
+        }
+        return snippet;
+    }
+    /**
+     * Returns the number of indexed documents.
+     */
+    get documentCount() {
+        return this.documents.size;
+    }
+    /**
+     * Serializes the index for storage.
+     */
+    toJSON() {
+        const serialized = {
+            index: this.miniSearch.toJSON(),
+            documents: this.documents,
+            version: INDEX_VERSION,
+        };
+        // Convert Map to array for JSON serialization
+        return JSON.stringify({
+            ...serialized,
+            documents: Array.from(this.documents.entries()),
+        });
+    }
+    /**
+     * Creates a SearchIndex from serialized JSON.
+     */
+    static fromJSON(json) {
+        const parsed = JSON.parse(json);
+        // Check version
+        if (parsed.version !== INDEX_VERSION) {
+            throw new Error(`Search index version mismatch: expected ${INDEX_VERSION}, got ${parsed.version}`);
+        }
+        const searchIndex = new SearchIndex();
+        // Restore MiniSearch index
+        searchIndex.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.index), {
+            fields: ["title", "headings", "content"],
+            storeFields: ["title"],
+            searchOptions: {
+                boost: { title: 3, headings: 2, content: 1 },
+                fuzzy: 0.2,
+                prefix: true,
+            },
+        });
+        // Restore documents map
+        searchIndex.documents = new Map(parsed.documents);
+        return searchIndex;
+    }
+    /**
+     * Clears the index.
+     */
+    clear() {
+        this.miniSearch.removeAll();
+        this.documents.clear();
+    }
+}
+/**
+ * Extracts a title from markdown content.
+ * Returns the first H1 heading, or undefined if none found.
+ */
+export function extractTitle(content) {
+    const match = content.match(/^#\s+(.+)$/m);
+    return match ? match[1].trim() : undefined;
+}
+/**
+ * Extracts all headings from markdown content.
+ * Returns a concatenated string of all headings.
+ */
+export function extractHeadings(content) {
+    const headings = [];
+    const regex = /^#{1,6}\s+(.+)$/gm;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        headings.push(match[1].trim());
+    }
+    return headings.join(" ");
+}
+/**
+ * Creates an indexable document from file content.
+ */
+export function createIndexableDocument(path, content) {
+    const title = extractTitle(content) || path.split("/").pop() || path;
+    const headings = extractHeadings(content);
+    return {
+        id: path,
+        title,
+        headings,
+        content,
+    };
+}
+//# sourceMappingURL=search-index.js.map

package/dist/services/search-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index.js","sourceRoot":"","sources":["../../src/services/search-index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,UAAgD,MAAM,YAAY,CAAC;AA0C1E,wBAAwB;AACxB,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,wBAAwB;AACxB,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B;;GAEG;AACH,MAAM,OAAO,WAAW;IACd,UAAU,CAAgC;IAC1C,SAAS,CAAkD;IAEnE;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAoB;YAClD,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACxC,WAAW,EAAE,CAAC,OAAO,CAAC;YACtB,aAAa,EAAE;gBACb,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC5C,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,GAAsB;QAChC,wCAAwC;QACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;SACrB,CAAC,CAAC;QAEH,sBAAsB;QACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAyB;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,QAAgB,EAAE;QACtC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,6DAA6D;QAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9C,OAAO,OAAO;aACX,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,MAAwB,EACxB,KAAa;QAEb,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,GAAG,EAAE,KAAK,IAAI,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,EAAE;YACf,KAAK;YACL,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC;YAC7C,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,OAAe,EAAE,KAAa;QACpD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,8CAA8C;QAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAE3C,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,iEAAiE;QACjE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAE9B,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,4CAA4C;gBAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAC5B,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,KAAK,CAAC;oBAClB,YAAY,GAAG,GAAG,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,eAAe,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,GAAG,cAAc,CAAC,CAAC;QAE7D,kBAAkB;QAClB,IAAI,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAExC,mBAAmB;QACnB,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,+BAA+B;aACpD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,uBAAuB;aAC5C,IAAI,EAAE,CAAC;QAEV,yBAAyB;QACzB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;QAC5B,CAAC;QACD,IAAI,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,MAAM,UAAU,GAA0B;YACxC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,aAAa;SACvB,CAAC;QAEF,8CAA8C;QAC9C,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,GAAG,UAAU;YACb,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAY;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEhC,gBAAgB;QAChB,IAAI,MAAM,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,2CAA2C,aAAa,SAAS,MAAM,CAAC,OAAO,EAAE,CAClF,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QAEtC,2BAA2B;QAC3B,WAAW,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACzE,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACxC,WAAW,EAAE,CAAC,OAAO,CAAC;YACtB,aAAa,EAAE;gBACb,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC5C,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,wBAAwB;QACxB,WAAW,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAElD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,mBAAmB,CAAC;IAClC,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,OAAe;IAEf,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE1C,OAAO;QACL,EAAE,EAAE,IAAI;QACR,KAAK;QACL,QAAQ;QACR,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/services/web-scraper.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Web Scraper Service - Crawls documentation websites.
+ *
+ * Responsibilities:
+ * - Crawl docs websites with depth limit
+ * - Respect robots.txt
+ * - Extract and normalize links
+ * - Rate limiting between requests
+ */
+/**
+ * Options for the web scraper.
+ */
+export interface ScraperOptions {
+    /** Maximum crawl depth (default: 2, max: 5) */
+    maxDepth?: number;
+    /** Delay between requests in ms (default: 500) */
+    requestDelay?: number;
+    /** Maximum pages to crawl (default: 100) */
+    maxPages?: number;
+    /** Whether to respect robots.txt (default: true) */
+    respectRobotsTxt?: boolean;
+    /** Custom user agent */
+    userAgent?: string;
+}
+/**
+ * Result of scraping a single page.
+ */
+export interface ScrapedPage {
+    /** Original URL */
+    url: string;
+    /** Normalized URL */
+    normalizedUrl: string;
+    /** Safe filename for caching */
+    filename: string;
+    /** Raw HTML content */
+    html: string;
+    /** HTTP status code */
+    status: number;
+    /** Content type header */
+    contentType: string;
+    /** Crawl depth this page was found at */
+    depth: number;
+    /** Links found on this page */
+    links: string[];
+}
+/**
+ * Result of a complete crawl operation.
+ */
+export interface CrawlResult {
+    /** Base URL that was crawled */
+    baseUrl: string;
+    /** All successfully scraped pages */
+    pages: ScrapedPage[];
+    /** URLs that failed to fetch */
+    failed: Array<{
+        url: string;
+        reason: string;
+    }>;
+    /** URLs that were skipped (robots.txt, external, etc.) */
+    skipped: Array<{
+        url: string;
+        reason: string;
+    }>;
+    /** Crawl statistics */
+    stats: {
+        totalDiscovered: number;
+        totalCrawled: number;
+        totalFailed: number;
+        totalSkipped: number;
+        maxDepthReached: number;
+        durationMs: number;
+    };
+}
+/**
+ * Crawls a documentation website starting from a base URL.
+ *
+ * @param startUrl The URL to start crawling from
+ * @param options Scraper options
+ * @returns CrawlResult with all scraped pages and statistics
+ */
+export declare function crawlWebsite(startUrl: string, options?: ScraperOptions): Promise<CrawlResult>;
+/**
+ * Web scraper singleton for convenience.
+ */
+export declare const webScraper: {
+    crawl: typeof crawlWebsite;
+};
+//# sourceMappingURL=web-scraper.d.ts.map

package/dist/services/web-scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-scraper.d.ts","sourceRoot":"","sources":["../../src/services/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,gCAAgC;IAChC,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,0DAA0D;IAC1D,OAAO,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,uBAAuB;IACvB,KAAK,EAAE;QACL,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AA4JD;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,WAAW,CAAC,CAqHtB;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;CAEtB,CAAC"}

package/dist/services/web-scraper.js

@@ -0,0 +1,244 @@
+/**
+ * Web Scraper Service - Crawls documentation websites.
+ *
+ * Responsibilities:
+ * - Crawl docs websites with depth limit
+ * - Respect robots.txt
+ * - Extract and normalize links
+ * - Rate limiting between requests
+ */
+import { normalizeUrl, extractLinks, isSameDomain, urlToFilename, } from "../utils/url.js";
+/**
+ * Default scraper options.
+ */
+const DEFAULT_OPTIONS = {
+    maxDepth: 2,
+    requestDelay: 500,
+    maxPages: 100,
+    respectRobotsTxt: true,
+    userAgent: "mcp-docs-scraper/1.0 (documentation indexer)",
+};
+/**
+ * Maximum allowed depth.
+ */
+const MAX_DEPTH = 5;
+/**
+ * Fetches and parses robots.txt for a domain.
+ */
+async function fetchRobotsTxt(baseUrl, userAgent) {
+    const rules = {
+        disallowedPaths: [],
+    };
+    try {
+        const robotsUrl = new URL("/robots.txt", baseUrl).href;
+        const response = await fetch(robotsUrl, {
+            headers: { "User-Agent": userAgent },
+            signal: AbortSignal.timeout(5000),
+        });
+        if (!response.ok) {
+            return rules; // No robots.txt or error - allow all
+        }
+        const text = await response.text();
+        const lines = text.split("\n");
+        let isRelevantUserAgent = false;
+        for (const line of lines) {
+            const trimmed = line.trim().toLowerCase();
+            // Check user-agent directive
+            if (trimmed.startsWith("user-agent:")) {
+                const agent = trimmed.slice(11).trim();
+                isRelevantUserAgent = agent === "*" || userAgent.toLowerCase().includes(agent);
+            }
+            // Parse disallow directive
+            if (isRelevantUserAgent && trimmed.startsWith("disallow:")) {
+                const path = line.trim().slice(9).trim();
+                if (path) {
+                    rules.disallowedPaths.push(path);
+                }
+            }
+            // Parse crawl-delay directive
+            if (isRelevantUserAgent && trimmed.startsWith("crawl-delay:")) {
+                const delay = parseInt(line.trim().slice(12).trim(), 10);
+                if (!isNaN(delay)) {
+                    rules.crawlDelay = delay * 1000; // Convert to ms
+                }
+            }
+        }
+    }
+    catch {
+        // Ignore errors - assume allowed
+    }
+    return rules;
+}
+/**
+ * Checks if a URL is disallowed by robots.txt rules.
+ */
+function isDisallowed(url, rules) {
+    try {
+        const parsed = new URL(url);
+        const pathname = parsed.pathname;
+        for (const disallowed of rules.disallowedPaths) {
+            // Handle wildcard patterns
+            if (disallowed.includes("*")) {
+                const regex = new RegExp("^" + disallowed.replace(/\*/g, ".*").replace(/\?/g, "\\?") + "$");
+                if (regex.test(pathname)) {
+                    return true;
+                }
+            }
+            else if (pathname.startsWith(disallowed)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Fetches a single page.
+ */
+async function fetchPage(url, userAgent) {
+    try {
+        const response = await fetch(url, {
+            headers: {
+                "User-Agent": userAgent,
+                Accept: "text/html,application/xhtml+xml",
+            },
+            redirect: "follow",
+            signal: AbortSignal.timeout(10000),
+        });
+        const contentType = response.headers.get("content-type") || "";
+        // Only process HTML content
+        if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) {
+            return null;
+        }
+        const html = await response.text();
+        return {
+            html,
+            status: response.status,
+            contentType,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Delays execution for a specified time.
+ */
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Crawls a documentation website starting from a base URL.
+ *
+ * @param startUrl The URL to start crawling from
+ * @param options Scraper options
+ * @returns CrawlResult with all scraped pages and statistics
+ */
+export async function crawlWebsite(startUrl, options = {}) {
+    const startTime = Date.now();
+    // Merge options with defaults
+    const opts = {
+        ...DEFAULT_OPTIONS,
+        ...options,
+        maxDepth: Math.min(options.maxDepth || DEFAULT_OPTIONS.maxDepth, MAX_DEPTH),
+    };
+    // Normalize starting URL
+    const baseUrl = normalizeUrl(startUrl);
+    // Initialize result
+    const result = {
+        baseUrl,
+        pages: [],
+        failed: [],
+        skipped: [],
+        stats: {
+            totalDiscovered: 1,
+            totalCrawled: 0,
+            totalFailed: 0,
+            totalSkipped: 0,
+            maxDepthReached: 0,
+            durationMs: 0,
+        },
+    };
+    // Fetch robots.txt if needed
+    let robotsRules = { disallowedPaths: [] };
+    if (opts.respectRobotsTxt) {
+        robotsRules = await fetchRobotsTxt(baseUrl, opts.userAgent);
+        if (robotsRules.crawlDelay) {
+            opts.requestDelay = Math.max(opts.requestDelay, robotsRules.crawlDelay);
+        }
+    }
+    // Track visited URLs and queue
+    const visited = new Set();
+    const queue = [{ url: baseUrl, depth: 0 }];
+    while (queue.length > 0 && result.pages.length < opts.maxPages) {
+        const current = queue.shift();
+        const { url, depth } = current;
+        // Skip if already visited
+        if (visited.has(url)) {
+            continue;
+        }
+        visited.add(url);
+        // Check robots.txt
+        if (opts.respectRobotsTxt && isDisallowed(url, robotsRules)) {
+            result.skipped.push({ url, reason: "disallowed by robots.txt" });
+            result.stats.totalSkipped++;
+            continue;
+        }
+        // Check same domain
+        if (!isSameDomain(url, baseUrl)) {
+            result.skipped.push({ url, reason: "external domain" });
+            result.stats.totalSkipped++;
+            continue;
+        }
+        // Add delay between requests (except first)
+        if (result.stats.totalCrawled > 0) {
+            await delay(opts.requestDelay);
+        }
+        // Fetch the page
+        const pageResult = await fetchPage(url, opts.userAgent);
+        if (!pageResult) {
+            result.failed.push({ url, reason: "fetch failed or non-HTML content" });
+            result.stats.totalFailed++;
+            continue;
+        }
+        // Extract links for further crawling
+        const links = depth < opts.maxDepth ? extractLinks(pageResult.html, url) : [];
+        // Create scraped page
+        const page = {
+            url,
+            normalizedUrl: normalizeUrl(url),
+            filename: urlToFilename(url),
+            html: pageResult.html,
+            status: pageResult.status,
+            contentType: pageResult.contentType,
+            depth,
+            links,
+        };
+        result.pages.push(page);
+        result.stats.totalCrawled++;
+        result.stats.maxDepthReached = Math.max(result.stats.maxDepthReached, depth);
+        // Add new links to queue
+        if (depth < opts.maxDepth) {
+            for (const link of links) {
+                if (!visited.has(link)) {
+                    queue.push({ url: link, depth: depth + 1 });
+                    result.stats.totalDiscovered++;
+                }
+            }
+        }
+        // Log progress
+        console.error(`[scraper] Crawled ${result.stats.totalCrawled}/${opts.maxPages}: ${url} (depth ${depth})`);
+    }
+    // Update final stats
+    result.stats.durationMs = Date.now() - startTime;
+    return result;
+}
+/**
+ * Web scraper singleton for convenience.
+ */
+export const webScraper = {
+    crawl: crawlWebsite,
+};
+//# sourceMappingURL=web-scraper.js.map

package/dist/services/web-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-scraper.js","sourceRoot":"","sources":["../../src/services/web-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,aAAa,GACd,MAAM,iBAAiB,CAAC;AAuEzB;;GAEG;AACH,MAAM,eAAe,GAA6B;IAChD,QAAQ,EAAE,CAAC;IACX,YAAY,EAAE,GAAG;IACjB,QAAQ,EAAE,GAAG;IACb,gBAAgB,EAAE,IAAI;IACtB,SAAS,EAAE,8CAA8C;CAC1D,CAAC;AAEF;;GAEG;AACH,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,SAAiB;IAEjB,MAAM,KAAK,GAAgB;QACzB,eAAe,EAAE,EAAE;KACpB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;YACtC,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;YACpC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC,CAAC,qCAAqC;QACrD,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/B,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAE1C,6BAA6B;YAC7B,IAAI,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACvC,mBAAmB,GAAG,KAAK,KAAK,GAAG,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACjF,CAAC;YAED,2BAA2B;YAC3B,IAAI,mBAAmB,IAAI,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,IAAI,EAAE,CAAC;oBACT,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,IAAI,mBAAmB,IAAI,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC9D,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;gBACzD,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClB,KAAK,CAAC,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,gBAAgB;gBACnD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;IACnC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,KAAkB;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAEjC,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YAC/C,2BAA2B;YAC3B,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,CAClE,CAAC;gBACF,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACzB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3C,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EAAE,SAAS;gBACvB,MAAM,EAAE,iCAAiC;aAC1C;YACD,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAE/D,4BAA4B;QAC5B,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACrF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnC,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAgB,EAChB,UAA0B,EAAE;IAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,IAAI,GAA6B;QACrC,GAAG,eAAe;QAClB,GAAG,OAAO;QACV,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC;KAC5E,CAAC;IAEF,yBAAyB;IACzB,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEvC,oBAAoB;IACpB,MAAM,MAAM,GAAgB;QAC1B,OAAO;QACP,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE;QACX,KAAK,EAAE;YACL,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,UAAU,EAAE,CAAC;SACd;KACF,CAAC;IAEF,6BAA6B;IAC7B,IAAI,WAAW,GAAgB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACvD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,WAAW,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAElF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/D,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC/B,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QAE/B,0BAA0B;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,mBAAmB;QACnB,IAAI,IAAI,CAAC,gBAAgB,IAAI,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;YAC5D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,oBAAoB;QACpB,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,4CAA4C;QAC5C,IAAI,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACjC,CAAC;QAED,iBAAiB;QACjB,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAExD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,kCAAkC,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,qCAAqC;QACrC,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE9E,sBAAsB;QACtB,MAAM,IAAI,GAAgB;YACxB,GAAG;YACH,aAAa,EAAE,YAAY,CAAC,GAAG,CAAC;YAChC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC;YAC5B,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,KAAK;YACL,KAAK;SACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAE7E,yBAAyB;QACzB,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC5C,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,eAAe;QACf,OAAO,CAAC,KAAK,CAAC,qBAAqB,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,WAAW,KAAK,GAAG,CAAC,CAAC;IAC5G,CAAC;IAED,qBAAqB;IACrB,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAEjD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,YAAY;CACpB,CAAC"}

package/dist/tools/clear-cache.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Input type for clear_cache tool.
+ */
+export interface ClearCacheInput {
+    /** Specific docs ID to clear (optional) */
+    docs_id?: string;
+    /** Clear all cached docs (default: false) */
+    all?: boolean;
+}
+/**
+ * Output type for clear_cache tool.
+ */
+export interface ClearCacheOutput {
+    /** IDs that were cleared */
+    cleared: string[];
+    /** Count of remaining cached docs */
+    remaining: number;
+}
+/**
+ * Removes cached documentation.
+ * Either clears a specific docs entry by ID, or all entries if `all` is true.
+ */
+export declare function clearCache(input: ClearCacheInput): Promise<ClearCacheOutput>;
+//# sourceMappingURL=clear-cache.d.ts.map

package/dist/tools/clear-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clear-cache.d.ts","sourceRoot":"","sources":["../../src/tools/clear-cache.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAyBlF"}

package/dist/tools/clear-cache.js

@@ -0,0 +1,29 @@
+import { cacheManager } from "../services/cache-manager.js";
+/**
+ * Removes cached documentation.
+ * Either clears a specific docs entry by ID, or all entries if `all` is true.
+ */
+export async function clearCache(input) {
+    await cacheManager.initialize();
+    const cleared = [];
+    if (input.all) {
+        // Clear everything
+        const allCleared = await cacheManager.clearAll();
+        cleared.push(...allCleared);
+    }
+    else if (input.docs_id) {
+        // Clear specific entry - try both sources
+        const meta = await cacheManager.findById(input.docs_id);
+        if (meta) {
+            await cacheManager.clearEntry(meta.source, meta.id);
+            cleared.push(meta.id);
+        }
+    }
+    // Count remaining entries
+    const remaining = (await cacheManager.listEntries()).length;
+    return {
+        cleared,
+        remaining,
+    };
+}
+//# sourceMappingURL=clear-cache.js.map

package/dist/tools/clear-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clear-cache.js","sourceRoot":"","sources":["../../src/tools/clear-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAsB5D;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAsB;IACrD,MAAM,YAAY,CAAC,UAAU,EAAE,CAAC;IAEhC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,mBAAmB;QACnB,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACzB,0CAA0C;QAC1C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,SAAS,GAAG,CAAC,MAAM,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;IAE5D,OAAO;QACL,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/tools/detect-github.d.ts

@@ -0,0 +1,21 @@
+/**
+ * detect_github_repo tool - Finds GitHub repository from a docs website URL.
+ */
+import { type GitHubDetectionResult } from "../services/github-detector.js";
+/**
+ * Input for the detect_github_repo tool.
+ */
+export interface DetectGitHubInput {
+    /** Docs website URL to analyze */
+    url: string;
+}
+/**
+ * Output from the detect_github_repo tool.
+ * Same as GitHubDetectionResult.
+ */
+export type DetectGitHubOutput = GitHubDetectionResult;
+/**
+ * Detects GitHub repository from a documentation website URL.
+ */
+export declare function detectGitHub(input: DetectGitHubInput): Promise<DetectGitHubOutput>;
+//# sourceMappingURL=detect-github.d.ts.map

package/dist/tools/detect-github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-github.d.ts","sourceRoot":"","sources":["../../src/tools/detect-github.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,gCAAgC,CAAC;AAGxC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,qBAAqB,CAAC;AAEvD;;GAEG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,kBAAkB,CAAC,CAU7B"}

package/dist/tools/detect-github.js

@@ -0,0 +1,18 @@
+/**
+ * detect_github_repo tool - Finds GitHub repository from a docs website URL.
+ */
+import { detectGitHubRepo, } from "../services/github-detector.js";
+import { ValidationError } from "../types/errors.js";
+/**
+ * Detects GitHub repository from a documentation website URL.
+ */
+export async function detectGitHub(input) {
+    const { url } = input;
+    // Validate required parameters
+    if (!url) {
+        throw new ValidationError("Missing required parameter: url", "url");
+    }
+    // Run detection
+    return detectGitHubRepo(url);
+}
+//# sourceMappingURL=detect-github.js.map