mcp-docs-scraper 0.1.0

package/dist/utils/rate-limit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limit.js","sourceRoot":"","sources":["../../src/utils/rate-limit.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH;;GAEG;AACH,MAAM,OAAO,gBAAgB;IACnB,IAAI,GAAyB,IAAI,CAAC;IAE1C;;OAEG;IACH,iBAAiB,CAAC,OAAgB;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAE/C,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,GAAG;gBACV,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC1B,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBAClC,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE;aACtB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAoB,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,WAAW;QACT,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,iBAAiB;QACf,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,gBAAgB;QAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,+BAA+B,CAAC;QACzC,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,KAAK,CAAC,CAAC;QAEtE,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,mCAAmC,iBAAiB,WAAW,CAAC;QACzE,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACjB,OAAO,mBAAmB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,yBAAyB,iBAAiB,WAAW,CAAC;QACxH,CAAC;QAED,OAAO,eAAe,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,YAAY,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO,SAAS,CAAC;QACjC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,gBAAgB,EAAE,CAAC"}

package/dist/utils/url.d.ts

@@ -0,0 +1,69 @@
+/**
+ * URL utilities for web scraping.
+ *
+ * Provides URL normalization, domain extraction, and link filtering.
+ */
+/**
+ * Normalizes a URL by removing fragments, tracking params, and normalizing format.
+ *
+ * @param url The URL to normalize
+ * @returns Normalized URL string
+ */
+export declare function normalizeUrl(url: string): string;
+/**
+ * Extracts the domain (host) from a URL.
+ *
+ * @param url The URL to extract domain from
+ * @returns Domain string or null if invalid
+ */
+export declare function extractDomain(url: string): string | null;
+/**
+ * Checks if two URLs are on the same domain.
+ *
+ * @param url1 First URL
+ * @param url2 Second URL
+ * @returns True if same domain
+ */
+export declare function isSameDomain(url1: string, url2: string): boolean;
+/**
+ * Resolves a potentially relative URL against a base URL.
+ *
+ * @param href The href to resolve (may be relative or absolute)
+ * @param baseUrl The base URL to resolve against
+ * @returns Absolute URL string or null if invalid
+ */
+export declare function resolveUrl(href: string, baseUrl: string): string | null;
+/**
+ * Checks if a URL should be crawled based on various criteria.
+ *
+ * @param url The URL to check
+ * @param baseUrl The base URL of the crawl
+ * @returns Object with isValid boolean and reason if invalid
+ */
+export declare function shouldCrawl(url: string, baseUrl: string): {
+    isValid: boolean;
+    reason?: string;
+};
+/**
+ * Extracts all links from an HTML document.
+ *
+ * @param html The HTML content
+ * @param baseUrl The base URL for resolving relative links
+ * @returns Array of absolute URLs found in the document
+ */
+export declare function extractLinks(html: string, baseUrl: string): string[];
+/**
+ * Gets the path depth of a URL (number of path segments).
+ *
+ * @param url The URL to analyze
+ * @returns Number of path segments
+ */
+export declare function getPathDepth(url: string): number;
+/**
+ * Converts a URL to a safe filename for caching.
+ *
+ * @param url The URL to convert
+ * @returns Safe filename string
+ */
+export declare function urlToFilename(url: string): string;
+//# sourceMappingURL=url.d.ts.map

package/dist/utils/url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAmBH;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAgChD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOxD;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAWhE;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAYvE;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,GACd;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAwCvC;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAoCpE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAQhD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA+BjD"}

package/dist/utils/url.js

@@ -0,0 +1,251 @@
+/**
+ * URL utilities for web scraping.
+ *
+ * Provides URL normalization, domain extraction, and link filtering.
+ */
+/**
+ * Tracking parameters to remove from URLs.
+ */
+const TRACKING_PARAMS = [
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_term",
+    "utm_content",
+    "ref",
+    "source",
+    "fbclid",
+    "gclid",
+    "msclkid",
+    "_ga",
+];
+/**
+ * Normalizes a URL by removing fragments, tracking params, and normalizing format.
+ *
+ * @param url The URL to normalize
+ * @returns Normalized URL string
+ */
+export function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        // Remove fragment
+        parsed.hash = "";
+        // Remove tracking parameters
+        for (const param of TRACKING_PARAMS) {
+            parsed.searchParams.delete(param);
+        }
+        // Sort remaining search params for consistency
+        parsed.searchParams.sort();
+        // Normalize path - remove trailing slash except for root
+        if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+            parsed.pathname = parsed.pathname.slice(0, -1);
+        }
+        // Decode URL-encoded characters in pathname for readability
+        try {
+            parsed.pathname = decodeURIComponent(parsed.pathname);
+        }
+        catch {
+            // Keep as-is if decoding fails
+        }
+        return parsed.href;
+    }
+    catch {
+        // Return as-is if URL is invalid
+        return url;
+    }
+}
+/**
+ * Extracts the domain (host) from a URL.
+ *
+ * @param url The URL to extract domain from
+ * @returns Domain string or null if invalid
+ */
+export function extractDomain(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.hostname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Checks if two URLs are on the same domain.
+ *
+ * @param url1 First URL
+ * @param url2 Second URL
+ * @returns True if same domain
+ */
+export function isSameDomain(url1, url2) {
+    const domain1 = extractDomain(url1);
+    const domain2 = extractDomain(url2);
+    if (!domain1 || !domain2) {
+        return false;
+    }
+    // Handle www prefix variations
+    const normalize = (d) => d.replace(/^www\./, "");
+    return normalize(domain1) === normalize(domain2);
+}
+/**
+ * Resolves a potentially relative URL against a base URL.
+ *
+ * @param href The href to resolve (may be relative or absolute)
+ * @param baseUrl The base URL to resolve against
+ * @returns Absolute URL string or null if invalid
+ */
+export function resolveUrl(href, baseUrl) {
+    try {
+        // Handle protocol-relative URLs
+        if (href.startsWith("//")) {
+            const base = new URL(baseUrl);
+            return new URL(`${base.protocol}${href}`).href;
+        }
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Checks if a URL should be crawled based on various criteria.
+ *
+ * @param url The URL to check
+ * @param baseUrl The base URL of the crawl
+ * @returns Object with isValid boolean and reason if invalid
+ */
+export function shouldCrawl(url, baseUrl) {
+    try {
+        const parsed = new URL(url);
+        // Only HTTP/HTTPS
+        if (!["http:", "https:"].includes(parsed.protocol)) {
+            return { isValid: false, reason: "non-http protocol" };
+        }
+        // Must be same domain
+        if (!isSameDomain(url, baseUrl)) {
+            return { isValid: false, reason: "external domain" };
+        }
+        // Skip common non-content paths
+        const skipPatterns = [
+            /\/api\//i,
+            /\/auth\//i,
+            /\/login/i,
+            /\/logout/i,
+            /\/signup/i,
+            /\/register/i,
+            /\/admin/i,
+            /\/cdn-cgi\//i,
+            /\.(pdf|zip|tar|gz|exe|dmg|pkg|deb|rpm)$/i,
+            /\.(png|jpg|jpeg|gif|svg|ico|webp)$/i,
+            /\.(css|js|json|xml|rss|atom)$/i,
+            /\.(mp3|mp4|avi|mov|wmv|flv|webm)$/i,
+        ];
+        for (const pattern of skipPatterns) {
+            if (pattern.test(parsed.pathname)) {
+                return { isValid: false, reason: "non-content path" };
+            }
+        }
+        return { isValid: true };
+    }
+    catch {
+        return { isValid: false, reason: "invalid URL" };
+    }
+}
+/**
+ * Extracts all links from an HTML document.
+ *
+ * @param html The HTML content
+ * @param baseUrl The base URL for resolving relative links
+ * @returns Array of absolute URLs found in the document
+ */
+export function extractLinks(html, baseUrl) {
+    const links = [];
+    const seen = new Set();
+    // Simple regex to extract href attributes
+    // This is faster than parsing full DOM for link extraction
+    const hrefRegex = /href=["']([^"']+)["']/gi;
+    let match;
+    while ((match = hrefRegex.exec(html)) !== null) {
+        const href = match[1];
+        // Skip anchors, javascript, mailto, tel
+        if (href.startsWith("#") ||
+            href.startsWith("javascript:") ||
+            href.startsWith("mailto:") ||
+            href.startsWith("tel:")) {
+            continue;
+        }
+        const resolved = resolveUrl(href, baseUrl);
+        if (resolved) {
+            const normalized = normalizeUrl(resolved);
+            // Check if should crawl and not seen
+            const { isValid } = shouldCrawl(normalized, baseUrl);
+            if (isValid && !seen.has(normalized)) {
+                seen.add(normalized);
+                links.push(normalized);
+            }
+        }
+    }
+    return links;
+}
+/**
+ * Gets the path depth of a URL (number of path segments).
+ *
+ * @param url The URL to analyze
+ * @returns Number of path segments
+ */
+export function getPathDepth(url) {
+    try {
+        const parsed = new URL(url);
+        const segments = parsed.pathname.split("/").filter(Boolean);
+        return segments.length;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Converts a URL to a safe filename for caching.
+ *
+ * @param url The URL to convert
+ * @returns Safe filename string
+ */
+export function urlToFilename(url) {
+    try {
+        const parsed = new URL(url);
+        // Use pathname as base
+        let filename = parsed.pathname;
+        // Add search params hash if present
+        if (parsed.search) {
+            const hash = simpleHash(parsed.search);
+            filename += `_${hash}`;
+        }
+        // Clean up the filename
+        filename = filename
+            .replace(/^\//, "") // Remove leading slash
+            .replace(/\//g, "_") // Replace slashes with underscores
+            .replace(/[^a-zA-Z0-9_.-]/g, "_") // Replace special chars
+            .replace(/_+/g, "_") // Collapse multiple underscores
+            .slice(0, 200); // Limit length
+        // Ensure it ends with .md for markdown files
+        if (!filename.endsWith(".md")) {
+            filename = filename || "index";
+            filename += ".md";
+        }
+        return filename;
+    }
+    catch {
+        return "page.md";
+    }
+}
+/**
+ * Simple string hash for deduplication.
+ */
+function simpleHash(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash = hash & hash; // Convert to 32bit integer
+    }
+    return Math.abs(hash).toString(36);
+}
+//# sourceMappingURL=url.js.map

package/dist/utils/url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,eAAe,GAAG;IACtB,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,UAAU;IACV,aAAa;IACb,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,KAAK;CACN,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,kBAAkB;QAClB,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QAEjB,6BAA6B;QAC7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QAED,+CAA+C;QAC/C,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAE3B,yDAAyD;QACzD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,4DAA4D;QAC5D,IAAI,CAAC;YACH,MAAM,CAAC,QAAQ,GAAG,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;QACjC,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY;IACrD,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,+BAA+B;IAC/B,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACtD,IAAI,CAAC;QACH,gCAAgC;QAChC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;QACjD,CAAC;QAED,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,GAAW,EACX,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,kBAAkB;QAClB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;QACzD,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QACvD,CAAC;QAED,gCAAgC;QAChC,MAAM,YAAY,GAAG;YACnB,UAAU;YACV,WAAW;YACX,UAAU;YACV,WAAW;YACX,WAAW;YACX,aAAa;YACb,UAAU;YACV,cAAc;YACd,0CAA0C;YAC1C,qCAAqC;YACrC,gCAAgC;YAChC,oCAAoC;SACrC,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,OAAe;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,0CAA0C;IAC1C,2DAA2D;IAC3D,MAAM,SAAS,GAAG,yBAAyB,CAAC;IAC5C,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEtB,wCAAwC;QACxC,IACE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EACvB,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YAE1C,qCAAqC;YACrC,MAAM,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC5D,OAAO,QAAQ,CAAC,MAAM,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,uBAAuB;QACvB,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAE/B,oCAAoC;QACpC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACvC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC;QACzB,CAAC;QAED,wBAAwB;QACxB,QAAQ,GAAG,QAAQ;aAChB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,uBAAuB;aAC1C,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,mCAAmC;aACvD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,wBAAwB;aACzD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,gCAAgC;aACpD,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,eAAe;QAEjC,6CAA6C;QAC7C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,QAAQ,GAAG,QAAQ,IAAI,OAAO,CAAC;YAC/B,QAAQ,IAAI,KAAK,CAAC;QACpB,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACjC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACrC,CAAC"}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "mcp-docs-scraper",
+  "version": "0.1.0",
+  "description": "An MCP server that gives coding agents fast, efficient access to library documentation",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "mcp-docs-scraper": "dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "documentation",
+    "scraper",
+    "ai",
+    "llm",
+    "claude",
+    "cursor",
+    "github",
+    "web-scraper"
+  ],
+  "author": "Kuba Kwiecien <kwiscion@gmail.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/kwiscion/mcp-docs-scraper.git"
+  },
+  "homepage": "https://github.com/kwiscion/mcp-docs-scraper#readme",
+  "bugs": {
+    "url": "https://github.com/kwiscion/mcp-docs-scraper/issues"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.25.0",
+    "cheerio": "^1.1.2",
+    "minisearch": "^7.2.0",
+    "turndown": "^7.2.2",
+    "zod": "^4.3.5"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/turndown": "^5.0.6",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts"
+  }
+}