mcp-docs-scraper 0.1.0

package/dist/services/github-detector.d.ts

@@ -0,0 +1,49 @@
+/**
+ * GitHub Detector Service - Detects GitHub repository from documentation websites.
+ *
+ * Detection strategies (in order of confidence):
+ * 1. Direct GitHub URL → high confidence
+ * 2. github.io pattern → high confidence
+ * 3. "Edit on GitHub" links → high confidence
+ * 4. GitHub links in page content → medium confidence
+ * 5. Meta tags with repo info → medium confidence
+ */
+/**
+ * Result of GitHub repository detection.
+ */
+export interface GitHubDetectionResult {
+    /** Whether a GitHub repository was found */
+    found: boolean;
+    /** Repository in "owner/repo" format */
+    repo?: string;
+    /** Path within the repo where docs are located */
+    docs_path?: string;
+    /** Confidence level of the detection */
+    confidence: "high" | "medium" | "low";
+    /** How the repo was detected (for debugging) */
+    detection_method?: string;
+}
+/**
+ * Options for GitHub detection.
+ */
+export interface DetectionOptions {
+    /** Timeout for fetching the page (ms) */
+    timeout?: number;
+    /** User agent for requests */
+    userAgent?: string;
+}
+/**
+ * Detects GitHub repository from a documentation website URL.
+ *
+ * @param url The URL to analyze
+ * @param options Detection options
+ * @returns Detection result with repo info and confidence
+ */
+export declare function detectGitHubRepo(url: string, options?: DetectionOptions): Promise<GitHubDetectionResult>;
+/**
+ * GitHub detector singleton for convenience.
+ */
+export declare const githubDetector: {
+    detect: typeof detectGitHubRepo;
+};
+//# sourceMappingURL=github-detector.d.ts.map

package/dist/services/github-detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-detector.d.ts","sourceRoot":"","sources":["../../src/services/github-detector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,4CAA4C;IAC5C,KAAK,EAAE,OAAO,CAAC;IACf,wCAAwC;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA+OD;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA2ChC;AAED;;GAEG;AACH,eAAO,MAAM,cAAc;;CAE1B,CAAC"}

package/dist/services/github-detector.js

@@ -0,0 +1,276 @@
+/**
+ * GitHub Detector Service - Detects GitHub repository from documentation websites.
+ *
+ * Detection strategies (in order of confidence):
+ * 1. Direct GitHub URL → high confidence
+ * 2. github.io pattern → high confidence
+ * 3. "Edit on GitHub" links → high confidence
+ * 4. GitHub links in page content → medium confidence
+ * 5. Meta tags with repo info → medium confidence
+ */
+import { parseGitHubUrl } from "../tools/index-docs.js";
+const DEFAULT_OPTIONS = {
+    timeout: 10000,
+    userAgent: "mcp-docs-scraper/1.0 (github-detector)",
+};
+/**
+ * Patterns for detecting GitHub links in HTML.
+ */
+const GITHUB_LINK_PATTERNS = [
+    // Edit on GitHub links (highest priority)
+    /href=["']([^"']*github\.com\/[^"'\/]+\/[^"'\/]+(?:\/[^"']*)?)["'][^>]*>(?:[^<]*(?:edit|view|source)[^<]*github|github[^<]*(?:edit|view|source))/gi,
+    // General GitHub repo links
+    /href=["'](https?:\/\/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+))(?:\/[^"']*)?["']/gi,
+];
+/**
+ * Extracts owner/repo from a GitHub URL.
+ */
+function extractRepoFromUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (!parsed.hostname.includes("github.com") && !parsed.hostname.includes("github.io")) {
+            return null;
+        }
+        // Handle github.io URLs
+        if (parsed.hostname.endsWith(".github.io")) {
+            const owner = parsed.hostname.replace(".github.io", "");
+            // For github.io, the first path segment is usually the repo
+            const pathParts = parsed.pathname.split("/").filter(Boolean);
+            const repo = pathParts[0] || `${owner}.github.io`;
+            return { owner, repo };
+        }
+        // Handle github.com URLs
+        const pathParts = parsed.pathname.split("/").filter(Boolean);
+        if (pathParts.length >= 2) {
+            const owner = pathParts[0];
+            const repo = pathParts[1].replace(/\.git$/, "");
+            const path = pathParts.length > 2 ? pathParts.slice(2).join("/") : undefined;
+            return { owner, repo, path };
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Detects if the URL itself is a GitHub URL or github.io URL.
+ */
+function detectFromUrl(url) {
+    // Check if it's already a GitHub URL
+    const githubInfo = parseGitHubUrl(url);
+    if (githubInfo) {
+        return {
+            found: true,
+            repo: `${githubInfo.owner}/${githubInfo.repo}`,
+            docs_path: githubInfo.path,
+            confidence: "high",
+            detection_method: "direct_github_url",
+        };
+    }
+    // Check for github.io pattern
+    try {
+        const parsed = new URL(url);
+        if (parsed.hostname.endsWith(".github.io")) {
+            const owner = parsed.hostname.replace(".github.io", "");
+            const pathParts = parsed.pathname.split("/").filter(Boolean);
+            // For github.io sites, the repo is often the first path segment
+            // or it could be a user/org pages site (owner.github.io)
+            if (pathParts.length > 0) {
+                // Could be a project page: owner.github.io/repo-name
+                return {
+                    found: true,
+                    repo: `${owner}/${pathParts[0]}`,
+                    confidence: "high",
+                    detection_method: "github_io_project",
+                };
+            }
+            else {
+                // User/org pages: owner.github.io → owner/owner.github.io
+                return {
+                    found: true,
+                    repo: `${owner}/${owner}.github.io`,
+                    confidence: "high",
+                    detection_method: "github_io_user",
+                };
+            }
+        }
+    }
+    catch {
+        // Invalid URL
+    }
+    return null;
+}
+/**
+ * Extracts GitHub repo from "Edit on GitHub" or similar links in HTML.
+ */
+function detectFromEditLinks(html) {
+    // Look for "Edit on GitHub" style links
+    const editPatterns = [
+        /href=["']([^"']*github\.com\/[^"']+)["'][^>]*>[^<]*edit[^<]*on[^<]*github/gi,
+        /href=["']([^"']*github\.com\/[^"']+)["'][^>]*>[^<]*view[^<]*on[^<]*github/gi,
+        /href=["']([^"']*github\.com\/[^"']+)["'][^>]*>[^<]*source[^<]*on[^<]*github/gi,
+        /href=["']([^"']*github\.com\/[^"']+)["'][^>]*>[^<]*github[^<]*source/gi,
+        /"editUrl":\s*"([^"]*github\.com\/[^"]+)"/gi,
+        /"edit_uri":\s*"([^"]*github\.com\/[^"]+)"/gi,
+    ];
+    for (const pattern of editPatterns) {
+        const match = pattern.exec(html);
+        if (match) {
+            const repoInfo = extractRepoFromUrl(match[1]);
+            if (repoInfo) {
+                return {
+                    found: true,
+                    repo: `${repoInfo.owner}/${repoInfo.repo}`,
+                    docs_path: repoInfo.path,
+                    confidence: "high",
+                    detection_method: "edit_on_github_link",
+                };
+            }
+        }
+        // Reset regex lastIndex
+        pattern.lastIndex = 0;
+    }
+    return null;
+}
+/**
+ * Extracts GitHub repo from general GitHub links in HTML.
+ */
+function detectFromLinks(html) {
+    // Find all GitHub links
+    const linkPattern = /href=["'](https?:\/\/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+))(?:\/[^"']*)?["']/gi;
+    const repos = new Map();
+    let match;
+    while ((match = linkPattern.exec(html)) !== null) {
+        const owner = match[2];
+        const repo = match[3].replace(/\.git$/, "");
+        // Skip common non-repo patterns
+        if (["issues", "pulls", "discussions", "sponsors", "marketplace"].includes(repo)) {
+            continue;
+        }
+        const key = `${owner}/${repo}`;
+        repos.set(key, (repos.get(key) || 0) + 1);
+    }
+    if (repos.size === 0) {
+        return null;
+    }
+    // Find the most frequently linked repo
+    let bestRepo = "";
+    let maxCount = 0;
+    for (const [repo, count] of repos) {
+        if (count > maxCount) {
+            maxCount = count;
+            bestRepo = repo;
+        }
+    }
+    if (bestRepo) {
+        return {
+            found: true,
+            repo: bestRepo,
+            confidence: maxCount >= 3 ? "medium" : "low",
+            detection_method: `github_links_found_${maxCount}`,
+        };
+    }
+    return null;
+}
+/**
+ * Extracts GitHub repo from meta tags.
+ */
+function detectFromMetaTags(html) {
+    // Look for og:url or other meta tags with GitHub info
+    const metaPatterns = [
+        /<meta[^>]+property=["']og:url["'][^>]+content=["']([^"']+github\.com[^"']+)["']/gi,
+        /<meta[^>]+name=["']github:repo["'][^>]+content=["']([^"']+)["']/gi,
+        /<meta[^>]+content=["']([^"']+github\.com[^"']+)["'][^>]+property=["']og:url["']/gi,
+    ];
+    for (const pattern of metaPatterns) {
+        const match = pattern.exec(html);
+        if (match) {
+            const repoInfo = extractRepoFromUrl(match[1]);
+            if (repoInfo) {
+                return {
+                    found: true,
+                    repo: `${repoInfo.owner}/${repoInfo.repo}`,
+                    confidence: "medium",
+                    detection_method: "meta_tag",
+                };
+            }
+        }
+        pattern.lastIndex = 0;
+    }
+    return null;
+}
+/**
+ * Fetches a webpage and returns its HTML content.
+ */
+async function fetchPage(url, options) {
+    try {
+        const response = await fetch(url, {
+            headers: {
+                "User-Agent": options.userAgent,
+                Accept: "text/html,application/xhtml+xml",
+            },
+            redirect: "follow",
+            signal: AbortSignal.timeout(options.timeout),
+        });
+        if (!response.ok) {
+            return null;
+        }
+        return await response.text();
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Detects GitHub repository from a documentation website URL.
+ *
+ * @param url The URL to analyze
+ * @param options Detection options
+ * @returns Detection result with repo info and confidence
+ */
+export async function detectGitHubRepo(url, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    // Strategy 1: Check if URL itself is GitHub or github.io
+    const urlResult = detectFromUrl(url);
+    if (urlResult) {
+        return urlResult;
+    }
+    // Strategy 2-5: Fetch the page and analyze content
+    const html = await fetchPage(url, opts);
+    if (!html) {
+        return {
+            found: false,
+            confidence: "low",
+            detection_method: "fetch_failed",
+        };
+    }
+    // Strategy 2: Look for "Edit on GitHub" links (highest confidence)
+    const editResult = detectFromEditLinks(html);
+    if (editResult) {
+        return editResult;
+    }
+    // Strategy 3: Look for meta tags
+    const metaResult = detectFromMetaTags(html);
+    if (metaResult) {
+        return metaResult;
+    }
+    // Strategy 4: Look for any GitHub links
+    const linkResult = detectFromLinks(html);
+    if (linkResult) {
+        return linkResult;
+    }
+    // No GitHub repo found
+    return {
+        found: false,
+        confidence: "low",
+        detection_method: "no_github_found",
+    };
+}
+/**
+ * GitHub detector singleton for convenience.
+ */
+export const githubDetector = {
+    detect: detectGitHubRepo,
+};
+//# sourceMappingURL=github-detector.js.map

package/dist/services/github-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-detector.js","sourceRoot":"","sources":["../../src/services/github-detector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AA4BxD,MAAM,eAAe,GAA+B;IAClD,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,wCAAwC;CACpD,CAAC;AAEF;;GAEG;AACH,MAAM,oBAAoB,GAAG;IAC3B,0CAA0C;IAC1C,mJAAmJ;IACnJ,4BAA4B;IAC5B,2FAA2F;CAC5F,CAAC;AAEF;;GAEG;AACH,SAAS,kBAAkB,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wBAAwB;QACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACxD,4DAA4D;YAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7D,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,YAAY,CAAC;YAClD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACzB,CAAC;QAED,yBAAyB;QACzB,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,GAAW;IAChC,qCAAqC;IACrC,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO;YACL,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,GAAG,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,IAAI,EAAE;YAC9C,SAAS,EAAE,UAAU,CAAC,IAAI;YAC1B,UAAU,EAAE,MAAM;YAClB,gBAAgB,EAAE,mBAAmB;SACtC,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACxD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE7D,gEAAgE;YAChE,yDAAyD;YACzD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,qDAAqD;gBACrD,OAAO;oBACL,KAAK,EAAE,IAAI;oBACX,IAAI,EAAE,GAAG,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE;oBAChC,UAAU,EAAE,MAAM;oBAClB,gBAAgB,EAAE,mBAAmB;iBACtC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,0DAA0D;gBAC1D,OAAO;oBACL,KAAK,EAAE,IAAI;oBACX,IAAI,EAAE,GAAG,KAAK,IAAI,KAAK,YAAY;oBACnC,UAAU,EAAE,MAAM;oBAClB,gBAAgB,EAAE,gBAAgB;iBACnC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,wCAAwC;IACxC,MAAM,YAAY,GAAG;QACnB,6EAA6E;QAC7E,6EAA6E;QAC7E,+EAA+E;QAC/E,wEAAwE;QACxE,4CAA4C;QAC5C,6CAA6C;KAC9C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO;oBACL,KAAK,EAAE,IAAI;oBACX,IAAI,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;oBAC1C,SAAS,EAAE,QAAQ,CAAC,IAAI;oBACxB,UAAU,EAAE,MAAM;oBAClB,gBAAgB,EAAE,qBAAqB;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,wBAAwB;QACxB,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,wBAAwB;IACxB,MAAM,WAAW,GAAG,2FAA2F,CAAC;IAChH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE5C,gCAAgC;QAChC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjF,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uCAAuC;IACvC,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QAClC,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;YACrB,QAAQ,GAAG,KAAK,CAAC;YACjB,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO;YACL,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;YAC5C,gBAAgB,EAAE,sBAAsB,QAAQ,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,sDAAsD;IACtD,MAAM,YAAY,GAAG;QACnB,mFAAmF;QACnF,mEAAmE;QACnE,mFAAmF;KACpF,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO;oBACL,KAAK,EAAE,IAAI;oBACX,IAAI,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;oBAC1C,UAAU,EAAE,QAAQ;oBACpB,gBAAgB,EAAE,UAAU;iBAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAAmC;IACvE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS;gBAC/B,MAAM,EAAE,iCAAiC;aAC1C;YACD,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;SAC7C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAA4B,EAAE;IAE9B,MAAM,IAAI,GAA+B,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAE5E,yDAAyD;IACzD,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mDAAmD;IACnD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,KAAK;YACjB,gBAAgB,EAAE,cAAc;SACjC,CAAC;IACJ,CAAC;IAED,mEAAmE;IACnE,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,iCAAiC;IACjC,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,wCAAwC;IACxC,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,uBAAuB;IACvB,OAAO;QACL,KAAK,EAAE,KAAK;QACZ,UAAU,EAAE,KAAK;QACjB,gBAAgB,EAAE,iBAAiB;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,MAAM,EAAE,gBAAgB;CACzB,CAAC"}

package/dist/services/github-fetcher.d.ts

@@ -0,0 +1,94 @@
+import type { DocsTreeNode } from "../types/index.js";
+/**
+ * Result of fetching a repository tree.
+ */
+export interface FetchTreeResult {
+    /** The repository in "owner/repo" format */
+    repo: string;
+    /** The branch that was used */
+    branch: string;
+    /** The hierarchical file tree */
+    tree: DocsTreeNode[];
+    /** Total number of files found */
+    fileCount: number;
+    /** Total size of all files in bytes */
+    totalSize: number;
+    /** Whether the tree was truncated due to size limits */
+    truncated: boolean;
+}
+/**
+ * Options for tree fetching.
+ */
+export interface FetchTreeOptions {
+    /** Starting path within the repo (default: root) */
+    path?: string;
+    /** Branch to use (default: auto-detect main/master) */
+    branch?: string;
+    /** File extensions to include (default: all markdown) */
+    extensions?: string[];
+    /** Maximum depth to traverse (default: 10) */
+    maxDepth?: number;
+}
+/**
+ * Parses "owner/repo" format into separate parts.
+ */
+export declare function parseRepoString(repoString: string): {
+    owner: string;
+    repo: string;
+};
+/**
+ * Checks if GitHub authentication is configured.
+ */
+export declare function isAuthenticated(): boolean;
+/**
+ * Fetches the file tree from a GitHub repository.
+ * Uses the Git Trees API for efficiency (1 request for entire tree).
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param options Fetch options
+ * @returns Tree structure with metadata
+ */
+export declare function fetchRepoTree(repoString: string, options?: FetchTreeOptions): Promise<FetchTreeResult>;
+/**
+ * Result of fetching file content.
+ */
+export interface FetchContentResult {
+    /** The file path */
+    path: string;
+    /** The file content */
+    content: string;
+    /** Size in bytes */
+    size: number;
+}
+/**
+ * Fetches raw file content from GitHub via raw.githubusercontent.com.
+ * This endpoint has no rate limits (served from CDN).
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param branch Branch name
+ * @param filePath Path to the file within the repo
+ * @returns File content or null if not found
+ */
+export declare function fetchFileContent(repoString: string, branch: string, filePath: string): Promise<FetchContentResult | null>;
+/**
+ * Fetches multiple files from a repository.
+ * Handles 404s gracefully by skipping missing files.
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param branch Branch name
+ * @param filePaths Array of file paths to fetch
+ * @returns Array of successfully fetched files and array of not found paths
+ */
+export declare function fetchMultipleFiles(repoString: string, branch: string, filePaths: string[]): Promise<{
+    files: FetchContentResult[];
+    notFound: string[];
+}>;
+/**
+ * Gets the current GitHub API rate limit status.
+ */
+export declare function getRateLimitStatus(): string;
+/**
+ * Gets the raw rate limit info.
+ */
+export declare function getRateLimitInfo(): import("../utils/rate-limit.js").RateLimitInfo | null;
+//# sourceMappingURL=github-fetcher.d.ts.map

package/dist/services/github-fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-fetcher.d.ts","sourceRoot":"","sources":["../../src/services/github-fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAiCtD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oDAAoD;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;GAEG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd,CAQA;AAoBD;;GAEG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AA8SD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CAqC1B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA6BpC;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC;IACT,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC,CA2BD;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,MAAM,CAE3C;AAED;;GAEG;AACH,wBAAgB,gBAAgB,0DAE/B"}