mcp-docs-scraper 0.1.0

package/dist/services/github-fetcher.js

@@ -0,0 +1,393 @@
+import { githubRateLimit } from "../utils/rate-limit.js";
+import { GitHubRateLimitError, GitHubNotFoundError, GitHubAccessDeniedError, NetworkError, } from "../types/errors.js";
+const GITHUB_API_BASE = "https://api.github.com";
+const DEFAULT_EXTENSIONS = [".md", ".mdx", ".markdown"];
+/**
+ * Parses "owner/repo" format into separate parts.
+ */
+export function parseRepoString(repoString) {
+    const parts = repoString.split("/");
+    if (parts.length !== 2) {
+        throw new Error(`Invalid repo format: "${repoString}". Expected "owner/repo".`);
+    }
+    return { owner: parts[0], repo: parts[1] };
+}
+/**
+ * Gets GitHub API headers, including auth token if available.
+ */
+function getGitHubHeaders() {
+    const headers = {
+        Accept: "application/vnd.github.v3+json",
+        "User-Agent": "mcp-docs-scraper",
+    };
+    // Use GITHUB_TOKEN if available (increases rate limit from 60 to 5000/hour)
+    const token = process.env.GITHUB_TOKEN;
+    if (token) {
+        headers.Authorization = `Bearer ${token}`;
+    }
+    return headers;
+}
+/**
+ * Checks if GitHub authentication is configured.
+ */
+export function isAuthenticated() {
+    return !!process.env.GITHUB_TOKEN;
+}
+/**
+ * Makes a GitHub API request with proper headers and rate limit tracking.
+ */
+async function githubFetch(url) {
+    const response = await fetch(url, {
+        headers: getGitHubHeaders(),
+    });
+    githubRateLimit.updateFromHeaders(response.headers);
+    return response;
+}
+/**
+ * Detects the default branch for a repository using the Git Trees API.
+ * Tries "main" first, then "master".
+ */
+async function detectDefaultBranch(owner, repo) {
+    const repoString = `${owner}/${repo}`;
+    // Check rate limit before making requests
+    if (githubRateLimit.isExhausted()) {
+        throw new GitHubRateLimitError(githubRateLimit.getResetTime());
+    }
+    try {
+        // Try main first using Git Trees API
+        const mainUrl = `${GITHUB_API_BASE}/repos/${owner}/${repo}/git/trees/main`;
+        const mainResponse = await githubFetch(mainUrl);
+        if (mainResponse.ok) {
+            return "main";
+        }
+        // Check for specific error types
+        if (mainResponse.status === 403) {
+            throw new GitHubAccessDeniedError(repoString);
+        }
+        // Try master
+        const masterUrl = `${GITHUB_API_BASE}/repos/${owner}/${repo}/git/trees/master`;
+        const masterResponse = await githubFetch(masterUrl);
+        if (masterResponse.ok) {
+            return "master";
+        }
+        if (masterResponse.status === 403) {
+            throw new GitHubAccessDeniedError(repoString);
+        }
+        // Neither branch found - repo might not exist
+        throw new GitHubNotFoundError(repoString);
+    }
+    catch (error) {
+        if (error instanceof GitHubRateLimitError ||
+            error instanceof GitHubAccessDeniedError ||
+            error instanceof GitHubNotFoundError) {
+            throw error;
+        }
+        throw new NetworkError(`https://github.com/${repoString}`, error instanceof Error ? error : undefined);
+    }
+}
+/**
+ * Fetches the entire tree from GitHub using the Git Trees API.
+ * This uses only 1 API request instead of 1 per directory.
+ */
+async function fetchGitTree(owner, repo, branch) {
+    const repoString = `${owner}/${repo}`;
+    // Check rate limit before making request
+    if (githubRateLimit.isExhausted()) {
+        throw new GitHubRateLimitError(githubRateLimit.getResetTime());
+    }
+    if (githubRateLimit.isLow()) {
+        console.error(`Warning: ${githubRateLimit.getStatusMessage()}`);
+    }
+    const url = `${GITHUB_API_BASE}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;
+    try {
+        const response = await githubFetch(url);
+        if (!response.ok) {
+            if (response.status === 404) {
+                throw new GitHubNotFoundError(repoString);
+            }
+            if (response.status === 403) {
+                // Could be rate limit or access denied
+                if (githubRateLimit.isExhausted()) {
+                    throw new GitHubRateLimitError(githubRateLimit.getResetTime());
+                }
+                throw new GitHubAccessDeniedError(repoString);
+            }
+            throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+        }
+        return response.json();
+    }
+    catch (error) {
+        if (error instanceof GitHubRateLimitError ||
+            error instanceof GitHubNotFoundError ||
+            error instanceof GitHubAccessDeniedError) {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes("GitHub API error")) {
+            throw error;
+        }
+        throw new NetworkError(url, error instanceof Error ? error : undefined);
+    }
+}
+/**
+ * Checks if a file should be included based on extension filter.
+ */
+function shouldIncludeFile(filename, extensions) {
+    if (extensions.length === 0)
+        return true;
+    const lowerName = filename.toLowerCase();
+    return extensions.some((ext) => lowerName.endsWith(ext.toLowerCase()));
+}
+/**
+ * Gets the depth of a path (number of directory levels).
+ */
+function getPathDepth(path) {
+    if (!path)
+        return 0;
+    return path.split("/").length;
+}
+/**
+ * Gets the file/folder name from a path.
+ */
+function getNameFromPath(path) {
+    const parts = path.split("/");
+    return parts[parts.length - 1];
+}
+/**
+ * Gets the parent path from a path.
+ */
+function getParentPath(path) {
+    const parts = path.split("/");
+    parts.pop();
+    return parts.join("/");
+}
+/**
+ * Converts a flat list of tree items into a hierarchical DocsTreeNode structure.
+ */
+function buildHierarchicalTree(items, basePath, extensions, maxDepth) {
+    // Filter items by base path and extensions
+    const filteredItems = items.filter((item) => {
+        // Must be under base path
+        if (basePath &&
+            !item.path.startsWith(basePath + "/") &&
+            item.path !== basePath) {
+            return false;
+        }
+        // Calculate relative depth
+        const relativePath = basePath
+            ? item.path.slice(basePath.length + 1)
+            : item.path;
+        const depth = getPathDepth(relativePath);
+        if (depth > maxDepth) {
+            return false;
+        }
+        // For files, check extension filter
+        if (item.type === "blob") {
+            return shouldIncludeFile(item.path, extensions);
+        }
+        return true; // Include directories for now, we'll filter empty ones later
+    });
+    // Build a map of path -> node for quick lookups
+    const nodeMap = new Map();
+    let fileCount = 0;
+    let totalSize = 0;
+    // First pass: create nodes for all matching files
+    for (const item of filteredItems) {
+        if (item.type === "blob") {
+            const node = {
+                name: getNameFromPath(item.path),
+                path: item.path,
+                type: "file",
+                size_bytes: item.size,
+            };
+            nodeMap.set(item.path, node);
+            fileCount++;
+            totalSize += item.size || 0;
+        }
+    }
+    // Second pass: create folder nodes for paths that have files under them
+    const folderPaths = new Set();
+    for (const item of filteredItems) {
+        if (item.type === "blob") {
+            // Walk up the path and add all parent folders
+            let parentPath = getParentPath(item.path);
+            while (parentPath &&
+                (!basePath ||
+                    parentPath.startsWith(basePath) ||
+                    parentPath === basePath)) {
+                // Don't add folders above basePath
+                if (basePath && parentPath.length < basePath.length)
+                    break;
+                folderPaths.add(parentPath);
+                parentPath = getParentPath(parentPath);
+            }
+        }
+    }
+    // Create folder nodes
+    for (const folderPath of folderPaths) {
+        if (!nodeMap.has(folderPath)) {
+            nodeMap.set(folderPath, {
+                name: getNameFromPath(folderPath),
+                path: folderPath,
+                type: "folder",
+                children: [],
+            });
+        }
+    }
+    // Third pass: build the hierarchy
+    for (const [path, node] of nodeMap) {
+        const parentPath = getParentPath(path);
+        // Skip if this is at the root level (relative to basePath)
+        if (!parentPath || (basePath && parentPath.length < basePath.length)) {
+            continue;
+        }
+        const parent = nodeMap.get(parentPath);
+        if (parent && parent.type === "folder") {
+            parent.children = parent.children || [];
+            parent.children.push(node);
+        }
+    }
+    // Collect root-level nodes
+    const rootNodes = [];
+    for (const [path, node] of nodeMap) {
+        const parentPath = getParentPath(path);
+        const isRootLevel = basePath
+            ? path.startsWith(basePath + "/") &&
+                !parentPath.includes("/", basePath.length + 1) &&
+                parentPath === basePath
+            : !path.includes("/");
+        // For base path, check if direct child
+        if (basePath) {
+            const relativePath = path.slice(basePath.length + 1);
+            if (!relativePath.includes("/")) {
+                rootNodes.push(node);
+            }
+        }
+        else if (!path.includes("/")) {
+            rootNodes.push(node);
+        }
+    }
+    // Sort all nodes: folders first, then files, both alphabetically
+    const sortNodes = (nodes) => {
+        nodes.sort((a, b) => {
+            if (a.type !== b.type) {
+                return a.type === "folder" ? -1 : 1;
+            }
+            return a.name.localeCompare(b.name);
+        });
+        for (const node of nodes) {
+            if (node.children) {
+                sortNodes(node.children);
+            }
+        }
+    };
+    sortNodes(rootNodes);
+    return { tree: rootNodes, fileCount, totalSize };
+}
+/**
+ * Fetches the file tree from a GitHub repository.
+ * Uses the Git Trees API for efficiency (1 request for entire tree).
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param options Fetch options
+ * @returns Tree structure with metadata
+ */
+export async function fetchRepoTree(repoString, options = {}) {
+    const { owner, repo } = parseRepoString(repoString);
+    const { path = "", branch: requestedBranch, extensions = DEFAULT_EXTENSIONS, maxDepth = 10, } = options;
+    // Detect or use specified branch
+    const branch = requestedBranch || (await detectDefaultBranch(owner, repo));
+    // Fetch the entire tree in one API call
+    const gitTree = await fetchGitTree(owner, repo, branch);
+    if (gitTree.truncated) {
+        console.error("Warning: Repository tree was truncated due to size. Some files may be missing.");
+    }
+    // Build hierarchical tree from flat list
+    const result = buildHierarchicalTree(gitTree.tree, path, extensions, maxDepth);
+    return {
+        repo: `${owner}/${repo}`,
+        branch,
+        tree: result.tree,
+        fileCount: result.fileCount,
+        totalSize: result.totalSize,
+        truncated: gitTree.truncated,
+    };
+}
+/**
+ * Fetches raw file content from GitHub via raw.githubusercontent.com.
+ * This endpoint has no rate limits (served from CDN).
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param branch Branch name
+ * @param filePath Path to the file within the repo
+ * @returns File content or null if not found
+ */
+export async function fetchFileContent(repoString, branch, filePath) {
+    const { owner, repo } = parseRepoString(repoString);
+    // Use raw.githubusercontent.com - no rate limits, served from CDN
+    const url = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${filePath}`;
+    const response = await fetch(url, {
+        headers: {
+            "User-Agent": "mcp-docs-scraper",
+        },
+    });
+    if (!response.ok) {
+        if (response.status === 404) {
+            // File not found - return null gracefully
+            return null;
+        }
+        throw new Error(`Failed to fetch file content: ${response.status} ${response.statusText}`);
+    }
+    const content = await response.text();
+    return {
+        path: filePath,
+        content,
+        size: new TextEncoder().encode(content).length,
+    };
+}
+/**
+ * Fetches multiple files from a repository.
+ * Handles 404s gracefully by skipping missing files.
+ *
+ * @param repoString Repository in "owner/repo" format
+ * @param branch Branch name
+ * @param filePaths Array of file paths to fetch
+ * @returns Array of successfully fetched files and array of not found paths
+ */
+export async function fetchMultipleFiles(repoString, branch, filePaths) {
+    const files = [];
+    const notFound = [];
+    // Fetch files in parallel for better performance
+    const results = await Promise.all(filePaths.map(async (filePath) => {
+        try {
+            const result = await fetchFileContent(repoString, branch, filePath);
+            return { filePath, result };
+        }
+        catch (error) {
+            // Log error but don't fail the whole batch
+            console.error(`Error fetching ${filePath}:`, error);
+            return { filePath, result: null };
+        }
+    }));
+    for (const { filePath, result } of results) {
+        if (result) {
+            files.push(result);
+        }
+        else {
+            notFound.push(filePath);
+        }
+    }
+    return { files, notFound };
+}
+/**
+ * Gets the current GitHub API rate limit status.
+ */
+export function getRateLimitStatus() {
+    return githubRateLimit.getStatusMessage();
+}
+/**
+ * Gets the raw rate limit info.
+ */
+export function getRateLimitInfo() {
+    return githubRateLimit.getInfo();
+}
+//# sourceMappingURL=github-fetcher.js.map

package/dist/services/github-fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-fetcher.js","sourceRoot":"","sources":["../../src/services/github-fetcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,uBAAuB,EACvB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAE5B,MAAM,eAAe,GAAG,wBAAwB,CAAC;AAwDjD,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;AAExD;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB;IAIhD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,yBAAyB,UAAU,2BAA2B,CAC/D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB;IACvB,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,gCAAgC;QACxC,YAAY,EAAE,kBAAkB;KACjC,CAAC;IAEF,4EAA4E;IAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACvC,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,aAAa,GAAG,UAAU,KAAK,EAAE,CAAC;IAC5C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE,gBAAgB,EAAE;KAC5B,CAAC,CAAC;IAEH,eAAe,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEpD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAChC,KAAa,EACb,IAAY;IAEZ,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;IAEtC,0CAA0C;IAC1C,IAAI,eAAe,CAAC,WAAW,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,oBAAoB,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC;QACH,qCAAqC;QACrC,MAAM,OAAO,GAAG,GAAG,eAAe,UAAU,KAAK,IAAI,IAAI,iBAAiB,CAAC;QAC3E,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAEhD,IAAI,YAAY,CAAC,EAAE,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,iCAAiC;QACjC,IAAI,YAAY,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAChC,MAAM,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,aAAa;QACb,MAAM,SAAS,GAAG,GAAG,eAAe,UAAU,KAAK,IAAI,IAAI,mBAAmB,CAAC;QAC/E,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;QAEpD,IAAI,cAAc,CAAC,EAAE,EAAE,CAAC;YACtB,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAClC,MAAM,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,8CAA8C;QAC9C,MAAM,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,oBAAoB;YACrC,KAAK,YAAY,uBAAuB;YACxC,KAAK,YAAY,mBAAmB,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,UAAU,EAAE,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACzG,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,YAAY,CACzB,KAAa,EACb,IAAY,EACZ,MAAc;IAEd,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;IAEtC,yCAAyC;IACzC,IAAI,eAAe,CAAC,WAAW,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,oBAAoB,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,eAAe,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,YAAY,eAAe,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,eAAe,UAAU,KAAK,IAAI,IAAI,cAAc,MAAM,cAAc,CAAC;IAExF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,uCAAuC;gBACvC,IAAI,eAAe,CAAC,WAAW,EAAE,EAAE,CAAC;oBAClC,MAAM,IAAI,oBAAoB,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC,CAAC;gBACjE,CAAC;gBACD,MAAM,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;YAChD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC9D,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAiC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,oBAAoB;YACrC,KAAK,YAAY,mBAAmB;YACpC,KAAK,YAAY,uBAAuB,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzE,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,GAAG,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,UAAoB;IAC/D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACzC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,KAAK,CAAC,GAAG,EAAE,CAAC;IACZ,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAC5B,KAAuB,EACvB,QAAgB,EAChB,UAAoB,EACpB,QAAgB;IAEhB,2CAA2C;IAC3C,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,0BAA0B;QAC1B,IACE,QAAQ;YACR,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC;YACrC,IAAI,CAAC,IAAI,KAAK,QAAQ,EACtB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,2BAA2B;QAC3B,MAAM,YAAY,GAAG,QAAQ;YAC3B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACd,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QACzC,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,IAAI,CAAC,CAAC,6DAA6D;IAC5E,CAAC,CAAC,CAAC;IAEH,gDAAgD;IAChD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAChD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,kDAAkD;IAClD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAiB;gBACzB,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChC,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,IAAI,CAAC,IAAI;aACtB,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7B,SAAS,EAAE,CAAC;YACZ,SAAS,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,8CAA8C;YAC9C,IAAI,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,OACE,UAAU;gBACV,CAAC,CAAC,QAAQ;oBACR,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC;oBAC/B,UAAU,KAAK,QAAQ,CAAC,EAC1B,CAAC;gBACD,mCAAmC;gBACnC,IAAI,QAAQ,IAAI,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM;oBAAE,MAAM;gBAC3D,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC5B,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE;gBACtB,IAAI,EAAE,eAAe,CAAC,UAAU,CAAC;gBACjC,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,2DAA2D;QAC3D,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;YACxC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,QAAQ;YAC1B,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC;gBAC/B,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC9C,UAAU,KAAK,QAAQ;YACzB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAExB,uCAAuC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACrD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,MAAM,SAAS,GAAG,CAAC,KAAqB,EAAE,EAAE;QAC1C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClB,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,SAAS,CAAC,SAAS,CAAC,CAAC;IAErB,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AACnD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAkB,EAClB,UAA4B,EAAE;IAE9B,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EACJ,IAAI,GAAG,EAAE,EACT,MAAM,EAAE,eAAe,EACvB,UAAU,GAAG,kBAAkB,EAC/B,QAAQ,GAAG,EAAE,GACd,GAAG,OAAO,CAAC;IAEZ,iCAAiC;IACjC,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC,MAAM,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAE3E,wCAAwC;IACxC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAExD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CACX,gFAAgF,CACjF,CAAC;IACJ,CAAC;IAED,yCAAyC;IACzC,MAAM,MAAM,GAAG,qBAAqB,CAClC,OAAO,CAAC,IAAI,EACZ,IAAI,EACJ,UAAU,EACV,QAAQ,CACT,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE;QACxB,MAAM;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC;AAcD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAkB,EAClB,MAAc,EACd,QAAgB;IAEhB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAEpD,kEAAkE;IAClE,MAAM,GAAG,GAAG,qCAAqC,KAAK,IAAI,IAAI,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;IAEvF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE;YACP,YAAY,EAAE,kBAAkB;SACjC;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,0CAA0C;YAC1C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,KAAK,CACb,iCAAiC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC1E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEtC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO;QACP,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;KAC/C,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAkB,EAClB,MAAc,EACd,SAAmB;IAKnB,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,iDAAiD;IACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,OAAO,CAAC,KAAK,CAAC,kBAAkB,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;YACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QAC3C,IAAI,MAAM,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,eAAe,CAAC,gBAAgB,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,eAAe,CAAC,OAAO,EAAE,CAAC;AACnC,CAAC"}

package/dist/services/search-index.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Search index service - Full-text search using MiniSearch.
+ *
+ * Indexes documentation content for fast searching.
+ * Index is built during docs indexing and stored in cache.
+ */
+import MiniSearch from "minisearch";
+/**
+ * Document to be indexed.
+ */
+export interface IndexableDocument {
+    /** File path (unique identifier) */
+    id: string;
+    /** Document title (from first H1 or filename) */
+    title: string;
+    /** All headings concatenated */
+    headings: string;
+    /** Full text content */
+    content: string;
+}
+/**
+ * Search result with snippet.
+ */
+export interface SearchResult {
+    /** File path */
+    path: string;
+    /** Document title */
+    title: string;
+    /** Matching excerpt with context */
+    snippet: string;
+    /** Relevance score (higher = more relevant) */
+    score: number;
+}
+/**
+ * Serialized index format for storage.
+ */
+export interface SerializedSearchIndex {
+    /** MiniSearch serialized index */
+    index: ReturnType<MiniSearch<IndexableDocument>["toJSON"]>;
+    /** Document metadata for generating snippets */
+    documents: Map<string, {
+        title: string;
+        content: string;
+    }>;
+    /** Version for future compatibility */
+    version: number;
+}
+/**
+ * Creates and manages a full-text search index for documentation.
+ */
+export declare class SearchIndex {
+    private miniSearch;
+    private documents;
+    constructor();
+    /**
+     * Adds a document to the index.
+     */
+    addDocument(doc: IndexableDocument): void;
+    /**
+     * Adds multiple documents to the index.
+     */
+    addDocuments(docs: IndexableDocument[]): void;
+    /**
+     * Searches the index and returns results with snippets.
+     */
+    search(query: string, limit?: number): SearchResult[];
+    /**
+     * Converts a MiniSearch result to our SearchResult format.
+     */
+    private toSearchResult;
+    /**
+     * Generates a snippet with the matching text highlighted.
+     */
+    private generateSnippet;
+    /**
+     * Returns the number of indexed documents.
+     */
+    get documentCount(): number;
+    /**
+     * Serializes the index for storage.
+     */
+    toJSON(): string;
+    /**
+     * Creates a SearchIndex from serialized JSON.
+     */
+    static fromJSON(json: string): SearchIndex;
+    /**
+     * Clears the index.
+     */
+    clear(): void;
+}
+/**
+ * Extracts a title from markdown content.
+ * Returns the first H1 heading, or undefined if none found.
+ */
+export declare function extractTitle(content: string): string | undefined;
+/**
+ * Extracts all headings from markdown content.
+ * Returns a concatenated string of all headings.
+ */
+export declare function extractHeadings(content: string): string;
+/**
+ * Creates an indexable document from file content.
+ */
+export declare function createIndexableDocument(path: string, content: string): IndexableDocument;
+//# sourceMappingURL=search-index.d.ts.map

package/dist/services/search-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index.d.ts","sourceRoot":"","sources":["../../src/services/search-index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,UAAgD,MAAM,YAAY,CAAC;AAE1E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oCAAoC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,kCAAkC;IAClC,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,gDAAgD;IAChD,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3D,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;CACjB;AASD;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,UAAU,CAAgC;IAClD,OAAO,CAAC,SAAS,CAAkD;;IAenE;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,iBAAiB,GAAG,IAAI;IAWzC;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,IAAI;IAM7C;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,YAAY,EAAE;IAazD;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmDvB;;OAEG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;OAEG;IACH,MAAM,IAAI,MAAM;IAchB;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IA6B1C;;OAEG;IACH,KAAK,IAAI,IAAI;CAId;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGhE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAUvD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GACd,iBAAiB,CAUnB"}