docs-agent 1.1.0

package/src/FileUtility.js

@@ -0,0 +1,130 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { validateGithubOrGitlabUrl } from './UrlValidator.js';
+
+// Default allowed directory - can be overridden via environment variable
+const DEFAULT_ALLOWED_DIRECTORY = process.env.ALLOWED_FILE_DIRECTORY || process.cwd();
+
+/**
+ * Safely resolve a file path to prevent directory traversal attacks
+ * @param {string} filePath - The file path to resolve
+ * @param {string} basePath - The base directory to resolve from
+ * @param {string} accessMode - The access mode ('api' or 'mcp')
+ * @returns {Promise<string>} - The safely resolved absolute path
+ * @throws {Error} - If the path is outside the allowed directory
+ */
+async function safeResolvePath(filePath, basePath, accessMode = 'mcp') {
+    // For MCP mode, allow access to any file in the system (no restrictions)
+    if (accessMode === 'mcp') {
+        return path.resolve(filePath);
+    }
+    
+    // For API mode, restrict to basePath only
+    // Normalize the path to remove any ".." segments
+    const normalizedPath = path.resolve(basePath, filePath);
+    
+    // Check if the normalized path is within the base directory before resolving symlinks
+    if (!normalizedPath.startsWith(path.resolve(basePath))) {
+        throw new Error(`Path traversal detected: ${filePath} resolves outside allowed directory (${basePath})`);
+    }
+    
+    // Try to resolve any symbolic links, but don't fail if file doesn't exist
+    try {
+        const realPath = await fs.realpath(normalizedPath);
+        // Double-check after resolving symlinks
+        if (!realPath.startsWith(path.resolve(basePath))) {
+            throw new Error(`Path traversal detected: ${filePath} resolves outside allowed directory (${basePath})`);
+        }
+        return realPath;
+    } catch (error) {
+        // If realpath fails (file doesn't exist), return the normalized path
+        // The path traversal check above already validated it's within bounds
+        return normalizedPath;
+    }
+}
+
+/**
+ * Read the content of the file
+ * @param {string} filePath - The path to the file, supports absolute paths, remote URLs and relative paths
+ * @param {string} accessMode - The access mode ('api' or 'mcp')
+ * @returns {Promise<string>} - The content of the file
+ */
+async function readFile(filePath, accessMode = 'mcp'){
+    if(isRemoteFileUrl(filePath)){
+        const safeUrl = validateGithubOrGitlabUrl(filePath);
+        const response = await fetch(safeUrl);
+        return await response.text();
+    }
+    
+    // Determine base path based on access mode
+    let basePath;
+    if (accessMode === 'api') {
+        basePath = path.join(process.cwd(), 'public');
+    } else {
+        // MCP mode - no restrictions, use current working directory as base
+        basePath = process.cwd();
+    }
+    
+    // Validate paths to prevent unauthorized file access
+    filePath = await safeResolvePath(filePath, basePath, accessMode);
+    
+    return await fs.readFile(filePath, 'utf8');
+}
+
+/**
+ * Write the content to the file
+ * @param {string} filePath - The path to the file, supports absolute paths, remote URLs and relative paths
+ * @param {string} content - The content to write to the file
+ * @param {string} accessMode - The access mode ('api' or 'mcp')
+ * @returns {Promise<void>} - The promise that resolves when the file is written
+ */
+async function writeFile(filePath, content, accessMode = 'mcp'){
+    if(isRemoteFileUrl(filePath)){
+        throw new Error("Cannot write to remote file");
+    }
+    
+    // Determine base path based on access mode
+    let basePath;
+    if (accessMode === 'api') {
+        basePath = path.join(process.cwd(), 'public');
+    } else {
+        // MCP mode - no restrictions, use current working directory as base
+        basePath = process.cwd();
+    }
+    
+    // Validate paths to prevent unauthorized file access
+    filePath = await safeResolvePath(filePath, basePath, accessMode);
+    
+    await fs.mkdir(path.dirname(filePath), { recursive: true }); // Create directory if it does not exist
+    return await fs.writeFile(filePath, content);
+}
+
+/**
+ * Check if the file path is absolute
+ * @param {string} filePath - The path to the file
+ * @returns {boolean} - True if the file path is absolute, false otherwise
+ */
+function isAbsoluteFilePath(filePath){
+    return path.isAbsolute(filePath);
+}
+
+/**
+ * Check if the file path is a remote URL
+ * @param {string} url - The URL to check
+ * @returns {boolean} - True if the URL is a remote URL, false otherwise
+ */
+function isRemoteFileUrl(url){
+    return url.startsWith("http") || url.startsWith("https");
+}
+
+/**
+ * Convert a relative file path to an absolute file path
+ * @param {string} filePath - The relative file path
+ * @param {string} basePath - The base path to resolve the relative file path from
+ * @returns {string} - The absolute file path
+ */
+function toAbsoluteFilePath(filePath, basePath){
+    return path.resolve(basePath, filePath);
+}
+
+export { readFile, writeFile, isAbsoluteFilePath, isRemoteFileUrl, toAbsoluteFilePath, safeResolvePath };

package/src/GitHubApi.js

@@ -0,0 +1,337 @@
+/**
+ * GitHub API
+ */
+
+import { Octokit } from "@octokit/core";
+import { retry } from "@octokit/plugin-retry";
+import { throttling } from "@octokit/plugin-throttling";
+
+class GitHubApi {
+  constructor() {
+    this.MyOctokit = Octokit.plugin(retry, throttling);
+    this.cache = new Map();
+  }
+
+  async getRepoInfo(repository) {
+    const { owner, repo } = this._parseRepoNameAndOwner(repository);
+    if(!owner || !repo){
+      throw new Error(`Invalid repository string: ${repository}`);
+    }
+    // Check cache first
+    if (this.cache.has(`repo:${owner}/${repo}`)) {
+      console.log(`Cache hit for repository: ${owner}/${repo}`);
+      return this.cache.get(`repo:${owner}/${repo}`);
+    }
+    try {
+      const octokit = this._createOctokit();
+      const repoResponse = await octokit.request("GET /repos/{owner}/{repo}", {
+        owner,
+        repo
+      });
+      const repoInfo = repoResponse.data;
+      // Cache the repo info
+      this.cache.set(`repo:${owner}/${repo}`, repoInfo);
+      console.log(`Repository info: ${JSON.stringify(repoInfo, null, 2)}`);
+      return repoInfo;
+    } catch (error) {
+      console.error("Error getting repository info for "+repository+": "+error.message);
+      return { owner, repo };
+    }
+  }
+
+  /**
+   * Search for code in GitHub repositories
+   * Limitations:
+   *    - Only default branch is searched
+   *    - Symbol search is not supported by GitHub API
+   * @param {string} query - The search query e.g. "keyword" | "repo:owner/repo" (symbol search not supported)
+   * @param {object} context - Search context and options
+   * @param {string} context.repository - GitHub repository URL e.g. "https://github.com/owner/repo" or "owner/repo"
+   * @param {number} context.scoreThreshold - Minimum relevance score (default: 0, maximum: 1)
+   * @param {number} context.maxResults - Maximum number of results (default: 10)
+   * @param {boolean} context.cache - Whether to use caching (default: true)
+   * @returns {Promise<Array<Object>>} searchResults - Array of search results with search results containing filepath and metadata
+   * @returns {Promise<string>} searchResults[].repository - Full repository path, supports both remote or local repository paths
+   * @returns {Promise<string>} searchResults[].path - Relative path to the file from the repository root
+   * @returns {Promise<number>} searchResults[].score - Relevance score of the search result (0-1)
+   * @returns {Promise<string>} searchResults[].sha - SHA of the search result (if available e.g. dsf2w32.. or main)
+   * @returns {Promise<string>} searchResults[].snippet - Snippet of the search result (not available as of now)
+   * @example
+   * const searchResults = await githubApi.searchCode('function main', {
+   *   repository: 'https://github.com/owner/repo',
+   *   scoreThreshold: 0,
+   *   maxResults: 10,
+   *   cache: true
+   * });
+   * console.log(searchResults);
+   * // [
+   * //   {
+   * //     repository: 'https://github.com/owner/repo',
+   * //     path: 'path/to/file',
+   * //     score: 0.8,
+   * //     snippet: 'function main() { ... }'
+   * //   }
+   * // ]
+   */
+  async searchCode(query, context = {}) {
+    const {
+      repository,
+      githubToken = process.env.GITHUB_TOKEN,
+      scoreThreshold = 0,
+      maxResults = 10,
+      cache = true
+    } = context;
+
+    if(query?.includes('symbol:')) {
+        // Symbol search is not supported by GitHub API
+        // Remove the symbol prefix from the query
+        query = query.replace('symbol:', '');
+        console.log(`Symbol search is not supported by GitHub API, removing the symbol prefix from the query: "${query}"`);
+    }
+
+    let owner, repo;
+    if (repository) {
+      ({ owner, repo } = this._parseRepoNameAndOwner(repository));
+      if(owner && repo && !query.includes(`repo:`)) {
+        query = `${query} repo:${owner}/${repo}`;
+      }
+    }
+
+    let repoInfo;
+    if(repository){
+      repoInfo = await this.getRepoInfo(repository);
+    }
+
+    if(repoInfo?.fork){
+      query = `${query} fork:true`;
+    }
+
+    // Build cache key
+    const cacheKey = `search:${query}|${scoreThreshold}|${maxResults}`;
+    
+    // Check cache first
+    if (cache && this.cache.has(cacheKey)) {
+      console.log(`Cache hit for query: "${query}"`);
+      return this.cache.get(cacheKey);
+    }
+
+    console.log(`Searching GitHub for: "${query}" (max ${maxResults} results, score threshold: ${scoreThreshold})`);
+
+    try {
+      const octokit = this._createOctokit(githubToken);
+
+      // Search for code
+      const searchResponse = await octokit.request("GET /search/code", {
+        q: query,
+        per_page: Math.min(maxResults, 30), // GitHub API limit is 100
+        sort: "best-match"
+      });
+
+      const searchResults = searchResponse?.data?.items || [];
+      console.log(`Found ${searchResults.length} initial search results`);
+
+      // Process results and fetch file contents
+      const results = searchResults?.map(item => ({
+        repository: item.repository.html_url,
+        path: item.path,
+        score: item.score,
+        snippet: item.text_matches?.[0]?.fragment,
+        sha: item.sha || null,
+        filepath: this._convertToAbsoluteFilePath(item.path, item.repository.html_url, repoInfo?.default_branch || "main")
+      }))
+
+      console.log(`Returning ${results.length} GitHub code search results`);
+
+      // Cache the results
+      if (cache) {
+        this.cache.set(cacheKey, results);
+      }
+
+      return results;
+
+    } catch (error) {
+      this._handleError(error, "GitHub search");
+    }
+  }
+
+  /**
+   * Get file content from a GitHub repository
+   * @param {string} repository - Repository GitHub URL e.g. "https://github.com/owner/repo" or "owner/repo"
+   * @param {string} path - File path in the repository
+   * @param {string} [sha] - Optional commit SHA (default: uses default branch)
+   * @param {object} [context={}] - Additional context
+   * @param {string} [context.githubToken=process.env.GITHUB_TOKEN] - GitHub personal access token
+   * @param {boolean} [context.cache=true] - Whether to use caching (default: true)
+   * @returns {Promise<string>} File content as string
+   */
+  async getFileContent(repository, path, sha = null, context = {}) {
+    const {
+      githubToken = process.env.GITHUB_TOKEN,
+      cache = true
+    } = context;
+
+    const { owner, repo } = this._parseRepoNameAndOwner(repository);
+
+    // Build cache key - don't include SHA in cache key since we're not using it reliably
+    const cacheKey = `file:${owner}/${repo}/${path}`;
+    
+    // Check cache first
+    if (cache && this.cache.has(cacheKey)) {
+      console.log(`Cache hit for file: ${owner}/${repo}/${path}`);
+      return this.cache.get(cacheKey);
+    }
+
+    console.log(`Fetching file content: ${owner}/${repo}/${path}`);
+
+    try {
+      const octokit = this._createOctokit(githubToken);
+
+      // Always use default branch for file content - SHA from search results is unreliable
+      const requestParams = {
+        owner,
+        repo,
+        path
+      };
+
+      // Only use SHA if it's a valid branch name or tag, not a commit SHA
+      if (sha && !sha.match(/^[a-f0-9]{40}$/)) {
+        // This looks like a branch name or tag, not a commit SHA
+        requestParams.ref = sha;
+      }
+
+      const contentResponse = await octokit.request("GET /repos/{owner}/{repo}/contents/{path}", requestParams);
+
+      // Decode Base64 content
+      const content = Buffer.from(contentResponse?.data?.content, 'base64')?.toString('utf-8');
+
+      // Cache the content
+      if (cache) {
+        this.cache.set(cacheKey, content);
+      }
+
+      console.log(`Successfully fetched content for: ${owner}/${repo}/${path}`);
+      return content;
+
+    } catch (error) {
+      this._handleError(error, `File content fetch for ${owner}/${repo}/${path}`);
+    }
+  }
+
+  /**
+   * Clear the cache
+   */
+  clearCache() {
+    this.cache.clear();
+    console.log("GitHub API cache cleared");
+  }
+
+  /**
+   * Get cache statistics
+   * @returns {object} Cache statistics
+   */
+  getCacheStats() {
+    return {
+      size: this.cache.size,
+      keys: Array.from(this.cache.keys())
+    };
+  }
+
+  /**
+   * Convert search results to absolute file path
+   * @param {Array<Object>} searchResults - Array of search results
+   * @returns {Array<Object>} searchResults - Array of search results with absolute file path
+   */
+  _convertToAbsoluteFilePath(path, repository, sha="main") {
+    const { owner, repo } = this._parseRepoNameAndOwner(repository);
+    return `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${sha}/${path}`;
+  }
+
+  /**
+   * Initialize Octokit instance with authentication and plugins
+   * @param {string} githubToken - GitHub personal access token
+   * @returns {Octokit} Configured Octokit instance
+   */
+  _createOctokit(githubToken = process.env.GITHUB_TOKEN) {
+    if (!githubToken) {
+      throw new Error("GitHub token is required. Set GITHUB_TOKEN environment variable or pass githubToken parameter.");
+    }
+
+    return new this.MyOctokit({
+      auth: githubToken,
+      userAgent: "docs-agent/1.0.0",
+      retry: { 
+        maxRetries: 3, 
+        doNotRetry: ["429"] 
+      },
+      throttle: {
+        onRateLimit: (retryAfter, options, octo, retryCount) => {
+          console.log(`Rate limit hit, retry ${retryCount}/3 after ${retryAfter}s`);
+          return retryCount <= 3;
+        },
+        onSecondaryRateLimit: (retryAfter, options, octo, retryCount) => {
+          console.log(`Secondary rate limit hit, retry ${retryCount}/3 after ${retryAfter}s`);
+          return retryCount <= 3;
+        },
+        onAbuseLimit: (retryAfter, options, octo, retryCount) => {
+          console.log(`Abuse limit hit, retry ${retryCount}/3 after ${retryAfter}s`);
+          return retryCount <= 3;
+        },
+      },
+    });
+  }
+
+  /**
+   * Handle GitHub API errors with specific error messages
+   * @param {Error} error - The error to handle
+   * @param {string} operation - The operation that failed
+   * @throws {Error} Specific error message
+   */
+  _handleError(error, operation = "GitHub API operation") {
+    console.error(`${operation} failed:`, error.message);
+    
+    if (error.status === 401) {
+      throw new Error("Invalid GitHub token. Please check your authentication.");
+    } else if (error.status === 403) {
+      throw new Error("GitHub API access denied. Check your token permissions.");
+    } else if (error.status === 429) {
+      throw new Error("GitHub rate limit exceeded. Please try again later.");
+    } else {
+      throw new Error(`${operation} failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Parses owner and repo from repository string or http url
+   * @param {string} repository - Repository string in format "owner/repo" or a GitHub URL
+   * @returns {Object} Object with owner and repo properties
+   * @example
+   * const { owner, repo } = githubApi._parseRepoNameAndOwner('owner/repo');
+   * console.log(owner); // 'owner'
+   * console.log(repo); // 'repo'
+   * const { owner, repo } = githubApi._parseRepoNameAndOwner('https://github.com/owner/repo');
+   * console.log(owner); // 'owner'
+   * console.log(repo); // 'repo'
+   */
+  _parseRepoNameAndOwner(repository) {
+    // Check if it's a URL
+    let owner, repo;
+    try {
+      const url = new URL(repository);
+      // Handles URLs like https://github.com/owner/repo or http://github.com/owner/repo
+      const parts = url?.pathname?.replace(/^\/+|\/+$/g, '')?.split('/');
+      owner = parts?.[0];
+      repo = parts?.[1];
+    } catch (e) {
+      // Not a valid URL, assume "owner/repo" shorthand
+      const parts = repository.split('/');
+      owner = parts?.[0];
+      repo = parts?.[1];
+    }
+    if (!owner || !repo) {
+      throw new Error(`Invalid repository string: ${repository}`);
+    }
+    return { owner, repo };
+  }
+}
+
+export default GitHubApi;