git-ripper 1.3.6 → 1.4.0

package/README.md

@@ -31,6 +31,7 @@ Have you ever needed just a single component from a massive repository? Or wante
 - **Directory Structure**: Preserves complete folder structure
 - **Custom Output**: Specify your preferred output directory
 - **Branch Support**: Works with any branch, not just the default one
+- **Archive Export**: Create ZIP or TAR archives of downloaded content
 - **Simple Interface**: Clean, intuitive command-line experience
 - **Lightweight**: Minimal dependencies and fast execution
 - **No Authentication**: Works with public repositories without requiring credentials
@@ -67,11 +68,26 @@ git-ripper https://github.com/username/repository/tree/branch/folder
 git-ripper https://github.com/username/repository/tree/branch/folder -o ./my-output-folder
 ```
 
+### Creating ZIP Archive
+
+```bash
+git-ripper https://github.com/username/repository/tree/branch/folder --zip
+```
+
+### Creating TAR Archive with Custom Name
+
+```bash
+git-ripper https://github.com/username/repository/tree/branch/folder --tar="my-archive.tar"
+```
+
 ### Command Line Options
 
 | Option | Description | Default |
 |--------|-------------|---------|
 | `-o, --output <directory>` | Specify output directory | Current directory |
+| `--zip [filename]` | Create ZIP archive of downloaded content | - |
+| `--tar [filename]` | Create TAR archive of downloaded content | - |
+| `--compression-level <level>` | Set compression level (1-9) | 6 |
 | `-V, --version` | Show version number | - |
 | `-h, --help` | Show help | - |
 
@@ -105,6 +121,16 @@ git-ripper https://github.com/nodejs/node/tree/main/doc -o ./node-docs
 git-ripper https://github.com/tailwindlabs/tailwindcss/tree/master/src/components -o ./tailwind-components
 ```
 
+### Download and Create Archive
+
+```bash
+# Download React DOM package and create a ZIP archive
+git-ripper https://github.com/facebook/react/tree/main/packages/react-dom --zip
+
+# Extract VS Code build configuration with maximum compression
+git-ripper https://github.com/microsoft/vscode/tree/main/build --tar --compression-level=9
+```
+
 ## How It Works
 
 Git-ripper operates in four stages:
@@ -112,7 +138,7 @@ Git-ripper operates in four stages:
 1. **URL Parsing**: Extracts repository owner, name, branch, and target folder path
 2. **API Request**: Uses GitHub's API to fetch the folder structure
 3. **Content Download**: Retrieves each file individually while maintaining directory structure
-4. **Local Storage**: Saves files to your specified output directory
+4. **Local Storage or Archiving**: Saves files to your specified output directory or creates an archive
 
 ## Configuration
 
@@ -160,6 +186,7 @@ See the [open issues](https://github.com/sairajB/git-ripper/issues) for a list o
 
 ## Roadmap
 
+- [x] Add archive export options (ZIP/TAR)
 - [ ] Add GitHub token authentication
 - [ ] Support for GitLab and Bitbucket repositories
 - [ ] Download from specific commits or tags

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "git-ripper",
-    "version": "1.3.6",
+    "version": "1.4.0",
     "description": "CLI tool that lets you download specific folders from GitHub repositories without cloning the entire repo.",
     "main": "src/index.js",
     "type": "module",
@@ -30,6 +30,7 @@
     "author": "sairajb",
     "license": "MIT",
     "dependencies": {
+        "archiver": "^6.0.1",
         "axios": "^1.6.7",
         "chalk": "^5.3.0",
         "cli-progress": "^3.12.0",

package/src/archiver.js

@@ -0,0 +1,186 @@
+import fs from 'fs';
+import path from 'path';
+import archiver from 'archiver';
+import chalk from 'chalk';
+
+/**
+ * Validates the output path for an archive file
+ * @param {string} outputPath - Path where the archive should be saved
+ * @returns {boolean} - True if the path is valid, throws an error otherwise
+ * @throws {Error} - If the output path is invalid
+ */
+const validateArchivePath = (outputPath) => {
+  // Check if path is provided
+  if (!outputPath) {
+    throw new Error('Output path is required');
+  }
+  
+  // Check if the output directory exists or can be created
+  const outputDir = path.dirname(outputPath);
+  try {
+    if (!fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir, { recursive: true });
+    }
+    
+    // Check if the directory is writable
+    fs.accessSync(outputDir, fs.constants.W_OK);
+    
+    // Check if file already exists and is writable
+    if (fs.existsSync(outputPath)) {
+      fs.accessSync(outputPath, fs.constants.W_OK);
+      // File exists and is writable, so we'll overwrite it
+      console.warn(chalk.yellow(`Warning: File ${outputPath} already exists and will be overwritten`));
+    }
+    
+    return true;
+  } catch (error) {
+    if (error.code === 'EACCES') {
+      throw new Error(`Permission denied: Cannot write to ${outputPath}`);
+    }
+    throw new Error(`Invalid output path: ${error.message}`);
+  }
+};
+
+/**
+ * Creates an archive (zip or tar) from a directory
+ * 
+ * @param {string} sourceDir - Source directory to archive
+ * @param {string} outputPath - Path where the archive should be saved
+ * @param {object} options - Archive options
+ * @param {string} options.format - Archive format ('zip' or 'tar')
+ * @param {number} options.compressionLevel - Compression level (0-9, default: 6)
+ * @returns {Promise<string>} - Path to the created archive
+ */
+export const createArchive = (sourceDir, outputPath, options = {}) => {
+  return new Promise((resolve, reject) => {
+    try {
+      const { format = 'zip', compressionLevel = 6 } = options;
+      
+      // Validate source directory
+      if (!fs.existsSync(sourceDir)) {
+        return reject(new Error(`Source directory does not exist: ${sourceDir}`));
+      }
+      
+      const stats = fs.statSync(sourceDir);
+      if (!stats.isDirectory()) {
+        return reject(new Error(`Source path is not a directory: ${sourceDir}`));
+      }
+      
+      // Validate output path
+      validateArchivePath(outputPath);
+      
+      // Ensure the output directory exists
+      const outputDir = path.dirname(outputPath);
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+      
+      // Create output stream
+      const output = fs.createWriteStream(outputPath);
+      let archive;
+      
+      // Create the appropriate archive type
+      if (format === 'zip') {
+        archive = archiver('zip', {
+          zlib: { level: compressionLevel }
+        });
+      } else if (format === 'tar') {
+        archive = archiver('tar');
+        // Use gzip compression for tar if compressionLevel > 0
+        if (compressionLevel > 0) {
+          archive = archiver('tar', {
+            gzip: true,
+            gzipOptions: { level: compressionLevel }
+          });
+        }
+      } else {
+        return reject(new Error(`Unsupported archive format: ${format}`));
+      }
+      
+      // Listen for archive events
+      output.on('close', () => {
+        const size = archive.pointer();
+        console.log(chalk.green(`✓ Archive created: ${outputPath} (${(size / 1024 / 1024).toFixed(2)} MB)`));
+        resolve(outputPath);
+      });
+      
+      archive.on('error', (err) => {
+        reject(err);
+      });
+      
+      archive.on('warning', (err) => {
+        if (err.code === 'ENOENT') {
+          console.warn(chalk.yellow(`Warning: ${err.message}`));
+        } else {
+          reject(err);
+        }
+      });
+      
+      // Pipe archive data to the output file
+      archive.pipe(output);
+      
+      // Add the directory contents to the archive
+      archive.directory(sourceDir, false);
+      
+      // Finalize the archive
+      archive.finalize();
+    } catch (error) {
+      reject(error);
+    }
+  });
+};
+
+/**
+ * Downloads folder contents and creates an archive
+ * 
+ * @param {object} repoInfo - Repository information object
+ * @param {string} outputDir - Directory where files should be downloaded
+ * @param {string} archiveFormat - Archive format ('zip' or 'tar')
+ * @param {string} archiveName - Custom name for the archive file
+ * @param {number} compressionLevel - Compression level (0-9)
+ * @returns {Promise<string>} - Path to the created archive
+ */
+export const downloadAndArchive = async (repoInfo, outputDir, archiveFormat = 'zip', archiveName = null, compressionLevel = 6) => {
+  const { downloadFolder } = await import('./downloader.js');
+  
+  console.log(chalk.cyan(`Downloading folder and preparing to create ${archiveFormat.toUpperCase()} archive...`));
+  
+  // Create a temporary directory for the download
+  const tempDir = path.join(outputDir, `.temp-${Date.now()}`);
+  fs.mkdirSync(tempDir, { recursive: true });
+  
+  try {
+    // Download the folder contents
+    await downloadFolder(repoInfo, tempDir);
+    
+    // Determine archive filename
+    let archiveFileName = archiveName;
+    if (!archiveFileName) {
+      const { owner, repo, folderPath } = repoInfo;
+      const folderName = folderPath ? folderPath.split('/').pop() : repo;
+      archiveFileName = `${folderName || repo}-${owner}`;
+    }
+    
+    // Add extension if not present
+    if (!archiveFileName.endsWith(`.${archiveFormat}`)) {
+      archiveFileName += `.${archiveFormat}`;
+    }
+    
+    const archivePath = path.join(outputDir, archiveFileName);
+    
+    // Create the archive
+    console.log(chalk.cyan(`Creating ${archiveFormat.toUpperCase()} archive...`));
+    await createArchive(tempDir, archivePath, { format: archiveFormat, compressionLevel });
+    
+    return archivePath;
+  } catch (error) {
+    throw new Error(`Failed to create archive: ${error.message}`);
+  } finally {
+    // Clean up temporary directory
+    try {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    } catch (err) {
+      console.warn(chalk.yellow(`Warning: Failed to clean up temporary directory: ${err.message}`));
+    }
+  }
+};

package/src/downloader.js

@@ -9,7 +9,8 @@ import chalk from "chalk";
 import prettyBytes from "pretty-bytes";
 
 // Set concurrency limit (adjustable based on network performance)
-const limit = pLimit(500);
+// Reduced from 500 to 5 to prevent GitHub API rate limiting
+const limit = pLimit(5);
 
 // Ensure __dirname and __filename are available in ESM
 const __filename = fileURLToPath(import.meta.url);
@@ -49,13 +50,42 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
 
   try {
     const response = await axios.get(apiUrl);
+    
+    // Check if GitHub API returned truncated results
+    if (response.data.truncated) {
+      console.warn(chalk.yellow(
+        `Warning: The repository is too large and some files may be missing. ` +
+        `Consider using git clone for complete repositories.`
+      ));
+    }
+    
     return response.data.tree.filter((item) => item.path.startsWith(folderPath));
   } catch (error) {
-    if (error.response && error.response.status === 404) {
-      console.error(`Repository, branch, or folder not found: ${owner}/${repo}/${branch}/${folderPath}`);
-      return [];
+    if (error.response) {
+      // Handle specific HTTP error codes
+      switch(error.response.status) {
+        case 403:
+          if (error.response.headers['x-ratelimit-remaining'] === '0') {
+            console.error(chalk.red(
+              `GitHub API rate limit exceeded. Please wait until ${
+                new Date(parseInt(error.response.headers['x-ratelimit-reset']) * 1000).toLocaleTimeString()
+              } or add a GitHub token (feature coming soon).`
+            ));
+          } else {
+            console.error(chalk.red(`Access forbidden: ${error.response.data.message || 'Unknown reason'}`));
+          }
+          break;
+        case 404:
+          console.error(chalk.red(`Repository, branch, or folder not found: ${owner}/${repo}/${branch}/${folderPath}`));
+          break;
+        default:
+          console.error(chalk.red(`API error (${error.response.status}): ${error.response.data.message || error.message}`));
+      }
+    } else if (error.request) {
+      console.error(chalk.red(`Network error: No response received from GitHub. Please check your internet connection.`));
+    } else {
+      console.error(chalk.red(`Error preparing request: ${error.message}`));
     }
-    console.error(`Failed to fetch folder contents: ${error.message}`);
     return [];
   }
 };
@@ -74,18 +104,61 @@ const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
 
   try {
     const response = await axios.get(url, { responseType: "arraybuffer" });
-    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
-    fs.writeFileSync(outputPath, Buffer.from(response.data));
+    
+    // Ensure the directory exists
+    try {
+      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    } catch (dirError) {
+      return { 
+        filePath, 
+        success: false, 
+        error: `Failed to create directory: ${dirError.message}`,
+        size: 0
+      };
+    }
+    
+    // Write the file
+    try {
+      fs.writeFileSync(outputPath, Buffer.from(response.data));
+    } catch (fileError) {
+      return { 
+        filePath, 
+        success: false, 
+        error: `Failed to write file: ${fileError.message}`,
+        size: 0
+      };
+    }
+    
     return { 
       filePath, 
       success: true,
       size: response.data.length
     };
   } catch (error) {
+    // More detailed error handling for network requests
+    let errorMessage = error.message;
+    
+    if (error.response) {
+      // The request was made and the server responded with an error status
+      switch (error.response.status) {
+        case 403:
+          errorMessage = "Access forbidden (possibly rate limited)";
+          break;
+        case 404:
+          errorMessage = "File not found";
+          break;
+        default:
+          errorMessage = `HTTP error ${error.response.status}`;
+      }
+    } else if (error.request) {
+      // The request was made but no response was received
+      errorMessage = "No response from server";
+    }
+    
     return { 
       filePath, 
       success: false, 
-      error: error.message,
+      error: errorMessage,
       size: 0
     };
   }
@@ -160,9 +233,16 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
       return;
     }
 
+    // Filter for blob type (files)
     const files = contents.filter(item => item.type === "blob");
     const totalFiles = files.length;
     
+    if (totalFiles === 0) {
+      console.log(chalk.yellow(`No files found in ${folderPath || 'repository root'} (only directories)`));
+      console.log(chalk.green(`Folder cloned successfully!`));
+      return;
+    }
+    
     console.log(chalk.cyan(`Downloading ${totalFiles} files from ${chalk.white(owner + '/' + repo)}...`));
     
     // Simplified progress bar setup
@@ -177,6 +257,7 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
     // Track download metrics
     let downloadedSize = 0;
     const startTime = Date.now();
+    let failedFiles = [];
     
     // Start progress bar
     progressBar.start(totalFiles, 0, {
@@ -200,6 +281,12 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
           // Update progress metrics
           if (result.success) {
             downloadedSize += (result.size || 0);
+          } else {
+            // Track failed files for reporting
+            failedFiles.push({
+              path: item.path,
+              error: result.error
+            });
           }
           
           // Update progress bar with current metrics
@@ -209,12 +296,18 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
           
           return result;
         } catch (error) {
+          failedFiles.push({
+            path: item.path,
+            error: error.message
+          });
+          
+          progressBar.increment(1, { downloadedSize });
           return { filePath: item.path, success: false, error: error.message, size: 0 };
         }
       });
     });
 
-    // Execute downloads in parallel
+    // Execute downloads in parallel with controlled concurrency
     const results = await Promise.all(fileDownloadPromises);
     progressBar.stop();
     
@@ -222,10 +315,20 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
 
     // Count successful and failed downloads
     const succeeded = results.filter((r) => r.success).length;
-    const failed = results.filter((r) => !r.success).length;
+    const failed = failedFiles.length;
 
     if (failed > 0) {
       console.log(chalk.yellow(`Downloaded ${succeeded} files successfully, ${failed} files failed`));
+      
+      // Show detailed errors if there aren't too many
+      if (failed <= 5) {
+        console.log(chalk.yellow('Failed files:'));
+        failedFiles.forEach(file => {
+          console.log(chalk.yellow(`  - ${file.path}: ${file.error}`));
+        });
+      } else {
+        console.log(chalk.yellow(`${failed} files failed to download. Check your connection or repository access.`));
+      }
     } else {
       console.log(chalk.green(` All ${succeeded} files downloaded successfully!`));
     }

package/src/index.js

@@ -1,23 +1,103 @@
 import { program } from 'commander';
 import { parseGitHubUrl } from './parser.js';
 import { downloadFolder } from './downloader.js';
+import { downloadAndArchive } from './archiver.js';
+import { fileURLToPath } from 'url';
+import { dirname, join, resolve } from 'path';
+import fs from 'fs';
+
+// Get package.json for version
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packagePath = join(__dirname, '..', 'package.json');
+const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8'));
+
+/**
+ * Validates and ensures the output directory exists
+ * @param {string} outputDir - The directory path to validate
+ * @returns {string} - The resolved directory path
+ * @throws {Error} - If the directory is invalid or cannot be created
+ */
+const validateOutputDirectory = (outputDir) => {
+  if (!outputDir) {
+    throw new Error('Output directory is required');
+  }
+
+  // Resolve to absolute path
+  const resolvedDir = resolve(outputDir);
+  
+  try {
+    // Check if directory exists, if not try to create it
+    if (!fs.existsSync(resolvedDir)) {
+      fs.mkdirSync(resolvedDir, { recursive: true });
+    } else {
+      // Check if it's actually a directory
+      const stats = fs.statSync(resolvedDir);
+      if (!stats.isDirectory()) {
+        throw new Error(`Output path exists but is not a directory: ${outputDir}`);
+      }
+    }
+    
+    // Check if the directory is writable
+    fs.accessSync(resolvedDir, fs.constants.W_OK);
+    
+    return resolvedDir;
+  } catch (error) {
+    if (error.code === 'EACCES') {
+      throw new Error(`Permission denied: Cannot write to ${outputDir}`);
+    }
+    throw new Error(`Invalid output directory: ${error.message}`);
+  }
+};
 
 const initializeCLI = () => {
   program
-    .version('1.3.6')
+    .version(packageJson.version)
     .description('Clone specific folders from GitHub repositories')
     .argument('<url>', 'GitHub URL of the folder to clone')
     .option('-o, --output <directory>', 'Output directory', process.cwd())
+    .option('--zip [filename]', 'Create ZIP archive of downloaded files')
+    .option('--tar [filename]', 'Create TAR archive of downloaded files')
+    .option('--compression-level <level>', 'Compression level (1-9)', '6')
     .action(async (url, options) => {
       try {
         console.log(`Parsing URL: ${url}`);
         const parsedUrl = parseGitHubUrl(url);
-        console.log(`Parsed URL:`, parsedUrl);
+        
+        // Validate options
+        if (options.compressionLevel) {
+          const level = parseInt(options.compressionLevel, 10);
+          if (isNaN(level) || level < 1 || level > 9) {
+            throw new Error('Compression level must be a number between 1 and 9');
+          }
+        }
 
-        console.log(`Downloading folder to: ${options.output}`);
-        await downloadFolder(parsedUrl, options.output);
+        if (options.zip && options.tar) {
+          throw new Error('Cannot specify both --zip and --tar options at the same time');
+        }
+        
+        // Validate output directory
+        try {
+          options.output = validateOutputDirectory(options.output);
+        } catch (dirError) {
+          throw new Error(`Output directory error: ${dirError.message}`);
+        }
+        
+        // Handle archive options
+        const archiveFormat = options.zip ? 'zip' : options.tar ? 'tar' : null;
+        const archiveName = typeof options.zip === 'string' ? options.zip : 
+                           typeof options.tar === 'string' ? options.tar : null;
+        const compressionLevel = parseInt(options.compressionLevel, 10) || 6;
+        
+        if (archiveFormat) {
+          console.log(`Creating ${archiveFormat.toUpperCase()} archive...`);
+          await downloadAndArchive(parsedUrl, options.output, archiveFormat, archiveName, compressionLevel);
+        } else {
+          console.log(`Downloading folder to: ${options.output}`);
+          await downloadFolder(parsedUrl, options.output);
+        }
         
-        console.log('Folder cloned successfully!');
+        console.log('Operation completed successfully!');
       } catch (error) {
         console.error('Error:', error.message);
         process.exit(1);
@@ -32,5 +112,4 @@ if (import.meta.url === `file://${process.argv[1]}`) {
   initializeCLI();
 }
 
-// ✅ Fix the incorrect export
 export { initializeCLI, downloadFolder };

package/src/parser.js

@@ -1,8 +1,27 @@
 export function parseGitHubUrl(url) {
-    const urlParts = url.split('/');
-    const owner = urlParts[3];
-    const repo = urlParts[4];
-    const branch = urlParts[6] || 'main';
-    const folderPath = urlParts.slice(7).join('/');
+    // Validate the URL format
+    if (!url || typeof url !== 'string') {
+        throw new Error('Invalid URL: URL must be a non-empty string');
+    }
+
+    // Validate if it's a GitHub URL
+    const githubUrlPattern = /^https?:\/\/(?:www\.)?github\.com\/([^\/]+)\/([^\/]+)(?:\/tree\/([^\/]+)(?:\/(.+))?)?$/;
+    const match = url.match(githubUrlPattern);
+
+    if (!match) {
+        throw new Error('Invalid GitHub URL format. Expected: https://github.com/owner/repo/tree/branch/folder');
+    }
+
+    // Extract components from the matched pattern
+    const owner = match[1];
+    const repo = match[2];
+    const branch = match[3] || 'main'; // Default to 'main' if branch is not specified
+    const folderPath = match[4] || ''; // Empty string if no folder path
+
+    // Additional validation
+    if (!owner || !repo) {
+        throw new Error('Invalid GitHub URL: Missing repository owner or name');
+    }
+
     return { owner, repo, branch, folderPath };
 }