git-ripper 1.4.13 → 1.5.1

package/README.md

@@ -96,6 +96,7 @@ git-ripper https://github.com/username/repository/tree/branch/folder --zip="my-a
 | Option                     | Description                              | Default           |
 | -------------------------- | ---------------------------------------- | ----------------- |
 | `-o, --output <directory>` | Specify output directory                 | Current directory |
+| `--gh-token <token>`       | GitHub Personal Access Token             | -                 |
 | `--zip [filename]`         | Create ZIP archive of downloaded content | -                 |
 | `--no-resume`              | Disable resume functionality             | -                 |
 | `--force-restart`          | Ignore existing checkpoints and restart  | -                 |
@@ -103,6 +104,46 @@ git-ripper https://github.com/username/repository/tree/branch/folder --zip="my-a
 | `-V, --version`            | Show version number                      | -                 |
 | `-h, --help`               | Show help                                | -                 |
 
+## Authentication (Private Repositories & Rate Limits)
+
+To download from private repositories or to increase your API rate limit, you need to provide a GitHub Personal Access Token (PAT).
+
+### How to Generate a Token
+
+You can use either a **Fine-grained token** (Recommended) or a **Classic token**.
+
+#### Option A: Fine-grained Token (Recommended)
+
+1.  Go to **Settings** > **Developer settings** > **Personal access tokens** > **Fine-grained tokens**.
+2.  Click **Generate new token**.
+3.  Name it (e.g., "Git-ripper").
+4.  **Resource owner**: Select your user.
+5.  **Repository access**: Select **Only select repositories** and choose the private repository you want to download from.
+6.  **Permissions**:
+    - Click on **Repository permissions**.
+    - Find **Contents** and change Access to **Read-only**.
+    - _Note: Metadata permission is selected automatically._
+7.  Click **Generate token**.
+
+#### Option B: Classic Token
+
+1.  Go to **Settings** > **Developer settings** > **Personal access tokens** > **Tokens (classic)**.
+2.  Click **Generate new token** > **Generate new token (classic)**.
+3.  Give your token a descriptive name.
+4.  **Select Scopes:**
+    - **For Private Repositories:** Select the **`repo`** scope (Full control of private repositories).
+5.  Click **Generate token**.
+
+### Using the Token
+
+Pass the token using the `--gh-token` flag:
+
+```bash
+git-ripper https://github.com/username/private-repo/tree/main/src --gh-token ghp_YourTokenHere
+```
+
+> **Security Note:** Be careful not to share your token or commit it to public repositories.
+
 ## Examples
 
 ### Extract a Component Library
@@ -133,6 +174,13 @@ git-ripper https://github.com/nodejs/node/tree/main/doc -o ./node-docs
 git-ripper https://github.com/tailwindlabs/tailwindcss/tree/master/src/components -o ./tailwind-components
 ```
 
+### Download from Private Repository
+
+```bash
+# Download from a private repository using a token
+git-ripper https://github.com/my-org/private-project/tree/main/src --gh-token ghp_abc123...
+```
+
 ### Download and Create Archive
 
 ```bash
@@ -207,7 +255,7 @@ The resume functionality uses checkpoint files stored in `.git_ripper_checkpoint
 
 ## Configuration
 
-Git-ripper works out of the box without configuration. For rate-limited GitHub API usage, authentication support is under development.
+Git-ripper works out of the box without configuration. For rate-limited GitHub API usage or private repositories, use the `--gh-token` option as described in the [Authentication](#authentication-private-repositories--rate-limits) section.
 
 ## Troubleshooting
 
@@ -219,7 +267,10 @@ Git-ripper works out of the box without configuration. For rate-limited GitHub A
 Error: Request failed with status code 403
 ```
 
-**Solution**: GitHub limits unauthenticated API requests. Wait a few minutes and try again.
+**Solution**: GitHub limits unauthenticated API requests. You can either:
+
+1. Wait a few minutes and try again
+2. Use the `--gh-token` option with a Personal Access Token to significantly increase your rate limit
 
 #### Invalid URL Format
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "git-ripper",
-  "version": "1.4.13",
+  "version": "1.5.1",
   "description": "CLI tool that lets you download specific folders from GitHub repositories without cloning the entire repo.",
   "main": "src/index.js",
   "type": "module",

package/src/archiver.js

@@ -128,12 +128,14 @@ export const createArchive = (sourceDir, outputPath) => {
  * @param {object} repoInfo - Repository information object
  * @param {string} outputDir - Directory where files should be downloaded
  * @param {string} archiveName - Custom name for the archive file (optional)
+ * @param {object} [options] - Download options including token
  * @returns {Promise<string>} - Path to the created archive
  */
 export const downloadAndArchive = async (
   repoInfo,
   outputDir,
-  archiveName = null
+  archiveName = null,
+  options = {}
 ) => {
   const { downloadFolder } = await import("./downloader.js");
 
@@ -146,7 +148,7 @@ export const downloadAndArchive = async (
   fs.mkdirSync(tempDir, { recursive: true });
   try {
     // Download the folder contents
-    const downloadResult = await downloadFolder(repoInfo, tempDir);
+    const downloadResult = await downloadFolder(repoInfo, tempDir, options);
 
     // Check if download failed
     if (downloadResult && !downloadResult.success) {

package/src/downloader.js

@@ -45,10 +45,16 @@ const getSpinnerFrame = () => {
  * @param {string} repo - Repository name
  * @param {string} branch - Branch name
  * @param {string} folderPath - Path to the folder
+ * @param {string} [token] - GitHub Personal Access Token
  * @returns {Promise<Array>} - Promise resolving to an array of file objects
  * @throws {Error} - Throws error on API failures instead of returning empty array
  */
-const fetchFolderContents = async (owner, repo, branch, folderPath) => {
+const fetchFolderContents = async (owner, repo, branch, folderPath, token) => {
+  const headers = {
+    Accept: "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+  };
   let effectiveBranch = branch;
   if (!effectiveBranch) {
     // If no branch is specified, fetch the default branch for the repository
@@ -56,7 +62,7 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
       const repoInfoUrl = `https://api.github.com/repos/${encodeURIComponent(
         owner
       )}/${encodeURIComponent(repo)}`;
-      const repoInfoResponse = await axios.get(repoInfoUrl);
+      const repoInfoResponse = await axios.get(repoInfoUrl, { headers });
       effectiveBranch = repoInfoResponse.data.default_branch;
       if (!effectiveBranch) {
         throw new Error(
@@ -85,7 +91,7 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
   )}?recursive=1`;
 
   try {
-    const response = await axios.get(apiUrl);
+    const response = await axios.get(apiUrl, { headers });
 
     // Check if GitHub API returned truncated results
     if (response.data.truncated) {
@@ -125,7 +131,7 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
             isRateLimit = true;
             errorMessage = `GitHub API rate limit exceeded. Please wait until ${new Date(
               parseInt(error.response.headers["x-ratelimit-reset"]) * 1000
-            ).toLocaleTimeString()} or add a GitHub token (feature coming soon).`;
+            ).toLocaleTimeString()} or use the --gh-token option to increase your rate limit.`;
           } else {
             errorMessage = `Access forbidden: ${
               error.response.data.message ||
@@ -162,9 +168,20 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
  * @param {string} branch - Branch name
  * @param {string} filePath - Path to the file
  * @param {string} outputPath - Path where the file should be saved
+ * @param {string} [token] - GitHub Personal Access Token
  * @returns {Promise<Object>} - Object containing download status
  */
-const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
+const downloadFile = async (
+  owner,
+  repo,
+  branch,
+  filePath,
+  outputPath,
+  token
+) => {
+  const headers = {
+    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+  };
   let effectiveBranch = branch;
   if (!effectiveBranch) {
     // If no branch is specified, fetch the default branch for the repository
@@ -174,7 +191,7 @@ const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
       const repoInfoUrl = `https://api.github.com/repos/${encodeURIComponent(
         owner
       )}/${encodeURIComponent(repo)}`;
-      const repoInfoResponse = await axios.get(repoInfoUrl);
+      const repoInfoResponse = await axios.get(repoInfoUrl, { headers });
       effectiveBranch = repoInfoResponse.data.default_branch;
       if (!effectiveBranch) {
         // console.error(chalk.red(`Could not determine default branch for ${owner}/${repo} for file ${filePath}.`));
@@ -213,7 +230,10 @@ const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
   const url = `${baseUrl}${fileUrlPath}`;
 
   try {
-    const response = await axios.get(url, { responseType: "arraybuffer" });
+    const response = await axios.get(url, {
+      responseType: "arraybuffer",
+      headers,
+    });
 
     // Ensure the directory exists
     try {
@@ -338,18 +358,28 @@ const createProgressRenderer = (owner, repo, folderPath) => {
  * @param {string} repoInfo.branch - Branch name
  * @param {string} repoInfo.folderPath - Path to the folder
  * @param {string} outputDir - Directory where files should be saved
+ * @param {Object} [options] - Download options
+ * @param {string} [options.token] - GitHub Personal Access Token
  * @returns {Promise<void>} - Promise that resolves when all files are downloaded
  */
 const downloadFolder = async (
   { owner, repo, branch, folderPath },
-  outputDir
+  outputDir,
+  options = {}
 ) => {
+  const { token } = options;
   console.log(
     chalk.cyan(`Analyzing repository structure for ${owner}/${repo}...`)
   );
 
   try {
-    const contents = await fetchFolderContents(owner, repo, branch, folderPath);
+    const contents = await fetchFolderContents(
+      owner,
+      repo,
+      branch,
+      folderPath,
+      token
+    );
 
     if (!contents || contents.length === 0) {
       const message = `No files found in ${folderPath || "repository root"}`;
@@ -428,7 +458,8 @@ const downloadFolder = async (
             repo,
             branch,
             item.path,
-            outputFilePath
+            outputFilePath,
+            token
           );
 
           // Update progress metrics
@@ -537,7 +568,7 @@ const downloadFolder = async (
 };
 
 // Export functions in ESM format
-export { downloadFolder, downloadFolderWithResume };
+export { downloadFolder, downloadFolderWithResume, downloadFile };
 
 /**
  * Downloads all files from a folder in a GitHub repository with resume capability
@@ -547,10 +578,14 @@ const downloadFolderWithResume = async (
   outputDir,
   options = { resume: true, forceRestart: false }
 ) => {
-  const { resume = true, forceRestart = false } = options;
+  const { resume = true, forceRestart = false, token } = options;
 
   if (!resume) {
-    return downloadFolder({ owner, repo, branch, folderPath }, outputDir);
+    return downloadFolder(
+      { owner, repo, branch, folderPath },
+      outputDir,
+      options
+    );
   }
 
   const resumeManager = new ResumeManager();
@@ -622,7 +657,13 @@ const downloadFolderWithResume = async (
   );
 
   try {
-    const contents = await fetchFolderContents(owner, repo, branch, folderPath);
+    const contents = await fetchFolderContents(
+      owner,
+      repo,
+      branch,
+      folderPath,
+      token
+    );
     if (!contents || contents.length === 0) {
       const message = `No files found in ${folderPath || "repository root"}`;
       console.log(chalk.yellow(message));
@@ -741,7 +782,8 @@ const downloadFolderWithResume = async (
           repo,
           branch,
           item.path,
-          outputFilePath
+          outputFilePath,
+          token
         );
 
         if (result.success) {

package/src/index.js

@@ -1,10 +1,10 @@
 import { program } from "commander";
 import { parseGitHubUrl } from "./parser.js";
-import { downloadFolder, downloadFolderWithResume } from "./downloader.js";
+import { downloadFolder, downloadFolderWithResume, downloadFile } from "./downloader.js";
 import { downloadAndArchive } from "./archiver.js";
 import { ResumeManager } from "./resumeManager.js";
 import { fileURLToPath } from "node:url";
-import { dirname, join, resolve } from "node:path";
+import { dirname, join, resolve, basename } from "node:path";
 import fs from "node:fs";
 import process from "node:process";
 import chalk from "chalk";
@@ -61,6 +61,7 @@ const initializeCLI = () => {
     .description("Clone specific folders from GitHub repositories")
     .argument("[url]", "GitHub URL of the folder to clone")
     .option("-o, --output <directory>", "Output directory", process.cwd())
+    .option("--gh-token <token>", "GitHub Personal Access Token for private repositories")
     .option("--zip [filename]", "Create ZIP archive of downloaded files")
     .option("--no-resume", "Disable resume functionality")
     .option("--force-restart", "Ignore existing checkpoints and start fresh")
@@ -122,6 +123,7 @@ const initializeCLI = () => {
         const downloadOptions = {
           resume: options.resume !== false, // Default to true unless --no-resume
           forceRestart: options.forceRestart || false,
+          token: options.ghToken,
         };
 
         let operationType = createArchive ? "archive" : "download";
@@ -131,7 +133,19 @@ const initializeCLI = () => {
         try {
           if (createArchive) {
             console.log(`Creating ZIP archive...`);
-            await downloadAndArchive(parsedUrl, options.output, archiveName);
+            await downloadAndArchive(parsedUrl, options.output, archiveName, downloadOptions);
+          } else if (parsedUrl.type === "blob") {
+            console.log(`Downloading file to: ${options.output}`);
+            const fileName = basename(parsedUrl.folderPath);
+            const outputPath = join(options.output, fileName);
+            result = await downloadFile(
+              parsedUrl.owner,
+              parsedUrl.repo,
+              parsedUrl.branch,
+              parsedUrl.folderPath,
+              outputPath,
+              options.ghToken
+            );
           } else {
             console.log(`Downloading folder to: ${options.output}`);
             if (downloadOptions.resume) {
@@ -141,7 +155,7 @@ const initializeCLI = () => {
                 downloadOptions
               );
             } else {
-              result = await downloadFolder(parsedUrl, options.output);
+              result = await downloadFolder(parsedUrl, options.output, downloadOptions);
             }
           }
         } catch (opError) {

package/src/parser.js

@@ -6,7 +6,7 @@ export function parseGitHubUrl(url) {
 
   // Validate if it's a GitHub URL
   const githubUrlPattern =
-    /^https?:\/\/(?:www\.)?github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:tree|blob)\/([^\/]+)(?:\/(.+))?)?$/;
+    /^https?:\/\/(?:www\.)?github\.com\/([^\/]+)\/([^\/]+)(?:\/(tree|blob)\/([^\/]+)(?:\/(.+))?)?$/;
   const match = url.match(githubUrlPattern);
 
   if (!match) {
@@ -17,14 +17,21 @@ export function parseGitHubUrl(url) {
 
   // Extract components from the matched pattern
   const owner = decodeURIComponent(match[1]);
-  const repo = decodeURIComponent(match[2]);
-  const branch = match[3] ? decodeURIComponent(match[3]) : ""; // Branch is an empty string if not present
-  const folderPath = match[4] ? decodeURIComponent(match[4]) : ""; // Empty string if no folder path
+  let repo = decodeURIComponent(match[2]);
+
+  // Remove .git suffix if present
+  if (repo.endsWith(".git")) {
+    repo = repo.slice(0, -4);
+  }
+
+  const type = match[3] || "tree"; // Default to tree if not present (root of repo)
+  const branch = match[4] ? decodeURIComponent(match[4]) : ""; // Branch is an empty string if not present
+  const folderPath = match[5] ? decodeURIComponent(match[5]) : ""; // Empty string if no folder path
 
   // Additional validation
   if (!owner || !repo) {
     throw new Error("Invalid GitHub URL: Missing repository owner or name");
   }
 
-  return { owner, repo, branch, folderPath };
+  return { owner, repo, branch, folderPath, type };
 }