git-ripper 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/archiver.js +90 -61
- package/src/downloader.js +247 -90
- package/src/index.js +52 -35
- package/src/parser.js +23 -20
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "git-ripper",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.1",
|
|
4
4
|
"description": "CLI tool that lets you download specific folders from GitHub repositories without cloning the entire repo.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
8
|
-
"git-ripper": "
|
|
8
|
+
"git-ripper": "bin/git-ripper.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
11
|
"test": "echo \"Error: no test specified\" && exit 1",
|
package/src/archiver.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import fs from
|
|
2
|
-
import path from
|
|
3
|
-
import archiver from
|
|
4
|
-
import chalk from
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import archiver from "archiver";
|
|
4
|
+
import chalk from "chalk";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Validates the output path for an archive file
|
|
@@ -12,29 +12,33 @@ import chalk from 'chalk';
|
|
|
12
12
|
const validateArchivePath = (outputPath) => {
|
|
13
13
|
// Check if path is provided
|
|
14
14
|
if (!outputPath) {
|
|
15
|
-
throw new Error(
|
|
15
|
+
throw new Error("Output path is required");
|
|
16
16
|
}
|
|
17
|
-
|
|
17
|
+
|
|
18
18
|
// Check if the output directory exists or can be created
|
|
19
19
|
const outputDir = path.dirname(outputPath);
|
|
20
20
|
try {
|
|
21
21
|
if (!fs.existsSync(outputDir)) {
|
|
22
22
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
23
23
|
}
|
|
24
|
-
|
|
24
|
+
|
|
25
25
|
// Check if the directory is writable
|
|
26
26
|
fs.accessSync(outputDir, fs.constants.W_OK);
|
|
27
|
-
|
|
27
|
+
|
|
28
28
|
// Check if file already exists and is writable
|
|
29
29
|
if (fs.existsSync(outputPath)) {
|
|
30
30
|
fs.accessSync(outputPath, fs.constants.W_OK);
|
|
31
31
|
// File exists and is writable, so we'll overwrite it
|
|
32
|
-
console.warn(
|
|
32
|
+
console.warn(
|
|
33
|
+
chalk.yellow(
|
|
34
|
+
`Warning: File ${outputPath} already exists and will be overwritten`
|
|
35
|
+
)
|
|
36
|
+
);
|
|
33
37
|
}
|
|
34
|
-
|
|
38
|
+
|
|
35
39
|
return true;
|
|
36
40
|
} catch (error) {
|
|
37
|
-
if (error.code ===
|
|
41
|
+
if (error.code === "EACCES") {
|
|
38
42
|
throw new Error(`Permission denied: Cannot write to ${outputPath}`);
|
|
39
43
|
}
|
|
40
44
|
throw new Error(`Invalid output path: ${error.message}`);
|
|
@@ -43,7 +47,7 @@ const validateArchivePath = (outputPath) => {
|
|
|
43
47
|
|
|
44
48
|
/**
|
|
45
49
|
* Creates an archive (zip or tar) from a directory
|
|
46
|
-
*
|
|
50
|
+
*
|
|
47
51
|
* @param {string} sourceDir - Source directory to archive
|
|
48
52
|
* @param {string} outputPath - Path where the archive should be saved
|
|
49
53
|
* @param {object} options - Archive options
|
|
@@ -54,74 +58,80 @@ const validateArchivePath = (outputPath) => {
|
|
|
54
58
|
export const createArchive = (sourceDir, outputPath, options = {}) => {
|
|
55
59
|
return new Promise((resolve, reject) => {
|
|
56
60
|
try {
|
|
57
|
-
const { format =
|
|
58
|
-
|
|
61
|
+
const { format = "zip", compressionLevel = 6 } = options;
|
|
62
|
+
|
|
59
63
|
// Validate source directory
|
|
60
64
|
if (!fs.existsSync(sourceDir)) {
|
|
61
|
-
return reject(
|
|
65
|
+
return reject(
|
|
66
|
+
new Error(`Source directory does not exist: ${sourceDir}`)
|
|
67
|
+
);
|
|
62
68
|
}
|
|
63
|
-
|
|
69
|
+
|
|
64
70
|
const stats = fs.statSync(sourceDir);
|
|
65
71
|
if (!stats.isDirectory()) {
|
|
66
|
-
return reject(
|
|
72
|
+
return reject(
|
|
73
|
+
new Error(`Source path is not a directory: ${sourceDir}`)
|
|
74
|
+
);
|
|
67
75
|
}
|
|
68
|
-
|
|
76
|
+
|
|
69
77
|
// Validate output path
|
|
70
78
|
validateArchivePath(outputPath);
|
|
71
|
-
|
|
72
|
-
// Ensure the output directory exists
|
|
73
|
-
const outputDir = path.dirname(outputPath);
|
|
74
|
-
if (!fs.existsSync(outputDir)) {
|
|
75
|
-
fs.mkdirSync(outputDir, { recursive: true });
|
|
76
|
-
}
|
|
77
|
-
|
|
79
|
+
|
|
78
80
|
// Create output stream
|
|
79
81
|
const output = fs.createWriteStream(outputPath);
|
|
80
82
|
let archive;
|
|
81
|
-
|
|
83
|
+
|
|
82
84
|
// Create the appropriate archive type
|
|
83
|
-
if (format ===
|
|
84
|
-
archive = archiver(
|
|
85
|
-
zlib: { level: compressionLevel }
|
|
85
|
+
if (format === "zip") {
|
|
86
|
+
archive = archiver("zip", {
|
|
87
|
+
zlib: { level: compressionLevel },
|
|
86
88
|
});
|
|
87
|
-
} else if (format ===
|
|
88
|
-
archive = archiver('tar');
|
|
89
|
+
} else if (format === "tar") {
|
|
89
90
|
// Use gzip compression for tar if compressionLevel > 0
|
|
90
91
|
if (compressionLevel > 0) {
|
|
91
|
-
archive = archiver(
|
|
92
|
+
archive = archiver("tar", {
|
|
92
93
|
gzip: true,
|
|
93
|
-
gzipOptions: { level: compressionLevel }
|
|
94
|
+
gzipOptions: { level: compressionLevel },
|
|
94
95
|
});
|
|
96
|
+
} else {
|
|
97
|
+
// Create a tar archive without gzip compression
|
|
98
|
+
archive = archiver("tar");
|
|
95
99
|
}
|
|
96
100
|
} else {
|
|
97
101
|
return reject(new Error(`Unsupported archive format: ${format}`));
|
|
98
102
|
}
|
|
99
|
-
|
|
103
|
+
|
|
100
104
|
// Listen for archive events
|
|
101
|
-
output.on(
|
|
105
|
+
output.on("close", () => {
|
|
102
106
|
const size = archive.pointer();
|
|
103
|
-
console.log(
|
|
107
|
+
console.log(
|
|
108
|
+
chalk.green(
|
|
109
|
+
`✓ Archive created: ${outputPath} (${(size / 1024 / 1024).toFixed(
|
|
110
|
+
2
|
|
111
|
+
)} MB)`
|
|
112
|
+
)
|
|
113
|
+
);
|
|
104
114
|
resolve(outputPath);
|
|
105
115
|
});
|
|
106
|
-
|
|
107
|
-
archive.on(
|
|
116
|
+
|
|
117
|
+
archive.on("error", (err) => {
|
|
108
118
|
reject(err);
|
|
109
119
|
});
|
|
110
|
-
|
|
111
|
-
archive.on(
|
|
112
|
-
if (err.code ===
|
|
120
|
+
|
|
121
|
+
archive.on("warning", (err) => {
|
|
122
|
+
if (err.code === "ENOENT") {
|
|
113
123
|
console.warn(chalk.yellow(`Warning: ${err.message}`));
|
|
114
124
|
} else {
|
|
115
125
|
reject(err);
|
|
116
126
|
}
|
|
117
127
|
});
|
|
118
|
-
|
|
128
|
+
|
|
119
129
|
// Pipe archive data to the output file
|
|
120
130
|
archive.pipe(output);
|
|
121
|
-
|
|
131
|
+
|
|
122
132
|
// Add the directory contents to the archive
|
|
123
133
|
archive.directory(sourceDir, false);
|
|
124
|
-
|
|
134
|
+
|
|
125
135
|
// Finalize the archive
|
|
126
136
|
archive.finalize();
|
|
127
137
|
} catch (error) {
|
|
@@ -132,7 +142,7 @@ export const createArchive = (sourceDir, outputPath, options = {}) => {
|
|
|
132
142
|
|
|
133
143
|
/**
|
|
134
144
|
* Downloads folder contents and creates an archive
|
|
135
|
-
*
|
|
145
|
+
*
|
|
136
146
|
* @param {object} repoInfo - Repository information object
|
|
137
147
|
* @param {string} outputDir - Directory where files should be downloaded
|
|
138
148
|
* @param {string} archiveFormat - Archive format ('zip' or 'tar')
|
|
@@ -140,38 +150,53 @@ export const createArchive = (sourceDir, outputPath, options = {}) => {
|
|
|
140
150
|
* @param {number} compressionLevel - Compression level (0-9)
|
|
141
151
|
* @returns {Promise<string>} - Path to the created archive
|
|
142
152
|
*/
|
|
143
|
-
export const downloadAndArchive = async (
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
153
|
+
export const downloadAndArchive = async (
|
|
154
|
+
repoInfo,
|
|
155
|
+
outputDir,
|
|
156
|
+
archiveFormat = "zip",
|
|
157
|
+
archiveName = null,
|
|
158
|
+
compressionLevel = 6
|
|
159
|
+
) => {
|
|
160
|
+
const { downloadFolder } = await import("./downloader.js");
|
|
161
|
+
|
|
162
|
+
console.log(
|
|
163
|
+
chalk.cyan(
|
|
164
|
+
`Downloading folder and preparing to create ${archiveFormat.toUpperCase()} archive...`
|
|
165
|
+
)
|
|
166
|
+
);
|
|
167
|
+
|
|
148
168
|
// Create a temporary directory for the download
|
|
149
169
|
const tempDir = path.join(outputDir, `.temp-${Date.now()}`);
|
|
150
170
|
fs.mkdirSync(tempDir, { recursive: true });
|
|
151
|
-
|
|
171
|
+
|
|
152
172
|
try {
|
|
153
173
|
// Download the folder contents
|
|
154
174
|
await downloadFolder(repoInfo, tempDir);
|
|
155
|
-
|
|
175
|
+
|
|
156
176
|
// Determine archive filename
|
|
157
177
|
let archiveFileName = archiveName;
|
|
158
178
|
if (!archiveFileName) {
|
|
159
179
|
const { owner, repo, folderPath } = repoInfo;
|
|
160
|
-
const folderName = folderPath ? folderPath.split(
|
|
180
|
+
const folderName = folderPath ? folderPath.split("/").pop() : repo;
|
|
161
181
|
archiveFileName = `${folderName || repo}-${owner}`;
|
|
162
182
|
}
|
|
163
|
-
|
|
183
|
+
|
|
164
184
|
// Add extension if not present
|
|
165
185
|
if (!archiveFileName.endsWith(`.${archiveFormat}`)) {
|
|
166
186
|
archiveFileName += `.${archiveFormat}`;
|
|
167
187
|
}
|
|
168
|
-
|
|
188
|
+
|
|
169
189
|
const archivePath = path.join(outputDir, archiveFileName);
|
|
170
|
-
|
|
190
|
+
|
|
171
191
|
// Create the archive
|
|
172
|
-
console.log(
|
|
173
|
-
|
|
174
|
-
|
|
192
|
+
console.log(
|
|
193
|
+
chalk.cyan(`Creating ${archiveFormat.toUpperCase()} archive...`)
|
|
194
|
+
);
|
|
195
|
+
await createArchive(tempDir, archivePath, {
|
|
196
|
+
format: archiveFormat,
|
|
197
|
+
compressionLevel,
|
|
198
|
+
});
|
|
199
|
+
|
|
175
200
|
return archivePath;
|
|
176
201
|
} catch (error) {
|
|
177
202
|
throw new Error(`Failed to create archive: ${error.message}`);
|
|
@@ -180,7 +205,11 @@ export const downloadAndArchive = async (repoInfo, outputDir, archiveFormat = 'z
|
|
|
180
205
|
try {
|
|
181
206
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
182
207
|
} catch (err) {
|
|
183
|
-
console.warn(
|
|
208
|
+
console.warn(
|
|
209
|
+
chalk.yellow(
|
|
210
|
+
`Warning: Failed to clean up temporary directory: ${err.message}`
|
|
211
|
+
)
|
|
212
|
+
);
|
|
184
213
|
}
|
|
185
214
|
}
|
|
186
|
-
};
|
|
215
|
+
};
|
package/src/downloader.js
CHANGED
|
@@ -17,11 +17,11 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
17
17
|
const __dirname = dirname(__filename);
|
|
18
18
|
|
|
19
19
|
// Define spinner animation frames
|
|
20
|
-
const spinnerFrames = [
|
|
20
|
+
const spinnerFrames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
21
21
|
// Alternative progress bar characters for more visual appeal
|
|
22
22
|
const progressChars = {
|
|
23
|
-
complete:
|
|
24
|
-
incomplete:
|
|
23
|
+
complete: "▰", // Alternative: '■', '●', '◆', '▣'
|
|
24
|
+
incomplete: "▱", // Alternative: '□', '○', '◇', '▢'
|
|
25
25
|
};
|
|
26
26
|
|
|
27
27
|
// Track frame index for spinner animation
|
|
@@ -46,43 +46,112 @@ const getSpinnerFrame = () => {
|
|
|
46
46
|
* @returns {Promise<Array>} - Promise resolving to an array of file objects
|
|
47
47
|
*/
|
|
48
48
|
const fetchFolderContents = async (owner, repo, branch, folderPath) => {
|
|
49
|
-
|
|
49
|
+
let effectiveBranch = branch;
|
|
50
|
+
if (!effectiveBranch) {
|
|
51
|
+
// If no branch is specified, fetch the default branch for the repository
|
|
52
|
+
try {
|
|
53
|
+
const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`;
|
|
54
|
+
const repoInfoResponse = await axios.get(repoInfoUrl);
|
|
55
|
+
effectiveBranch = repoInfoResponse.data.default_branch;
|
|
56
|
+
if (!effectiveBranch) {
|
|
57
|
+
console.error(
|
|
58
|
+
chalk.red(
|
|
59
|
+
`Could not determine default branch for ${owner}/${repo}. Please specify a branch in the URL.`
|
|
60
|
+
)
|
|
61
|
+
);
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
console.log(
|
|
65
|
+
chalk.blue(
|
|
66
|
+
`No branch specified, using default branch: ${effectiveBranch}`
|
|
67
|
+
)
|
|
68
|
+
);
|
|
69
|
+
} catch (error) {
|
|
70
|
+
console.error(
|
|
71
|
+
chalk.red(
|
|
72
|
+
`Failed to fetch default branch for ${owner}/${repo}: ${error.message}`
|
|
73
|
+
)
|
|
74
|
+
);
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${effectiveBranch}?recursive=1`;
|
|
50
80
|
|
|
51
81
|
try {
|
|
52
82
|
const response = await axios.get(apiUrl);
|
|
53
|
-
|
|
83
|
+
|
|
54
84
|
// Check if GitHub API returned truncated results
|
|
55
85
|
if (response.data.truncated) {
|
|
56
|
-
console.warn(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
86
|
+
console.warn(
|
|
87
|
+
chalk.yellow(
|
|
88
|
+
`Warning: The repository is too large and some files may be missing. ` +
|
|
89
|
+
`Consider using git clone for complete repositories.`
|
|
90
|
+
)
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Original filter:
|
|
95
|
+
// return response.data.tree.filter((item) =>
|
|
96
|
+
// item.path.startsWith(folderPath)
|
|
97
|
+
// );
|
|
98
|
+
|
|
99
|
+
// New filter logic:
|
|
100
|
+
if (folderPath === "") {
|
|
101
|
+
// For the root directory, all items from the recursive tree are relevant.
|
|
102
|
+
// item.path.startsWith("") would also achieve this.
|
|
103
|
+
return response.data.tree;
|
|
104
|
+
} else {
|
|
105
|
+
// For a specific folder, items must be *inside* that folder.
|
|
106
|
+
// Ensure folderPath is treated as a directory prefix by adding a trailing slash if not present.
|
|
107
|
+
const prefix = folderPath.endsWith("/") ? folderPath : folderPath + "/";
|
|
108
|
+
return response.data.tree.filter((item) => item.path.startsWith(prefix));
|
|
60
109
|
}
|
|
61
|
-
|
|
62
|
-
return response.data.tree.filter((item) => item.path.startsWith(folderPath));
|
|
63
110
|
} catch (error) {
|
|
64
111
|
if (error.response) {
|
|
65
112
|
// Handle specific HTTP error codes
|
|
66
|
-
switch(error.response.status) {
|
|
113
|
+
switch (error.response.status) {
|
|
67
114
|
case 403:
|
|
68
|
-
if (error.response.headers[
|
|
69
|
-
console.error(
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
115
|
+
if (error.response.headers["x-ratelimit-remaining"] === "0") {
|
|
116
|
+
console.error(
|
|
117
|
+
chalk.red(
|
|
118
|
+
`GitHub API rate limit exceeded. Please wait until ${new Date(
|
|
119
|
+
parseInt(error.response.headers["x-ratelimit-reset"]) * 1000
|
|
120
|
+
).toLocaleTimeString()} or add a GitHub token (feature coming soon).`
|
|
121
|
+
)
|
|
122
|
+
);
|
|
74
123
|
} else {
|
|
75
|
-
console.error(
|
|
124
|
+
console.error(
|
|
125
|
+
chalk.red(
|
|
126
|
+
`Access forbidden: ${
|
|
127
|
+
error.response.data.message || "Unknown reason"
|
|
128
|
+
}`
|
|
129
|
+
)
|
|
130
|
+
);
|
|
76
131
|
}
|
|
77
132
|
break;
|
|
78
133
|
case 404:
|
|
79
|
-
console.error(
|
|
134
|
+
console.error(
|
|
135
|
+
chalk.red(
|
|
136
|
+
`Repository, branch, or folder not found: ${owner}/${repo}/${branch}/${folderPath}`
|
|
137
|
+
)
|
|
138
|
+
);
|
|
80
139
|
break;
|
|
81
140
|
default:
|
|
82
|
-
console.error(
|
|
141
|
+
console.error(
|
|
142
|
+
chalk.red(
|
|
143
|
+
`API error (${error.response.status}): ${
|
|
144
|
+
error.response.data.message || error.message
|
|
145
|
+
}`
|
|
146
|
+
)
|
|
147
|
+
);
|
|
83
148
|
}
|
|
84
149
|
} else if (error.request) {
|
|
85
|
-
console.error(
|
|
150
|
+
console.error(
|
|
151
|
+
chalk.red(
|
|
152
|
+
`Network error: No response received from GitHub. Please check your internet connection.`
|
|
153
|
+
)
|
|
154
|
+
);
|
|
86
155
|
} else {
|
|
87
156
|
console.error(chalk.red(`Error preparing request: ${error.message}`));
|
|
88
157
|
}
|
|
@@ -100,44 +169,81 @@ const fetchFolderContents = async (owner, repo, branch, folderPath) => {
|
|
|
100
169
|
* @returns {Promise<Object>} - Object containing download status
|
|
101
170
|
*/
|
|
102
171
|
const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
|
|
103
|
-
|
|
172
|
+
let effectiveBranch = branch;
|
|
173
|
+
if (!effectiveBranch) {
|
|
174
|
+
// If no branch is specified, fetch the default branch for the repository
|
|
175
|
+
// This check might be redundant if fetchFolderContents already resolved it,
|
|
176
|
+
// but it's a good fallback for direct downloadFile calls if any.
|
|
177
|
+
try {
|
|
178
|
+
const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`;
|
|
179
|
+
const repoInfoResponse = await axios.get(repoInfoUrl);
|
|
180
|
+
effectiveBranch = repoInfoResponse.data.default_branch;
|
|
181
|
+
if (!effectiveBranch) {
|
|
182
|
+
// console.error(chalk.red(`Could not determine default branch for ${owner}/${repo} for file ${filePath}.`));
|
|
183
|
+
// Do not log error here as it might be a root file download where branch is not in URL
|
|
184
|
+
}
|
|
185
|
+
} catch (error) {
|
|
186
|
+
// console.error(chalk.red(`Failed to fetch default branch for ${owner}/${repo} for file ${filePath}: ${error.message}`));
|
|
187
|
+
// Do not log error here
|
|
188
|
+
}
|
|
189
|
+
// If still no branch, the raw URL might work for default branch, or fail.
|
|
190
|
+
// The original code didn't explicitly handle this for downloadFile, relying on raw.githubusercontent default behavior.
|
|
191
|
+
// For robustness, we should ensure effectiveBranch is set. If not, the URL will be malformed or use GitHub's default.
|
|
192
|
+
if (!effectiveBranch) {
|
|
193
|
+
// Fallback to a common default, or let the API call fail if truly ambiguous
|
|
194
|
+
// For raw content, GitHub often defaults to the main branch if not specified,
|
|
195
|
+
// but it's better to be explicit if we can.
|
|
196
|
+
// However, altering the URL structure for raw.githubusercontent.com without a branch
|
|
197
|
+
// might be tricky if the original URL didn't have it.
|
|
198
|
+
// The existing raw URL construction assumes branch is present or GitHub handles its absence.
|
|
199
|
+
// Let's stick to the original logic for raw URL construction if branch is not found,
|
|
200
|
+
// as `https://raw.githubusercontent.com/${owner}/${repo}/${filePath}` might work for root files on default branch.
|
|
201
|
+
// The critical part is `fetchFolderContents` determining the branch for listing.
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const baseUrl = `https://raw.githubusercontent.com/${owner}/${repo}`;
|
|
206
|
+
const fileUrlPath = effectiveBranch
|
|
207
|
+
? `/${effectiveBranch}/${filePath}`
|
|
208
|
+
: `/${filePath}`; // filePath might be at root
|
|
209
|
+
const url = `${baseUrl}${fileUrlPath}`;
|
|
104
210
|
|
|
105
211
|
try {
|
|
106
212
|
const response = await axios.get(url, { responseType: "arraybuffer" });
|
|
107
|
-
|
|
213
|
+
|
|
108
214
|
// Ensure the directory exists
|
|
109
215
|
try {
|
|
110
216
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
111
217
|
} catch (dirError) {
|
|
112
|
-
return {
|
|
113
|
-
filePath,
|
|
114
|
-
success: false,
|
|
218
|
+
return {
|
|
219
|
+
filePath,
|
|
220
|
+
success: false,
|
|
115
221
|
error: `Failed to create directory: ${dirError.message}`,
|
|
116
|
-
size: 0
|
|
222
|
+
size: 0,
|
|
117
223
|
};
|
|
118
224
|
}
|
|
119
|
-
|
|
225
|
+
|
|
120
226
|
// Write the file
|
|
121
227
|
try {
|
|
122
228
|
fs.writeFileSync(outputPath, Buffer.from(response.data));
|
|
123
229
|
} catch (fileError) {
|
|
124
|
-
return {
|
|
125
|
-
filePath,
|
|
126
|
-
success: false,
|
|
230
|
+
return {
|
|
231
|
+
filePath,
|
|
232
|
+
success: false,
|
|
127
233
|
error: `Failed to write file: ${fileError.message}`,
|
|
128
|
-
size: 0
|
|
234
|
+
size: 0,
|
|
129
235
|
};
|
|
130
236
|
}
|
|
131
|
-
|
|
132
|
-
return {
|
|
133
|
-
filePath,
|
|
237
|
+
|
|
238
|
+
return {
|
|
239
|
+
filePath,
|
|
134
240
|
success: true,
|
|
135
|
-
size: response.data.length
|
|
241
|
+
size: response.data.length,
|
|
136
242
|
};
|
|
137
243
|
} catch (error) {
|
|
138
244
|
// More detailed error handling for network requests
|
|
139
245
|
let errorMessage = error.message;
|
|
140
|
-
|
|
246
|
+
|
|
141
247
|
if (error.response) {
|
|
142
248
|
// The request was made and the server responded with an error status
|
|
143
249
|
switch (error.response.status) {
|
|
@@ -154,12 +260,12 @@ const downloadFile = async (owner, repo, branch, filePath, outputPath) => {
|
|
|
154
260
|
// The request was made but no response was received
|
|
155
261
|
errorMessage = "No response from server";
|
|
156
262
|
}
|
|
157
|
-
|
|
158
|
-
return {
|
|
159
|
-
filePath,
|
|
160
|
-
success: false,
|
|
263
|
+
|
|
264
|
+
return {
|
|
265
|
+
filePath,
|
|
266
|
+
success: false,
|
|
161
267
|
error: errorMessage,
|
|
162
|
-
size: 0
|
|
268
|
+
size: 0,
|
|
163
269
|
};
|
|
164
270
|
}
|
|
165
271
|
};
|
|
@@ -179,31 +285,40 @@ const createProgressRenderer = (owner, repo, folderPath) => {
|
|
|
179
285
|
try {
|
|
180
286
|
const { value, total, startTime } = params;
|
|
181
287
|
const { downloadedSize = 0 } = payload || { downloadedSize: 0 };
|
|
182
|
-
|
|
288
|
+
|
|
183
289
|
// Calculate progress percentage
|
|
184
290
|
const progress = Math.min(1, Math.max(0, value / Math.max(1, total)));
|
|
185
291
|
const percentage = Math.floor(progress * 100);
|
|
186
|
-
|
|
292
|
+
|
|
187
293
|
// Calculate elapsed time
|
|
188
294
|
const elapsedSecs = Math.max(0.1, (Date.now() - startTime) / 1000);
|
|
189
|
-
|
|
295
|
+
|
|
190
296
|
// Create the progress bar
|
|
191
|
-
const barLength = Math.max(
|
|
297
|
+
const barLength = Math.max(
|
|
298
|
+
20,
|
|
299
|
+
Math.min(40, Math.floor(terminalWidth / 2))
|
|
300
|
+
);
|
|
192
301
|
const completedLength = Math.round(barLength * progress);
|
|
193
302
|
const remainingLength = barLength - completedLength;
|
|
194
|
-
|
|
303
|
+
|
|
195
304
|
// Build the bar with custom progress characters
|
|
196
|
-
const completedBar = chalk.greenBright(
|
|
197
|
-
|
|
198
|
-
|
|
305
|
+
const completedBar = chalk.greenBright(
|
|
306
|
+
progressChars.complete.repeat(completedLength)
|
|
307
|
+
);
|
|
308
|
+
const remainingBar = chalk.gray(
|
|
309
|
+
progressChars.incomplete.repeat(remainingLength)
|
|
310
|
+
);
|
|
311
|
+
|
|
199
312
|
// Add spinner for animation
|
|
200
313
|
const spinner = chalk.cyanBright(getSpinnerFrame());
|
|
201
|
-
|
|
314
|
+
|
|
202
315
|
// Format the output
|
|
203
316
|
const progressInfo = `${chalk.cyan(`${value}/${total}`)} files`;
|
|
204
317
|
const sizeInfo = prettyBytes(downloadedSize || 0);
|
|
205
|
-
|
|
206
|
-
return `${spinner} ${completedBar}${remainingBar} ${chalk.yellow(
|
|
318
|
+
|
|
319
|
+
return `${spinner} ${completedBar}${remainingBar} ${chalk.yellow(
|
|
320
|
+
percentage + "%"
|
|
321
|
+
)} | ${progressInfo} | ${chalk.magenta(sizeInfo)}`;
|
|
207
322
|
} catch (error) {
|
|
208
323
|
// Fallback to a very simple progress indicator
|
|
209
324
|
return `${Math.floor((params.value / params.total) * 100)}% complete`;
|
|
@@ -221,88 +336,120 @@ const createProgressRenderer = (owner, repo, folderPath) => {
|
|
|
221
336
|
* @param {string} outputDir - Directory where files should be saved
|
|
222
337
|
* @returns {Promise<void>} - Promise that resolves when all files are downloaded
|
|
223
338
|
*/
|
|
224
|
-
const downloadFolder = async (
|
|
225
|
-
|
|
339
|
+
const downloadFolder = async (
|
|
340
|
+
{ owner, repo, branch, folderPath },
|
|
341
|
+
outputDir
|
|
342
|
+
) => {
|
|
343
|
+
console.log(
|
|
344
|
+
chalk.cyan(`Analyzing repository structure for ${owner}/${repo}...`)
|
|
345
|
+
);
|
|
226
346
|
|
|
227
347
|
try {
|
|
228
348
|
const contents = await fetchFolderContents(owner, repo, branch, folderPath);
|
|
229
|
-
|
|
349
|
+
|
|
230
350
|
if (!contents || contents.length === 0) {
|
|
231
|
-
console.log(
|
|
351
|
+
console.log(
|
|
352
|
+
chalk.yellow(`No files found in ${folderPath || "repository root"}`)
|
|
353
|
+
);
|
|
232
354
|
console.log(chalk.green(`Folder cloned successfully!`));
|
|
233
355
|
return;
|
|
234
356
|
}
|
|
235
357
|
|
|
236
358
|
// Filter for blob type (files)
|
|
237
|
-
const files = contents.filter(item => item.type === "blob");
|
|
359
|
+
const files = contents.filter((item) => item.type === "blob");
|
|
238
360
|
const totalFiles = files.length;
|
|
239
|
-
|
|
361
|
+
|
|
240
362
|
if (totalFiles === 0) {
|
|
241
|
-
console.log(
|
|
363
|
+
console.log(
|
|
364
|
+
chalk.yellow(
|
|
365
|
+
`No files found in ${
|
|
366
|
+
folderPath || "repository root"
|
|
367
|
+
} (only directories)`
|
|
368
|
+
)
|
|
369
|
+
);
|
|
242
370
|
console.log(chalk.green(`Folder cloned successfully!`));
|
|
243
371
|
return;
|
|
244
372
|
}
|
|
245
|
-
|
|
246
|
-
console.log(
|
|
247
|
-
|
|
373
|
+
|
|
374
|
+
console.log(
|
|
375
|
+
chalk.cyan(
|
|
376
|
+
`Downloading ${totalFiles} files from ${chalk.white(
|
|
377
|
+
owner + "/" + repo
|
|
378
|
+
)}...`
|
|
379
|
+
)
|
|
380
|
+
);
|
|
381
|
+
|
|
248
382
|
// Simplified progress bar setup
|
|
249
383
|
const progressBar = new cliProgress.SingleBar({
|
|
250
384
|
format: createProgressRenderer(owner, repo, folderPath),
|
|
251
385
|
hideCursor: true,
|
|
252
386
|
clearOnComplete: false,
|
|
253
387
|
stopOnComplete: true,
|
|
254
|
-
forceRedraw: true
|
|
388
|
+
forceRedraw: true,
|
|
255
389
|
});
|
|
256
|
-
|
|
390
|
+
|
|
257
391
|
// Track download metrics
|
|
258
392
|
let downloadedSize = 0;
|
|
259
393
|
const startTime = Date.now();
|
|
260
394
|
let failedFiles = [];
|
|
261
|
-
|
|
395
|
+
|
|
262
396
|
// Start progress bar
|
|
263
397
|
progressBar.start(totalFiles, 0, {
|
|
264
398
|
downloadedSize: 0,
|
|
265
|
-
startTime
|
|
399
|
+
startTime,
|
|
266
400
|
});
|
|
267
|
-
|
|
401
|
+
|
|
268
402
|
// Create download promises with concurrency control
|
|
269
403
|
const fileDownloadPromises = files.map((item) => {
|
|
270
404
|
// Keep the original structure by preserving the folder name
|
|
271
405
|
let relativePath = item.path;
|
|
272
|
-
if (folderPath && folderPath.trim() !==
|
|
273
|
-
relativePath = item.path
|
|
406
|
+
if (folderPath && folderPath.trim() !== "") {
|
|
407
|
+
relativePath = item.path
|
|
408
|
+
.substring(folderPath.length)
|
|
409
|
+
.replace(/^\//, "");
|
|
274
410
|
}
|
|
275
411
|
const outputFilePath = path.join(outputDir, relativePath);
|
|
276
|
-
|
|
412
|
+
|
|
277
413
|
return limit(async () => {
|
|
278
414
|
try {
|
|
279
|
-
const result = await downloadFile(
|
|
280
|
-
|
|
415
|
+
const result = await downloadFile(
|
|
416
|
+
owner,
|
|
417
|
+
repo,
|
|
418
|
+
branch,
|
|
419
|
+
item.path,
|
|
420
|
+
outputFilePath
|
|
421
|
+
);
|
|
422
|
+
|
|
281
423
|
// Update progress metrics
|
|
282
424
|
if (result.success) {
|
|
283
|
-
downloadedSize +=
|
|
425
|
+
downloadedSize += result.size || 0;
|
|
284
426
|
} else {
|
|
285
427
|
// Track failed files for reporting
|
|
286
428
|
failedFiles.push({
|
|
287
429
|
path: item.path,
|
|
288
|
-
error: result.error
|
|
430
|
+
error: result.error,
|
|
289
431
|
});
|
|
290
432
|
}
|
|
291
|
-
|
|
433
|
+
|
|
292
434
|
// Update progress bar with current metrics
|
|
293
435
|
progressBar.increment(1, {
|
|
294
|
-
downloadedSize
|
|
436
|
+
downloadedSize,
|
|
295
437
|
});
|
|
296
|
-
|
|
438
|
+
|
|
297
439
|
return result;
|
|
298
440
|
} catch (error) {
|
|
299
441
|
failedFiles.push({
|
|
300
442
|
path: item.path,
|
|
301
|
-
error: error.message
|
|
443
|
+
error: error.message,
|
|
302
444
|
});
|
|
303
|
-
|
|
445
|
+
|
|
304
446
|
progressBar.increment(1, { downloadedSize });
|
|
305
|
-
return {
|
|
447
|
+
return {
|
|
448
|
+
filePath: item.path,
|
|
449
|
+
success: false,
|
|
450
|
+
error: error.message,
|
|
451
|
+
size: 0,
|
|
452
|
+
};
|
|
306
453
|
}
|
|
307
454
|
});
|
|
308
455
|
});
|
|
@@ -310,7 +457,7 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
|
|
|
310
457
|
// Execute downloads in parallel with controlled concurrency
|
|
311
458
|
const results = await Promise.all(fileDownloadPromises);
|
|
312
459
|
progressBar.stop();
|
|
313
|
-
|
|
460
|
+
|
|
314
461
|
console.log(); // Add an empty line after progress bar
|
|
315
462
|
|
|
316
463
|
// Count successful and failed downloads
|
|
@@ -318,21 +465,31 @@ const downloadFolder = async ({ owner, repo, branch, folderPath }, outputDir) =>
|
|
|
318
465
|
const failed = failedFiles.length;
|
|
319
466
|
|
|
320
467
|
if (failed > 0) {
|
|
321
|
-
console.log(
|
|
322
|
-
|
|
468
|
+
console.log(
|
|
469
|
+
chalk.yellow(
|
|
470
|
+
`Downloaded ${succeeded} files successfully, ${failed} files failed`
|
|
471
|
+
)
|
|
472
|
+
);
|
|
473
|
+
|
|
323
474
|
// Show detailed errors if there aren't too many
|
|
324
475
|
if (failed <= 5) {
|
|
325
|
-
console.log(chalk.yellow(
|
|
326
|
-
failedFiles.forEach(file => {
|
|
476
|
+
console.log(chalk.yellow("Failed files:"));
|
|
477
|
+
failedFiles.forEach((file) => {
|
|
327
478
|
console.log(chalk.yellow(` - ${file.path}: ${file.error}`));
|
|
328
479
|
});
|
|
329
480
|
} else {
|
|
330
|
-
console.log(
|
|
481
|
+
console.log(
|
|
482
|
+
chalk.yellow(
|
|
483
|
+
`${failed} files failed to download. Check your connection or repository access.`
|
|
484
|
+
)
|
|
485
|
+
);
|
|
331
486
|
}
|
|
332
487
|
} else {
|
|
333
|
-
console.log(
|
|
488
|
+
console.log(
|
|
489
|
+
chalk.green(` All ${succeeded} files downloaded successfully!`)
|
|
490
|
+
);
|
|
334
491
|
}
|
|
335
|
-
|
|
492
|
+
|
|
336
493
|
console.log(chalk.green(`Folder cloned successfully!`));
|
|
337
494
|
} catch (error) {
|
|
338
495
|
console.error(chalk.red(`Error downloading folder: ${error.message}`));
|
package/src/index.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { program } from
|
|
2
|
-
import { parseGitHubUrl } from
|
|
3
|
-
import { downloadFolder } from
|
|
4
|
-
import { downloadAndArchive } from
|
|
5
|
-
import { fileURLToPath } from
|
|
6
|
-
import { dirname, join, resolve } from
|
|
7
|
-
import fs from
|
|
1
|
+
import { program } from "commander";
|
|
2
|
+
import { parseGitHubUrl } from "./parser.js";
|
|
3
|
+
import { downloadFolder } from "./downloader.js";
|
|
4
|
+
import { downloadAndArchive } from "./archiver.js";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { dirname, join, resolve } from "path";
|
|
7
|
+
import fs from "fs";
|
|
8
8
|
|
|
9
9
|
// Get package.json for version
|
|
10
10
|
const __filename = fileURLToPath(import.meta.url);
|
|
11
11
|
const __dirname = dirname(__filename);
|
|
12
|
-
const packagePath = join(__dirname,
|
|
13
|
-
const packageJson = JSON.parse(fs.readFileSync(packagePath,
|
|
12
|
+
const packagePath = join(__dirname, "..", "package.json");
|
|
13
|
+
const packageJson = JSON.parse(fs.readFileSync(packagePath, "utf8"));
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Validates and ensures the output directory exists
|
|
@@ -20,12 +20,12 @@ const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8'));
|
|
|
20
20
|
*/
|
|
21
21
|
const validateOutputDirectory = (outputDir) => {
|
|
22
22
|
if (!outputDir) {
|
|
23
|
-
throw new Error(
|
|
23
|
+
throw new Error("Output directory is required");
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
// Resolve to absolute path
|
|
27
27
|
const resolvedDir = resolve(outputDir);
|
|
28
|
-
|
|
28
|
+
|
|
29
29
|
try {
|
|
30
30
|
// Check if directory exists, if not try to create it
|
|
31
31
|
if (!fs.existsSync(resolvedDir)) {
|
|
@@ -34,16 +34,18 @@ const validateOutputDirectory = (outputDir) => {
|
|
|
34
34
|
// Check if it's actually a directory
|
|
35
35
|
const stats = fs.statSync(resolvedDir);
|
|
36
36
|
if (!stats.isDirectory()) {
|
|
37
|
-
throw new Error(
|
|
37
|
+
throw new Error(
|
|
38
|
+
`Output path exists but is not a directory: ${outputDir}`
|
|
39
|
+
);
|
|
38
40
|
}
|
|
39
41
|
}
|
|
40
|
-
|
|
42
|
+
|
|
41
43
|
// Check if the directory is writable
|
|
42
44
|
fs.accessSync(resolvedDir, fs.constants.W_OK);
|
|
43
|
-
|
|
45
|
+
|
|
44
46
|
return resolvedDir;
|
|
45
47
|
} catch (error) {
|
|
46
|
-
if (error.code ===
|
|
48
|
+
if (error.code === "EACCES") {
|
|
47
49
|
throw new Error(`Permission denied: Cannot write to ${outputDir}`);
|
|
48
50
|
}
|
|
49
51
|
throw new Error(`Invalid output directory: ${error.message}`);
|
|
@@ -53,53 +55,68 @@ const validateOutputDirectory = (outputDir) => {
|
|
|
53
55
|
const initializeCLI = () => {
|
|
54
56
|
program
|
|
55
57
|
.version(packageJson.version)
|
|
56
|
-
.description(
|
|
57
|
-
.argument(
|
|
58
|
-
.option(
|
|
59
|
-
.option(
|
|
60
|
-
.option(
|
|
61
|
-
.option(
|
|
58
|
+
.description("Clone specific folders from GitHub repositories")
|
|
59
|
+
.argument("<url>", "GitHub URL of the folder to clone")
|
|
60
|
+
.option("-o, --output <directory>", "Output directory", process.cwd())
|
|
61
|
+
.option("--zip [filename]", "Create ZIP archive of downloaded files")
|
|
62
|
+
.option("--tar [filename]", "Create TAR archive of downloaded files")
|
|
63
|
+
.option("--compression-level <level>", "Compression level (1-9)", "6")
|
|
62
64
|
.action(async (url, options) => {
|
|
63
65
|
try {
|
|
64
66
|
console.log(`Parsing URL: ${url}`);
|
|
65
67
|
const parsedUrl = parseGitHubUrl(url);
|
|
66
|
-
|
|
68
|
+
|
|
67
69
|
// Validate options
|
|
68
70
|
if (options.compressionLevel) {
|
|
69
71
|
const level = parseInt(options.compressionLevel, 10);
|
|
70
|
-
if (isNaN(level) || level <
|
|
71
|
-
|
|
72
|
+
if (isNaN(level) || level < 0 || level > 9) {
|
|
73
|
+
// Allow 0 for no compression
|
|
74
|
+
throw new Error(
|
|
75
|
+
"Compression level must be a number between 0 and 9"
|
|
76
|
+
);
|
|
72
77
|
}
|
|
73
78
|
}
|
|
74
79
|
|
|
75
80
|
if (options.zip && options.tar) {
|
|
76
|
-
throw new Error(
|
|
81
|
+
throw new Error(
|
|
82
|
+
"Cannot specify both --zip and --tar options at the same time"
|
|
83
|
+
);
|
|
77
84
|
}
|
|
78
|
-
|
|
85
|
+
|
|
79
86
|
// Validate output directory
|
|
80
87
|
try {
|
|
81
88
|
options.output = validateOutputDirectory(options.output);
|
|
82
89
|
} catch (dirError) {
|
|
83
90
|
throw new Error(`Output directory error: ${dirError.message}`);
|
|
84
91
|
}
|
|
85
|
-
|
|
92
|
+
|
|
86
93
|
// Handle archive options
|
|
87
|
-
const archiveFormat = options.zip ?
|
|
88
|
-
const archiveName =
|
|
89
|
-
|
|
94
|
+
const archiveFormat = options.zip ? "zip" : options.tar ? "tar" : null;
|
|
95
|
+
const archiveName =
|
|
96
|
+
typeof options.zip === "string"
|
|
97
|
+
? options.zip
|
|
98
|
+
: typeof options.tar === "string"
|
|
99
|
+
? options.tar
|
|
100
|
+
: null;
|
|
90
101
|
const compressionLevel = parseInt(options.compressionLevel, 10) || 6;
|
|
91
|
-
|
|
102
|
+
|
|
92
103
|
if (archiveFormat) {
|
|
93
104
|
console.log(`Creating ${archiveFormat.toUpperCase()} archive...`);
|
|
94
|
-
await downloadAndArchive(
|
|
105
|
+
await downloadAndArchive(
|
|
106
|
+
parsedUrl,
|
|
107
|
+
options.output,
|
|
108
|
+
archiveFormat,
|
|
109
|
+
archiveName,
|
|
110
|
+
compressionLevel
|
|
111
|
+
);
|
|
95
112
|
} else {
|
|
96
113
|
console.log(`Downloading folder to: ${options.output}`);
|
|
97
114
|
await downloadFolder(parsedUrl, options.output);
|
|
98
115
|
}
|
|
99
|
-
|
|
100
|
-
console.log(
|
|
116
|
+
|
|
117
|
+
console.log("Operation completed successfully!");
|
|
101
118
|
} catch (error) {
|
|
102
|
-
console.error(
|
|
119
|
+
console.error("Error:", error.message);
|
|
103
120
|
process.exit(1);
|
|
104
121
|
}
|
|
105
122
|
});
|
package/src/parser.js
CHANGED
|
@@ -1,27 +1,30 @@
|
|
|
1
1
|
export function parseGitHubUrl(url) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
// Validate the URL format
|
|
3
|
+
if (!url || typeof url !== "string") {
|
|
4
|
+
throw new Error("Invalid URL: URL must be a non-empty string");
|
|
5
|
+
}
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
// Validate if it's a GitHub URL
|
|
8
|
+
const githubUrlPattern =
|
|
9
|
+
/^https?:\/\/(?:www\.)?github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:tree|blob)\/([^\/]+)(?:\/(.+))?)?$/;
|
|
10
|
+
const match = url.match(githubUrlPattern);
|
|
10
11
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
if (!match) {
|
|
13
|
+
throw new Error(
|
|
14
|
+
"Invalid GitHub URL format. Expected: https://github.com/owner/repo[/tree|/blob]/branch/folder_or_file"
|
|
15
|
+
);
|
|
16
|
+
}
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
// Extract components from the matched pattern
|
|
19
|
+
const owner = match[1];
|
|
20
|
+
const repo = match[2];
|
|
21
|
+
const branch = match[3]; // Branch might not be in the URL for root downloads
|
|
22
|
+
const folderPath = match[4] || ""; // Empty string if no folder path
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
// Additional validation
|
|
25
|
+
if (!owner || !repo) {
|
|
26
|
+
throw new Error("Invalid GitHub URL: Missing repository owner or name");
|
|
27
|
+
}
|
|
25
28
|
|
|
26
|
-
|
|
29
|
+
return { owner, repo, branch, folderPath };
|
|
27
30
|
}
|