git-ripper 1.5.1 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -20
- package/README.md +340 -340
- package/bin/git-ripper.js +3 -3
- package/package.json +62 -62
- package/src/archiver.js +210 -210
- package/src/downloader.js +904 -904
- package/src/index.js +195 -195
- package/src/parser.js +37 -37
- package/src/resumeManager.js +213 -213
package/src/resumeManager.js
CHANGED
|
@@ -1,213 +1,213 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import crypto from "node:crypto";
|
|
4
|
-
import { fileURLToPath } from "node:url";
|
|
5
|
-
import { dirname } from "node:path";
|
|
6
|
-
|
|
7
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
8
|
-
const __dirname = dirname(__filename);
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* Manages download checkpoints for resuming interrupted downloads
|
|
12
|
-
*/
|
|
13
|
-
export class ResumeManager {
|
|
14
|
-
constructor(checkpointDir = ".git-ripper-checkpoints") {
|
|
15
|
-
this.checkpointDir = path.resolve(checkpointDir);
|
|
16
|
-
this.ensureCheckpointDir();
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Ensure checkpoint directory exists
|
|
21
|
-
*/
|
|
22
|
-
ensureCheckpointDir() {
|
|
23
|
-
if (!fs.existsSync(this.checkpointDir)) {
|
|
24
|
-
fs.mkdirSync(this.checkpointDir, { recursive: true });
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Create unique checkpoint ID based on URL and output directory
|
|
30
|
-
* @param {string} url - GitHub URL
|
|
31
|
-
* @param {string} outputDir - Output directory path
|
|
32
|
-
* @returns {string} - Unique checkpoint ID
|
|
33
|
-
*/
|
|
34
|
-
createCheckpointId(url, outputDir) {
|
|
35
|
-
const combined = `${url}|${path.resolve(outputDir)}`;
|
|
36
|
-
return crypto
|
|
37
|
-
.createHash("md5")
|
|
38
|
-
.update(combined)
|
|
39
|
-
.digest("hex")
|
|
40
|
-
.substring(0, 12);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Save download progress to checkpoint file
|
|
45
|
-
* @param {Object} checkpoint - Checkpoint data
|
|
46
|
-
* @returns {string} - Checkpoint ID
|
|
47
|
-
*/
|
|
48
|
-
saveCheckpoint(checkpoint) {
|
|
49
|
-
const checkpointId = this.createCheckpointId(
|
|
50
|
-
checkpoint.url,
|
|
51
|
-
checkpoint.outputDir
|
|
52
|
-
);
|
|
53
|
-
const checkpointFile = path.join(
|
|
54
|
-
this.checkpointDir,
|
|
55
|
-
`${checkpointId}.json`
|
|
56
|
-
);
|
|
57
|
-
|
|
58
|
-
const checkpointData = {
|
|
59
|
-
...checkpoint,
|
|
60
|
-
timestamp: new Date().toISOString(),
|
|
61
|
-
checkpointId,
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
try {
|
|
65
|
-
fs.writeFileSync(checkpointFile, JSON.stringify(checkpointData, null, 2));
|
|
66
|
-
return checkpointId;
|
|
67
|
-
} catch (error) {
|
|
68
|
-
console.error(`Failed to save checkpoint: ${error.message}`);
|
|
69
|
-
return null;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Load existing checkpoint if available
|
|
75
|
-
* @param {string} url - GitHub URL
|
|
76
|
-
* @param {string} outputDir - Output directory path
|
|
77
|
-
* @returns {Object|null} - Checkpoint data or null if not found
|
|
78
|
-
*/
|
|
79
|
-
loadCheckpoint(url, outputDir) {
|
|
80
|
-
const checkpointId = this.createCheckpointId(url, outputDir);
|
|
81
|
-
const checkpointFile = path.join(
|
|
82
|
-
this.checkpointDir,
|
|
83
|
-
`${checkpointId}.json`
|
|
84
|
-
);
|
|
85
|
-
|
|
86
|
-
if (!fs.existsSync(checkpointFile)) {
|
|
87
|
-
return null;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
try {
|
|
91
|
-
const data = fs.readFileSync(checkpointFile, "utf8");
|
|
92
|
-
return JSON.parse(data);
|
|
93
|
-
} catch (error) {
|
|
94
|
-
console.error(`Error loading checkpoint: ${error.message}`);
|
|
95
|
-
return null;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Verify downloaded file hasn't been corrupted
|
|
101
|
-
* @param {string} filepath - Path to the file
|
|
102
|
-
* @param {string} expectedHash - Expected MD5 hash
|
|
103
|
-
* @returns {boolean} - True if file is valid
|
|
104
|
-
*/
|
|
105
|
-
verifyFileIntegrity(filepath, expectedHash) {
|
|
106
|
-
if (!fs.existsSync(filepath)) {
|
|
107
|
-
return false;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
try {
|
|
111
|
-
const fileContent = fs.readFileSync(filepath);
|
|
112
|
-
const actualHash = crypto
|
|
113
|
-
.createHash("md5")
|
|
114
|
-
.update(fileContent)
|
|
115
|
-
.digest("hex");
|
|
116
|
-
return actualHash === expectedHash;
|
|
117
|
-
} catch (error) {
|
|
118
|
-
return false;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
/**
|
|
123
|
-
* Calculate MD5 hash of file content
|
|
124
|
-
* @param {Buffer} content - File content
|
|
125
|
-
* @returns {string} - MD5 hash
|
|
126
|
-
*/
|
|
127
|
-
calculateHash(content) {
|
|
128
|
-
return crypto.createHash("md5").update(content).digest("hex");
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* Remove checkpoint file after successful completion
|
|
133
|
-
* @param {string} url - GitHub URL
|
|
134
|
-
* @param {string} outputDir - Output directory path
|
|
135
|
-
*/
|
|
136
|
-
cleanupCheckpoint(url, outputDir) {
|
|
137
|
-
const checkpointId = this.createCheckpointId(url, outputDir);
|
|
138
|
-
const checkpointFile = path.join(
|
|
139
|
-
this.checkpointDir,
|
|
140
|
-
`${checkpointId}.json`
|
|
141
|
-
);
|
|
142
|
-
|
|
143
|
-
if (fs.existsSync(checkpointFile)) {
|
|
144
|
-
try {
|
|
145
|
-
fs.unlinkSync(checkpointFile);
|
|
146
|
-
if (fs.readdirSync(this.checkpointDir).length === 0) {
|
|
147
|
-
fs.rmdirSync(this.checkpointDir);
|
|
148
|
-
}
|
|
149
|
-
} catch (error) {
|
|
150
|
-
console.error(`Failed to cleanup checkpoint: ${error.message}`);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
/**
|
|
156
|
-
* List all existing checkpoints
|
|
157
|
-
* @returns {Array} - Array of checkpoint information
|
|
158
|
-
*/
|
|
159
|
-
listCheckpoints() {
|
|
160
|
-
if (!fs.existsSync(this.checkpointDir)) {
|
|
161
|
-
return [];
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
try {
|
|
165
|
-
const files = fs.readdirSync(this.checkpointDir);
|
|
166
|
-
const checkpoints = [];
|
|
167
|
-
|
|
168
|
-
for (const file of files) {
|
|
169
|
-
if (file.endsWith(".json")) {
|
|
170
|
-
try {
|
|
171
|
-
const filepath = path.join(this.checkpointDir, file);
|
|
172
|
-
const data = JSON.parse(fs.readFileSync(filepath, "utf8"));
|
|
173
|
-
checkpoints.push({
|
|
174
|
-
id: data.checkpointId,
|
|
175
|
-
url: data.url,
|
|
176
|
-
outputDir: data.outputDir,
|
|
177
|
-
timestamp: data.timestamp,
|
|
178
|
-
progress: `${data.downloadedFiles.length}/${data.totalFiles}`,
|
|
179
|
-
failedFiles: data.failedFiles.length,
|
|
180
|
-
});
|
|
181
|
-
} catch (error) {
|
|
182
|
-
// Skip corrupted checkpoint files
|
|
183
|
-
continue;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
return checkpoints;
|
|
189
|
-
} catch (error) {
|
|
190
|
-
console.error(`Failed to list checkpoints: ${error.message}`);
|
|
191
|
-
return [];
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Create a new checkpoint object
|
|
197
|
-
* @param {string} url - GitHub URL
|
|
198
|
-
* @param {string} outputDir - Output directory
|
|
199
|
-
* @param {number} totalFiles - Total number of files to download
|
|
200
|
-
* @returns {Object} - New checkpoint object
|
|
201
|
-
*/
|
|
202
|
-
createNewCheckpoint(url, outputDir, totalFiles) {
|
|
203
|
-
return {
|
|
204
|
-
url,
|
|
205
|
-
outputDir: path.resolve(outputDir),
|
|
206
|
-
totalFiles,
|
|
207
|
-
downloadedFiles: [],
|
|
208
|
-
failedFiles: [],
|
|
209
|
-
fileHashes: {},
|
|
210
|
-
timestamp: new Date().toISOString(),
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
}
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import crypto from "node:crypto";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { dirname } from "node:path";
|
|
6
|
+
|
|
7
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
8
|
+
const __dirname = dirname(__filename);
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Manages download checkpoints for resuming interrupted downloads
|
|
12
|
+
*/
|
|
13
|
+
export class ResumeManager {
|
|
14
|
+
constructor(checkpointDir = ".git-ripper-checkpoints") {
|
|
15
|
+
this.checkpointDir = path.resolve(checkpointDir);
|
|
16
|
+
this.ensureCheckpointDir();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Ensure checkpoint directory exists
|
|
21
|
+
*/
|
|
22
|
+
ensureCheckpointDir() {
|
|
23
|
+
if (!fs.existsSync(this.checkpointDir)) {
|
|
24
|
+
fs.mkdirSync(this.checkpointDir, { recursive: true });
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Create unique checkpoint ID based on URL and output directory
|
|
30
|
+
* @param {string} url - GitHub URL
|
|
31
|
+
* @param {string} outputDir - Output directory path
|
|
32
|
+
* @returns {string} - Unique checkpoint ID
|
|
33
|
+
*/
|
|
34
|
+
createCheckpointId(url, outputDir) {
|
|
35
|
+
const combined = `${url}|${path.resolve(outputDir)}`;
|
|
36
|
+
return crypto
|
|
37
|
+
.createHash("md5")
|
|
38
|
+
.update(combined)
|
|
39
|
+
.digest("hex")
|
|
40
|
+
.substring(0, 12);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Save download progress to checkpoint file
|
|
45
|
+
* @param {Object} checkpoint - Checkpoint data
|
|
46
|
+
* @returns {string} - Checkpoint ID
|
|
47
|
+
*/
|
|
48
|
+
saveCheckpoint(checkpoint) {
|
|
49
|
+
const checkpointId = this.createCheckpointId(
|
|
50
|
+
checkpoint.url,
|
|
51
|
+
checkpoint.outputDir
|
|
52
|
+
);
|
|
53
|
+
const checkpointFile = path.join(
|
|
54
|
+
this.checkpointDir,
|
|
55
|
+
`${checkpointId}.json`
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
const checkpointData = {
|
|
59
|
+
...checkpoint,
|
|
60
|
+
timestamp: new Date().toISOString(),
|
|
61
|
+
checkpointId,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
fs.writeFileSync(checkpointFile, JSON.stringify(checkpointData, null, 2));
|
|
66
|
+
return checkpointId;
|
|
67
|
+
} catch (error) {
|
|
68
|
+
console.error(`Failed to save checkpoint: ${error.message}`);
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Load existing checkpoint if available
|
|
75
|
+
* @param {string} url - GitHub URL
|
|
76
|
+
* @param {string} outputDir - Output directory path
|
|
77
|
+
* @returns {Object|null} - Checkpoint data or null if not found
|
|
78
|
+
*/
|
|
79
|
+
loadCheckpoint(url, outputDir) {
|
|
80
|
+
const checkpointId = this.createCheckpointId(url, outputDir);
|
|
81
|
+
const checkpointFile = path.join(
|
|
82
|
+
this.checkpointDir,
|
|
83
|
+
`${checkpointId}.json`
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
if (!fs.existsSync(checkpointFile)) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
try {
|
|
91
|
+
const data = fs.readFileSync(checkpointFile, "utf8");
|
|
92
|
+
return JSON.parse(data);
|
|
93
|
+
} catch (error) {
|
|
94
|
+
console.error(`Error loading checkpoint: ${error.message}`);
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Verify downloaded file hasn't been corrupted
|
|
101
|
+
* @param {string} filepath - Path to the file
|
|
102
|
+
* @param {string} expectedHash - Expected MD5 hash
|
|
103
|
+
* @returns {boolean} - True if file is valid
|
|
104
|
+
*/
|
|
105
|
+
verifyFileIntegrity(filepath, expectedHash) {
|
|
106
|
+
if (!fs.existsSync(filepath)) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
const fileContent = fs.readFileSync(filepath);
|
|
112
|
+
const actualHash = crypto
|
|
113
|
+
.createHash("md5")
|
|
114
|
+
.update(fileContent)
|
|
115
|
+
.digest("hex");
|
|
116
|
+
return actualHash === expectedHash;
|
|
117
|
+
} catch (error) {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Calculate MD5 hash of file content
|
|
124
|
+
* @param {Buffer} content - File content
|
|
125
|
+
* @returns {string} - MD5 hash
|
|
126
|
+
*/
|
|
127
|
+
calculateHash(content) {
|
|
128
|
+
return crypto.createHash("md5").update(content).digest("hex");
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Remove checkpoint file after successful completion
|
|
133
|
+
* @param {string} url - GitHub URL
|
|
134
|
+
* @param {string} outputDir - Output directory path
|
|
135
|
+
*/
|
|
136
|
+
cleanupCheckpoint(url, outputDir) {
|
|
137
|
+
const checkpointId = this.createCheckpointId(url, outputDir);
|
|
138
|
+
const checkpointFile = path.join(
|
|
139
|
+
this.checkpointDir,
|
|
140
|
+
`${checkpointId}.json`
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
if (fs.existsSync(checkpointFile)) {
|
|
144
|
+
try {
|
|
145
|
+
fs.unlinkSync(checkpointFile);
|
|
146
|
+
if (fs.readdirSync(this.checkpointDir).length === 0) {
|
|
147
|
+
fs.rmdirSync(this.checkpointDir);
|
|
148
|
+
}
|
|
149
|
+
} catch (error) {
|
|
150
|
+
console.error(`Failed to cleanup checkpoint: ${error.message}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* List all existing checkpoints
|
|
157
|
+
* @returns {Array} - Array of checkpoint information
|
|
158
|
+
*/
|
|
159
|
+
listCheckpoints() {
|
|
160
|
+
if (!fs.existsSync(this.checkpointDir)) {
|
|
161
|
+
return [];
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
try {
|
|
165
|
+
const files = fs.readdirSync(this.checkpointDir);
|
|
166
|
+
const checkpoints = [];
|
|
167
|
+
|
|
168
|
+
for (const file of files) {
|
|
169
|
+
if (file.endsWith(".json")) {
|
|
170
|
+
try {
|
|
171
|
+
const filepath = path.join(this.checkpointDir, file);
|
|
172
|
+
const data = JSON.parse(fs.readFileSync(filepath, "utf8"));
|
|
173
|
+
checkpoints.push({
|
|
174
|
+
id: data.checkpointId,
|
|
175
|
+
url: data.url,
|
|
176
|
+
outputDir: data.outputDir,
|
|
177
|
+
timestamp: data.timestamp,
|
|
178
|
+
progress: `${data.downloadedFiles.length}/${data.totalFiles}`,
|
|
179
|
+
failedFiles: data.failedFiles.length,
|
|
180
|
+
});
|
|
181
|
+
} catch (error) {
|
|
182
|
+
// Skip corrupted checkpoint files
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
return checkpoints;
|
|
189
|
+
} catch (error) {
|
|
190
|
+
console.error(`Failed to list checkpoints: ${error.message}`);
|
|
191
|
+
return [];
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Create a new checkpoint object
|
|
197
|
+
* @param {string} url - GitHub URL
|
|
198
|
+
* @param {string} outputDir - Output directory
|
|
199
|
+
* @param {number} totalFiles - Total number of files to download
|
|
200
|
+
* @returns {Object} - New checkpoint object
|
|
201
|
+
*/
|
|
202
|
+
createNewCheckpoint(url, outputDir, totalFiles) {
|
|
203
|
+
return {
|
|
204
|
+
url,
|
|
205
|
+
outputDir: path.resolve(outputDir),
|
|
206
|
+
totalFiles,
|
|
207
|
+
downloadedFiles: [],
|
|
208
|
+
failedFiles: [],
|
|
209
|
+
fileHashes: {},
|
|
210
|
+
timestamp: new Date().toISOString(),
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|