@neta-art/cohub-cli 3.6.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/work-download.d.ts +2 -1
- package/dist/work-download.js +11 -11
- package/package.json +1 -1
package/dist/work-download.d.ts
CHANGED
|
@@ -6,7 +6,8 @@ type DownloadResult = {
|
|
|
6
6
|
output: string;
|
|
7
7
|
files: number;
|
|
8
8
|
bytes: number;
|
|
9
|
-
verified:
|
|
9
|
+
verified: boolean;
|
|
10
|
+
unverifiedFiles: number;
|
|
10
11
|
};
|
|
11
12
|
export declare function downloadWork(detail: WorkGetResponse, outputOption?: string, fetcher?: typeof fetch): Promise<DownloadResult>;
|
|
12
13
|
export {};
|
package/dist/work-download.js
CHANGED
|
@@ -5,6 +5,7 @@ import { basename, dirname, join, posix, resolve } from "node:path";
|
|
|
5
5
|
import { Readable, Transform } from "node:stream";
|
|
6
6
|
import { pipeline } from "node:stream/promises";
|
|
7
7
|
const MANIFEST_MAX_BYTES = 4 * 1024 * 1024;
|
|
8
|
+
const MAX_CONTENT_FILE_BYTES = 1024 * 1024 * 1024;
|
|
8
9
|
const DOWNLOAD_CONCURRENCY = 4;
|
|
9
10
|
function safeRelativePath(value, label) {
|
|
10
11
|
if (!value || value.includes("\\") || value.includes("\0") || posix.isAbsolute(value)) {
|
|
@@ -110,22 +111,18 @@ async function downloadFile(url, output, expected, fetcher) {
|
|
|
110
111
|
throw new Error(`Failed to download ${expected.outputPath} (${response.status})`);
|
|
111
112
|
await mkdir(dirname(output), { recursive: true });
|
|
112
113
|
let sizeBytes = 0;
|
|
113
|
-
const hash = createHash("sha256");
|
|
114
114
|
const verifier = new Transform({
|
|
115
115
|
transform(chunk, _encoding, callback) {
|
|
116
116
|
sizeBytes += chunk.byteLength;
|
|
117
|
-
if (sizeBytes >
|
|
118
|
-
callback(new Error(`Downloaded file
|
|
117
|
+
if (sizeBytes > MAX_CONTENT_FILE_BYTES) {
|
|
118
|
+
callback(new Error(`Downloaded file is too large: ${expected.outputPath}`));
|
|
119
119
|
return;
|
|
120
120
|
}
|
|
121
|
-
hash.update(chunk);
|
|
122
121
|
callback(null, chunk);
|
|
123
122
|
},
|
|
124
123
|
});
|
|
125
124
|
await pipeline(Readable.fromWeb(response.body), verifier, createWriteStream(output, { flags: "wx" }));
|
|
126
|
-
|
|
127
|
-
throw new Error(`Downloaded file verification failed: ${expected.outputPath}`);
|
|
128
|
-
}
|
|
125
|
+
return sizeBytes;
|
|
129
126
|
}
|
|
130
127
|
function outputExistsError(output) {
|
|
131
128
|
return new Error(`Output already exists: ${output}`);
|
|
@@ -181,6 +178,7 @@ async function installDirectoryNoReplace(stage, output) {
|
|
|
181
178
|
}
|
|
182
179
|
}
|
|
183
180
|
async function downloadFiles(input) {
|
|
181
|
+
const downloadedBytes = new Array(input.files.length);
|
|
184
182
|
let next = 0;
|
|
185
183
|
let failed = false;
|
|
186
184
|
let failure;
|
|
@@ -191,7 +189,7 @@ async function downloadFiles(input) {
|
|
|
191
189
|
if (!file)
|
|
192
190
|
continue;
|
|
193
191
|
try {
|
|
194
|
-
await downloadFile(artifactUrl(input.contentUrl, file.artifactPath), join(input.stage, ...file.outputPath.split("/")), file, input.fetcher);
|
|
192
|
+
downloadedBytes[index] = await downloadFile(artifactUrl(input.contentUrl, file.artifactPath), join(input.stage, ...file.outputPath.split("/")), file, input.fetcher);
|
|
195
193
|
}
|
|
196
194
|
catch (cause) {
|
|
197
195
|
if (!failed)
|
|
@@ -203,6 +201,7 @@ async function downloadFiles(input) {
|
|
|
203
201
|
await Promise.all(workers);
|
|
204
202
|
if (failed)
|
|
205
203
|
throw failure;
|
|
204
|
+
return downloadedBytes.reduce((sum, size) => sum + size, 0);
|
|
206
205
|
}
|
|
207
206
|
export async function downloadWork(detail, outputOption, fetcher = fetch) {
|
|
208
207
|
const { work, content } = detail;
|
|
@@ -228,7 +227,7 @@ export async function downloadWork(detail, outputOption, fetcher = fetch) {
|
|
|
228
227
|
const stage = await mkdtemp(join(dirname(output), `.${basename(output)}.cohub-download-`));
|
|
229
228
|
try {
|
|
230
229
|
const files = hasDirectoryOutput ? manifest.files : [entry];
|
|
231
|
-
await downloadFiles({ files, contentUrl: content.url, stage, fetcher });
|
|
230
|
+
const downloadedBytes = await downloadFiles({ files, contentUrl: content.url, stage, fetcher });
|
|
232
231
|
if (hasDirectoryOutput) {
|
|
233
232
|
await installDirectoryNoReplace(stage, output);
|
|
234
233
|
}
|
|
@@ -243,8 +242,9 @@ export async function downloadWork(detail, outputOption, fetcher = fetch) {
|
|
|
243
242
|
kind: hasDirectoryOutput ? "directory" : "file",
|
|
244
243
|
output,
|
|
245
244
|
files: files.length,
|
|
246
|
-
bytes:
|
|
247
|
-
verified:
|
|
245
|
+
bytes: downloadedBytes,
|
|
246
|
+
verified: false,
|
|
247
|
+
unverifiedFiles: files.length,
|
|
248
248
|
};
|
|
249
249
|
}
|
|
250
250
|
catch (cause) {
|