@alwaysmeticulous/downloading-helpers 2.271.0 → 2.272.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/file-downloads/__tests__/download-file.spec.js +54 -13
- package/dist/file-downloads/__tests__/download-file.spec.js.map +1 -1
- package/dist/file-downloads/download-file.d.ts +20 -12
- package/dist/file-downloads/download-file.js +106 -30
- package/dist/file-downloads/download-file.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="
|
|
2
|
+
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="c1396eb4-872c-5c11-8991-cedd1f0683ab")}catch(e){}}();
|
|
3
3
|
|
|
4
4
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
5
5
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
@@ -122,15 +122,13 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
122
122
|
}
|
|
123
123
|
return Buffer.concat(chunks);
|
|
124
124
|
};
|
|
125
|
-
(0, vitest_1.describe)("
|
|
125
|
+
(0, vitest_1.describe)("streamDownloadAndExtractTar", () => {
|
|
126
126
|
let sourceDir;
|
|
127
127
|
let extractDir;
|
|
128
|
-
let tmpTarPath;
|
|
129
128
|
(0, vitest_1.beforeEach)(async () => {
|
|
130
|
-
const base = (0, path_1.join)((0, os_1.tmpdir)(), `test-tar
|
|
129
|
+
const base = (0, path_1.join)((0, os_1.tmpdir)(), `test-stream-tar-${Date.now()}`);
|
|
131
130
|
sourceDir = (0, path_1.join)(base, "source");
|
|
132
131
|
extractDir = (0, path_1.join)(base, "extract");
|
|
133
|
-
tmpTarPath = (0, path_1.join)(base, "tmp.tar.d");
|
|
134
132
|
await (0, promises_1.mkdir)(sourceDir, { recursive: true });
|
|
135
133
|
});
|
|
136
134
|
(0, vitest_1.afterEach)(async () => {
|
|
@@ -142,7 +140,7 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
142
140
|
// Ignore cleanup errors
|
|
143
141
|
}
|
|
144
142
|
});
|
|
145
|
-
(0, vitest_1.it)("
|
|
143
|
+
(0, vitest_1.it)("streams and extracts files without a temp file", async () => {
|
|
146
144
|
await (0, promises_1.writeFile)((0, path_1.join)(sourceDir, "hello.txt"), "Hello World");
|
|
147
145
|
await (0, promises_1.writeFile)((0, path_1.join)(sourceDir, "data.json"), '{"key": "value"}');
|
|
148
146
|
await (0, promises_1.mkdir)((0, path_1.join)(sourceDir, "subdir"), { recursive: true });
|
|
@@ -153,8 +151,8 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
153
151
|
res.end(compressed);
|
|
154
152
|
});
|
|
155
153
|
try {
|
|
156
|
-
await new Promise((resolve) => server.listen(
|
|
157
|
-
const entries = await (0, download_file_1.
|
|
154
|
+
await new Promise((resolve) => server.listen(1237, resolve));
|
|
155
|
+
const entries = await (0, download_file_1.streamDownloadAndExtractTar)("http://localhost:1237", extractDir);
|
|
158
156
|
(0, vitest_1.expect)(entries.length).toBeGreaterThan(0);
|
|
159
157
|
const hello = await (0, promises_1.readFile)((0, path_1.join)(extractDir, "hello.txt"), "utf8");
|
|
160
158
|
(0, vitest_1.expect)(hello).toBe("Hello World");
|
|
@@ -162,13 +160,12 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
162
160
|
(0, vitest_1.expect)(data).toBe('{"key": "value"}');
|
|
163
161
|
const nested = await (0, promises_1.readFile)((0, path_1.join)(extractDir, "subdir", "nested.txt"), "utf8");
|
|
164
162
|
(0, vitest_1.expect)(nested).toBe("Nested!");
|
|
165
|
-
(0, vitest_1.expect)((0, fs_1.existsSync)(tmpTarPath)).toBe(false);
|
|
166
163
|
}
|
|
167
164
|
finally {
|
|
168
165
|
server.close();
|
|
169
166
|
}
|
|
170
167
|
});
|
|
171
|
-
(0, vitest_1.it)("
|
|
168
|
+
(0, vitest_1.it)("streams and extracts a large file", async () => {
|
|
172
169
|
const largeContent = "A".repeat(5 * 1024 * 1024);
|
|
173
170
|
await (0, promises_1.writeFile)((0, path_1.join)(sourceDir, "large.txt"), largeContent);
|
|
174
171
|
const compressed = await createRawDeflatedTar(sourceDir);
|
|
@@ -177,8 +174,8 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
177
174
|
res.end(compressed);
|
|
178
175
|
});
|
|
179
176
|
try {
|
|
180
|
-
await new Promise((resolve) => server.listen(
|
|
181
|
-
await (0, download_file_1.
|
|
177
|
+
await new Promise((resolve) => server.listen(1238, resolve));
|
|
178
|
+
await (0, download_file_1.streamDownloadAndExtractTar)("http://localhost:1238", extractDir);
|
|
182
179
|
const result = await (0, promises_1.readFile)((0, path_1.join)(extractDir, "large.txt"), "utf8");
|
|
183
180
|
(0, vitest_1.expect)(result).toBe(largeContent);
|
|
184
181
|
}
|
|
@@ -186,6 +183,50 @@ const createRawDeflatedTar = async (folderPath) => {
|
|
|
186
183
|
server.close();
|
|
187
184
|
}
|
|
188
185
|
});
|
|
186
|
+
(0, vitest_1.it)("times out on a stalled download", async () => {
|
|
187
|
+
const server = http_1.default.createServer((_req, res) => {
|
|
188
|
+
res.writeHead(200, { "Content-Type": "application/octet-stream" });
|
|
189
|
+
});
|
|
190
|
+
await new Promise((resolve) => server.listen(1239, resolve));
|
|
191
|
+
let caughtError = null;
|
|
192
|
+
try {
|
|
193
|
+
await (0, download_file_1.streamDownloadAndExtractTar)("http://localhost:1239", extractDir, {
|
|
194
|
+
totalTimeoutInMs: 100,
|
|
195
|
+
maxRetries: 0,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
catch (err) {
|
|
199
|
+
caughtError = err;
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
server.closeAllConnections();
|
|
203
|
+
server.close();
|
|
204
|
+
}
|
|
205
|
+
(0, vitest_1.expect)(caughtError).not.toBeNull();
|
|
206
|
+
(0, vitest_1.expect)(caughtError.message).toContain("timed out");
|
|
207
|
+
}, 15000);
|
|
208
|
+
(0, vitest_1.it)("preserves the underlying error when retries are exhausted (not generic AbortError)", async () => {
|
|
209
|
+
const server = http_1.default.createServer((_req, res) => {
|
|
210
|
+
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
211
|
+
res.end("internal server error");
|
|
212
|
+
});
|
|
213
|
+
await new Promise((resolve) => server.listen(1240, resolve));
|
|
214
|
+
let caughtError = null;
|
|
215
|
+
try {
|
|
216
|
+
await (0, download_file_1.streamDownloadAndExtractTar)("http://localhost:1240", extractDir, {
|
|
217
|
+
maxRetries: 0,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
caughtError = err;
|
|
222
|
+
}
|
|
223
|
+
finally {
|
|
224
|
+
server.close();
|
|
225
|
+
}
|
|
226
|
+
(0, vitest_1.expect)(caughtError).not.toBeNull();
|
|
227
|
+
(0, vitest_1.expect)(caughtError.name).not.toBe("AbortError");
|
|
228
|
+
(0, vitest_1.expect)(caughtError.message).toMatch(/500/);
|
|
229
|
+
});
|
|
189
230
|
});
|
|
190
231
|
//# sourceMappingURL=download-file.spec.js.map
|
|
191
|
-
//# debugId=
|
|
232
|
+
//# debugId=c1396eb4-872c-5c11-8991-cedd1f0683ab
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-file.spec.js","sources":["../../../src/file-downloads/__tests__/download-file.spec.ts"],"sourceRoot":"","names":[],"mappings":";;;;;;;AAAA,2BAAgC;AAChC,0CAA6D;AAC7D,gDAAwB;AACxB,2BAA4B;AAC5B,+BAA4B;AAC5B,+BAAkD;AAClD,yCAAuC;AACvC,6BAA0C;AAC1C,mCAAqE;AACrE,
|
|
1
|
+
{"version":3,"file":"download-file.spec.js","sources":["../../../src/file-downloads/__tests__/download-file.spec.ts"],"sourceRoot":"","names":[],"mappings":";;;;;;;AAAA,2BAAgC;AAChC,0CAA6D;AAC7D,gDAAwB;AACxB,2BAA4B;AAC5B,+BAA4B;AAC5B,+BAAkD;AAClD,yCAAuC;AACvC,6BAA0C;AAC1C,mCAAqE;AACrE,oDAA6E;AAE7E,IAAA,iBAAQ,EAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAA,WAAE,EAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5C,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAEpB,MAAM,IAAA,4BAAY,EAAC,uBAAuB,EAAE,UAAU,EAAE;gBACtD,oBAAoB,EAAE,IAAI;gBAC1B,2BAA2B,EAAE,IAAI;aAClC,CAAC,CAAC;YAEH,qBAAqB;YACrB,MAAM,YAAY,GAAG,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACxD,IAAA,eAAM,EAAC,YAAY,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,IAAA,eAAU,EAAC,UAAU,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAA,aAAE,EAAC,UAAU,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5C,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACd,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YAC7B,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAEpB,MAAM,IAAA,4BAAY,EAAC,uBAAuB,EAAE,UAAU,EAAE;gBACtD,oBAAoB,EAAE,IAAI;gBAC1B,2BAA2B,EAAE,CAAC;gBAC9B,yBAAyB,EAAE,CAAC;gBAC5B,yBAAyB,EAAE,CAAC;aAC7B,CAAC,CAAC;YAEH,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAA,eAAM,EAAE,CAAW,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QACvE,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;YAEf,IAAI,IAAA,eAAU,EAAC,UAAU,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAA,aAAE,EAAC,UAAU,CAAC,CAAC;gBACrB,6CAA6C;gBAC7C,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5C,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC;YAED,OAAO,EAAE,CAAC;YACV,UAAU,CAAC,GAAG,EAAE;gBACd,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YAC7B,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAEpB,MAAM,IAAA,4BAAY,EAAC,uBAAuB,EAAE,UAAU,EAAE;gBACtD,oBAAoB,EAAE,IAAI;gBAC1B,2BAA2B,EAAE,EAAE;gBAC/B,yBAAyB,EAAE,CAAC;gBAC5B,yBAAyB,EAAE,CAAC;aAC7B,CAAC,CAAC;YAEH,qBAAqB;YACrB,MAAM,YAAY,GAAG,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACxD,IAAA,eAAM,EAAC,YAAY,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,IAAA,eAAU,EAAC,UAAU,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAA,aAAE,EAAC,UAAU,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,oBAAoB,GAAG,KAAK,EAAE,UAAkB,EAAmB,EAAE;IACzE,MAAM,OAAO,GAAG,IAAI,sBAAU,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,SAAS,GAAG,IAAA,YAAS,EAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACrC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7B,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,gBAAa,CAAC,QAAQ,CAAC,CAAC;IAC5E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,IAAA,iBAAQ,EAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,IAAI,SAAiB,CAAC;IACtB,IAAI,UAAkB,CAAC;IAEvB,IAAA,mBAAU,EAAC,KAAK,IAAI,EAAE;QACpB,MAAM,IAAI,GAAG,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,SAAS,GAAG,IAAA,WAAI,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACjC,UAAU,GAAG,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACnC,MAAM,IAAA,gBAAK,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,KAAK,IAAI,EAAE;QACnB,MAAM,IAAI,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAA,aAAE,EAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,WAAW,CAAC,EAAE,aAAa,CAAC,CAAC;QAC7D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,WAAW,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAClE,MAAM,IAAA,gBAAK,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,SAAS,CAAC,CAAC;QAEpE,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC7C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACvE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAEnE,MAAM,OAAO,GAAG,MAAM,IAAA,2CAA2B,EAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;YAEvF,IAAA,eAAM,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAE1C,MAAM,KAAK,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAA,WAAI,EAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;YACpE,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAElC,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAA,WAAI,EAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;YACnE,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAA,WAAI,EAAC,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC;YAChF,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QACjD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,WAAW,CAAC,EAAE,YAAY,CAAC,CAAC;QAE5D,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC7C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACvE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAEnE,MAAM,IAAA,2CAA2B,EAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;YAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAA,WAAI,EAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;YACrE,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC7C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAEnE,IAAI,WAAW,GAAiB,IAAI,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,IAAA,2CAA2B,EAAC,uBAAuB,EAAE,UAAU,EAAE;gBACrE,gBAAgB,EAAE,GAAG;gBACrB,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,WAAW,GAAG,GAAY,CAAC;QAC7B,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC7B,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;QAED,IAAA,eAAM,EAAC,WAAW,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAA,eAAM,EAAC,WAAY,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACtD,CAAC,EAAE,KAAM,CAAC,CAAC;IAEX,IAAA,WAAE,EAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;QAClG,MAAM,MAAM,GAAG,cAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC7C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC,CAAC;YACrD,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAEnE,IAAI,WAAW,GAAiB,IAAI,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,IAAA,2CAA2B,EAAC,uBAAuB,EAAE,UAAU,EAAE;gBACrE,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,WAAW,GAAG,GAAY,CAAC;QAC7B,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;QAED,IAAA,eAAM,EAAC,WAAW,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAA,eAAM,EAAC,WAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,WAAY,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","debugId":"c1396eb4-872c-5c11-8991-cedd1f0683ab"}
|
|
@@ -25,20 +25,28 @@ export declare const downloadFile: (fileUrl: string, path: string, opts?: Downlo
|
|
|
25
25
|
* @returns The list of the extracted files.
|
|
26
26
|
*/
|
|
27
27
|
export declare const downloadAndExtractFile: (fileUrl: string, tmpZipFilePath: string, extractPath: string, extractTimeoutInMs?: number) => Promise<string[]>;
|
|
28
|
+
export interface StreamDownloadAndExtractTarOptions {
|
|
29
|
+
firstDataTimeoutInMs?: number;
|
|
30
|
+
totalTimeoutInMs?: number;
|
|
31
|
+
maxRetries?: number;
|
|
32
|
+
retryDelay?: number;
|
|
33
|
+
}
|
|
28
34
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
35
|
+
* Streams a raw-deflated tar blob from a URL directly through inflate and
|
|
36
|
+
* tar extraction without writing the archive to disk first.
|
|
37
|
+
*
|
|
38
|
+
* This eliminates the temp file that the legacy `downloadAndExtractTar` used,
|
|
39
|
+
* halving disk I/O and allowing download and extraction to overlap. Uses
|
|
40
|
+
* fast-zlib for decompression and tuned buffer sizes for better throughput
|
|
41
|
+
* on large (>1GB) files.
|
|
42
|
+
*
|
|
43
|
+
* __Warning__: this function is not thread safe. Do not extract to a
|
|
44
|
+
* directory that may already be in use by another process.
|
|
33
45
|
*
|
|
34
46
|
* @param fileUrl The URL of the deflated tar blob to download.
|
|
35
|
-
* @param
|
|
36
|
-
*
|
|
37
|
-
* @
|
|
38
|
-
* Do not try extracting to a dir that may already be in use by another process b/c overlapping
|
|
39
|
-
* file names can cause data corruption.
|
|
40
|
-
* @param extractTimeoutInMs The timeout for the tar extraction, in milliseconds.
|
|
41
|
-
* @returns The list of the extracted files.
|
|
47
|
+
* @param extractPath The directory to extract tar entries into.
|
|
48
|
+
* @param opts Timeout, retry, and buffer configuration.
|
|
49
|
+
* @returns The list of extracted file paths.
|
|
42
50
|
*/
|
|
43
|
-
export declare const
|
|
51
|
+
export declare const streamDownloadAndExtractTar: (fileUrl: string, extractPath: string, opts?: StreamDownloadAndExtractTarOptions) => Promise<string[]>;
|
|
44
52
|
export {};
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="
|
|
2
|
+
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="49217d9d-c552-53ec-93e8-6cbb59e2de04")}catch(e){}}();
|
|
3
3
|
|
|
4
4
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
5
5
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
6
6
|
};
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
-
exports.
|
|
8
|
+
exports.streamDownloadAndExtractTar = exports.downloadAndExtractFile = exports.downloadFile = void 0;
|
|
9
9
|
const fs_1 = require("fs");
|
|
10
10
|
const promises_1 = require("fs/promises");
|
|
11
11
|
const stream_1 = require("stream");
|
|
@@ -16,8 +16,14 @@ const axios_1 = __importDefault(require("axios"));
|
|
|
16
16
|
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
17
17
|
const cli_progress_1 = __importDefault(require("cli-progress"));
|
|
18
18
|
const extract_zip_1 = __importDefault(require("extract-zip"));
|
|
19
|
+
const fast_zlib_1 = require("fast-zlib");
|
|
19
20
|
const tar_1 = require("tar");
|
|
20
21
|
const promisifiedFinished = (0, util_1.promisify)(stream_1.finished);
|
|
22
|
+
/**
|
|
23
|
+
* Larger buffer sizes reduce the number of syscalls and context switches
|
|
24
|
+
* in the streaming pipeline, improving throughput for large (>1GB) files.
|
|
25
|
+
*/
|
|
26
|
+
const STREAMING_HIGH_WATER_MARK = 256 * 1024;
|
|
21
27
|
const shouldShowProgressBar = () => {
|
|
22
28
|
return process.env.METICULOUS_IS_CLOUD_REPLAY !== "true";
|
|
23
29
|
};
|
|
@@ -158,50 +164,120 @@ const downloadAndExtractFile = async (fileUrl, tmpZipFilePath, extractPath, extr
|
|
|
158
164
|
};
|
|
159
165
|
exports.downloadAndExtractFile = downloadAndExtractFile;
|
|
160
166
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
167
|
+
* Streams a raw-deflated tar blob from a URL directly through inflate and
|
|
168
|
+
* tar extraction without writing the archive to disk first.
|
|
169
|
+
*
|
|
170
|
+
* This eliminates the temp file that the legacy `downloadAndExtractTar` used,
|
|
171
|
+
* halving disk I/O and allowing download and extraction to overlap. Uses
|
|
172
|
+
* fast-zlib for decompression and tuned buffer sizes for better throughput
|
|
173
|
+
* on large (>1GB) files.
|
|
174
|
+
*
|
|
175
|
+
* __Warning__: this function is not thread safe. Do not extract to a
|
|
176
|
+
* directory that may already be in use by another process.
|
|
165
177
|
*
|
|
166
178
|
* @param fileUrl The URL of the deflated tar blob to download.
|
|
167
|
-
* @param
|
|
168
|
-
*
|
|
169
|
-
* @
|
|
170
|
-
* Do not try extracting to a dir that may already be in use by another process b/c overlapping
|
|
171
|
-
* file names can cause data corruption.
|
|
172
|
-
* @param extractTimeoutInMs The timeout for the tar extraction, in milliseconds.
|
|
173
|
-
* @returns The list of the extracted files.
|
|
179
|
+
* @param extractPath The directory to extract tar entries into.
|
|
180
|
+
* @param opts Timeout, retry, and buffer configuration.
|
|
181
|
+
* @returns The list of extracted file paths.
|
|
174
182
|
*/
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
});
|
|
183
|
+
const streamDownloadAndExtractTar = async (fileUrl, extractPath, opts = {}) => {
|
|
184
|
+
var _a, _b, _c, _d;
|
|
185
|
+
const firstDataTimeoutInMs = (_a = opts.firstDataTimeoutInMs) !== null && _a !== void 0 ? _a : 60000;
|
|
186
|
+
const totalTimeoutInMs = (_b = opts.totalTimeoutInMs) !== null && _b !== void 0 ? _b : 600000;
|
|
187
|
+
const maxRetries = (_c = opts.maxRetries) !== null && _c !== void 0 ? _c : 3;
|
|
188
|
+
const retryDelay = (_d = opts.retryDelay) !== null && _d !== void 0 ? _d : 1000;
|
|
189
|
+
for (let attempt = 0;; attempt++) {
|
|
183
190
|
const abortController = new AbortController();
|
|
191
|
+
const timeoutId = setTimeout(() => {
|
|
192
|
+
abortController.abort(new Error(`Streaming download and extraction timed out after ${totalTimeoutInMs}ms`));
|
|
193
|
+
}, totalTimeoutInMs);
|
|
184
194
|
try {
|
|
195
|
+
const client = axios_1.default.create({ timeout: firstDataTimeoutInMs });
|
|
196
|
+
(0, axios_retry_1.default)(client, { retries: 3, shouldResetTimeout: true });
|
|
197
|
+
const entries = [];
|
|
185
198
|
await (0, promises_1.mkdir)(extractPath, { recursive: true });
|
|
186
|
-
const
|
|
199
|
+
const response = await client.request({
|
|
200
|
+
method: "GET",
|
|
201
|
+
url: fileUrl,
|
|
202
|
+
responseType: "stream",
|
|
203
|
+
signal: abortController.signal,
|
|
204
|
+
});
|
|
205
|
+
await (0, promises_2.pipeline)(response.data, createFastInflateRawStream(), (0, tar_1.extract)({
|
|
187
206
|
cwd: extractPath,
|
|
188
207
|
onReadEntry: (entry) => entries.push(entry.path),
|
|
189
208
|
}), { signal: abortController.signal });
|
|
190
|
-
|
|
209
|
+
return entries;
|
|
191
210
|
}
|
|
192
211
|
catch (error) {
|
|
212
|
+
const wasAbortedBeforeCleanup = abortController.signal.aborted;
|
|
213
|
+
const reasonBeforeCleanup = abortController.signal.reason;
|
|
193
214
|
abortController.abort();
|
|
194
|
-
|
|
215
|
+
await (0, promises_1.rm)(extractPath, { recursive: true, force: true }).catch(() => { });
|
|
216
|
+
if (attempt >= maxRetries) {
|
|
217
|
+
const errToThrow = wasAbortedBeforeCleanup && reasonBeforeCleanup instanceof Error
|
|
218
|
+
? reasonBeforeCleanup
|
|
219
|
+
: error instanceof Error
|
|
220
|
+
? error
|
|
221
|
+
: new Error(String(error));
|
|
222
|
+
throw errToThrow;
|
|
223
|
+
}
|
|
224
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
195
225
|
}
|
|
196
226
|
finally {
|
|
197
227
|
clearTimeout(timeoutId);
|
|
198
228
|
}
|
|
199
229
|
}
|
|
200
|
-
finally {
|
|
201
|
-
await (0, promises_1.rm)(tmpTarFilePath);
|
|
202
|
-
}
|
|
203
|
-
return entries;
|
|
204
230
|
};
|
|
205
|
-
exports.
|
|
231
|
+
exports.streamDownloadAndExtractTar = streamDownloadAndExtractTar;
|
|
232
|
+
/**
|
|
233
|
+
* Wraps fast-zlib's synchronous InflateRaw in a Transform stream with
|
|
234
|
+
* a tuned highWaterMark for better large-file throughput.
|
|
235
|
+
*
|
|
236
|
+
* fast-zlib processes chunks synchronously via zlib's _processChunk,
|
|
237
|
+
* avoiding the thread-pool overhead of Node's built-in async zlib streams.
|
|
238
|
+
*/
|
|
239
|
+
const createFastInflateRawStream = () => {
|
|
240
|
+
const inflate = new fast_zlib_1.InflateRaw();
|
|
241
|
+
let inflateClosed = false;
|
|
242
|
+
const closeInflateOnce = () => {
|
|
243
|
+
if (inflateClosed) {
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
inflateClosed = true;
|
|
247
|
+
inflate.close();
|
|
248
|
+
};
|
|
249
|
+
return new stream_1.Transform({
|
|
250
|
+
highWaterMark: STREAMING_HIGH_WATER_MARK,
|
|
251
|
+
transform(chunk, _encoding, callback) {
|
|
252
|
+
try {
|
|
253
|
+
const result = inflate.process(chunk);
|
|
254
|
+
if (result.length > 0) {
|
|
255
|
+
this.push(result);
|
|
256
|
+
}
|
|
257
|
+
callback();
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
callback(err);
|
|
261
|
+
}
|
|
262
|
+
},
|
|
263
|
+
flush(callback) {
|
|
264
|
+
try {
|
|
265
|
+
const result = inflate.process(Buffer.alloc(0), zlib_1.constants.Z_FINISH);
|
|
266
|
+
if (result.length > 0) {
|
|
267
|
+
this.push(result);
|
|
268
|
+
}
|
|
269
|
+
closeInflateOnce();
|
|
270
|
+
callback();
|
|
271
|
+
}
|
|
272
|
+
catch (err) {
|
|
273
|
+
callback(err);
|
|
274
|
+
}
|
|
275
|
+
},
|
|
276
|
+
destroy(_err, callback) {
|
|
277
|
+
closeInflateOnce();
|
|
278
|
+
callback(_err);
|
|
279
|
+
},
|
|
280
|
+
});
|
|
281
|
+
};
|
|
206
282
|
//# sourceMappingURL=download-file.js.map
|
|
207
|
-
//# debugId=
|
|
283
|
+
//# debugId=49217d9d-c552-53ec-93e8-6cbb59e2de04
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-file.js","sources":["../../src/file-downloads/download-file.ts"],"sourceRoot":"","names":[],"mappings":";;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"download-file.js","sources":["../../src/file-downloads/download-file.ts"],"sourceRoot":"","names":[],"mappings":";;;;;;;;AAAA,2BAAmD;AACnD,0CAAwC;AACxC,mCAA+D;AAC/D,8CAA2C;AAC3C,+BAAiC;AACjC,+BAAkD;AAClD,kDAA0B;AAC1B,8DAAqC;AACrC,gEAAuC;AACvC,8DAAkC;AAClC,yCAAuC;AACvC,6BAA4C;AAE5C,MAAM,mBAAmB,GAAG,IAAA,gBAAS,EAAC,iBAAQ,CAAC,CAAC;AAEhD;;;GAGG;AACH,MAAM,yBAAyB,GAAG,GAAG,GAAG,IAAI,CAAC;AAE7C,MAAM,qBAAqB,GAAG,GAAY,EAAE;IAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,MAAM,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,KAAa,EAAU,EAAE;IAC5C,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,CAAC;IACf,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAS9C;;;;GAIG;AACI,MAAM,YAAY,GAAG,KAAK,EAC/B,OAAe,EACf,IAAY,EACZ,OAA4B,EAAE,EACf,EAAE;;IACjB,gGAAgG;IAChG,MAAM,oBAAoB,GAAG,MAAA,IAAI,CAAC,oBAAoB,mCAAI,KAAM,CAAC;IACjE,MAAM,2BAA2B,GAC/B,MAAA,IAAI,CAAC,2BAA2B,mCAAI,MAAO,CAAC;IAC9C,MAAM,yBAAyB,GAAG,MAAA,IAAI,CAAC,yBAAyB,mCAAI,CAAC,CAAC;IACtE,MAAM,yBAAyB,GAAG,MAAA,IAAI,CAAC,yBAAyB,mCAAI,IAAI,CAAC;IAEzE,MAAM,MAAM,GAAG,eAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAC/D,IAAA,qBAAU,EAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,eAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;IAE1C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,OAAO;QACZ,YAAY,EAAE,QAAQ;QACtB,WAAW,EAAE,MAAM,CAAC,KAAK;KAC1B,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAA,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,mCAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAE9E,IAAI,WAAW,GAAiC,IAAI,CAAC;IACrD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IACE,qBAAqB,EAAE;QACvB,aAAa,IAAI,8BAA8B,EAC/C,CAAC;QACD,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CACrC;YACE,MAAM,EAAE,8DAA8D;YACtE,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,KAAK;YAClB,cAAc,EAAE,IAAI;SACrB,EACD,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,EAAE;YAClC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;YAC1B,SAAS,EAAE,WAAW,CAAC,aAAa,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,kBAAS,CAAC;QACtC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;YAClC,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;YAChC,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,MAAM,CAAC,eAAe,EAAE;oBAClC,UAAU,EAAE,WAAW,CAAC,eAAe,CAAC;oBACxC,SAAS,EAAE,WAAW,CAAC,aAAa,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;YACD,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAA,sBAAiB,EAAC,IAAI,CAAC,CAAC;IACtC,QAAQ,CAAC,IAAe,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QACtC,MAAM,KAAK,GAAG,4BAA4B,2BAA2B,IAAI,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACnC,CAAC,EAAE,2BAA2B,CAAC,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,SAAS,EAAE,CAAC;YACd,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QACD,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,SAAS,EAAE,CAAC;YACd,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QACD,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,EAAE,CAAC;QAEpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAEtD,IAAI,IAAA,eAAU,EAAC,IAAI,CAAC,EAAE,CAAC;YACrB,2GAA2G;YAC3G,sBAAsB;YACtB,MAAM,IAAA,aAAE,EAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,IAAI,yBAAyB,KAAK,CAAC,EAAE,CAAC;YACpC,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,sCAAsC;QACtC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC5B,UAAU,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAC/C,CAAC;QACF,MAAM,IAAA,oBAAY,EAAC,OAAO,EAAE,IAAI,EAAE;YAChC,oBAAoB;YACpB,2BAA2B;YAC3B,yBAAyB,EAAE,yBAAyB,GAAG,CAAC;SACzD,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC;AAtGW,QAAA,YAAY,gBAsGvB;AAEF;;;;;;;;;;;;;GAaG;AACI,MAAM,sBAAsB,GAKV,KAAK,EAC5B,OAAO,EACP,cAAc,EACd,WAAW,EACX,kBAAkB,GAAG,MAAO,EAC5B,EAAE;IACF,MAAM,IAAA,oBAAY,EAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,CAAC;QACH,IAAI,SAAwC,CAAC;QAC7C,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,SAAS,GAAG,UAAU,CACpB,GAAG,EAAE,CACH,MAAM,CACJ,IAAI,KAAK,CAAC,kCAAkC,kBAAkB,IAAI,CAAC,CACpE,EACH,kBAAkB,CACnB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAA,qBAAO,EAAC,cAAc,EAAE;gBAC7C,GAAG,EAAE,WAAW;gBAChB,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjD,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC;QACvD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAU,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,aAAE,EAAC,cAAc,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAvCW,QAAA,sBAAsB,0BAuCjC;AASF;;;;;;;;;;;;;;;;GAgBG;AACI,MAAM,2BAA2B,GAAG,KAAK,EAC9C,OAAe,EACf,WAAmB,EACnB,OAA2C,EAAE,EAC1B,EAAE;;IACrB,MAAM,oBAAoB,GAAG,MAAA,IAAI,CAAC,oBAAoB,mCAAI,KAAM,CAAC;IACjE,MAAM,gBAAgB,GAAG,MAAA,IAAI,CAAC,gBAAgB,mCAAI,MAAO,CAAC;IAC1D,MAAM,UAAU,GAAG,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,MAAA,IAAI,CAAC,UAAU,mCAAI,IAAI,CAAC;IAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QAClC,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,eAAe,CAAC,KAAK,CACnB,IAAI,KAAK,CACP,qDAAqD,gBAAgB,IAAI,CAC1E,CACF,CAAC;QACJ,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;YAC/D,IAAA,qBAAU,EAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YAE7D,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,IAAA,gBAAK,EAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;gBACpC,MAAM,EAAE,KAAK;gBACb,GAAG,EAAE,OAAO;gBACZ,YAAY,EAAE,QAAQ;gBACtB,MAAM,EAAE,eAAe,CAAC,MAAM;aAC/B,CAAC,CAAC;YAEH,MAAM,IAAA,mBAAQ,EACZ,QAAQ,CAAC,IAAgB,EACzB,0BAA0B,EAAE,EAC5B,IAAA,aAAU,EAAC;gBACT,GAAG,EAAE,WAAW;gBAChB,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;aACjD,CAAC,EACF,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,EAAE,CACnC,CAAC;YAEF,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,uBAAuB,GAAG,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;YAC/D,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;YAC1D,eAAe,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAA,aAAE,EAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAExE,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBAC1B,MAAM,UAAU,GACd,uBAAuB,IAAI,mBAAmB,YAAY,KAAK;oBAC7D,CAAC,CAAC,mBAAmB;oBACrB,CAAC,CAAC,KAAK,YAAY,KAAK;wBACtB,CAAC,CAAC,KAAK;wBACP,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjC,MAAM,UAAU,CAAC;YACnB,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAClE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAlEW,QAAA,2BAA2B,+BAkEtC;AAEF;;;;;;GAMG;AACH,MAAM,0BAA0B,GAAG,GAAc,EAAE;IACjD,MAAM,OAAO,GAAG,IAAI,sBAAU,EAAE,CAAC;IACjC,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,aAAa,GAAG,IAAI,CAAC;QACrB,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,IAAI,kBAAS,CAAC;QACnB,aAAa,EAAE,yBAAyB;QACxC,SAAS,CAAC,KAAa,EAAE,SAAyB,EAAE,QAAQ;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACtC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpB,CAAC;gBACD,QAAQ,EAAE,CAAC;YACb,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,CAAC,GAAY,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,QAAQ;YACZ,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,gBAAa,CAAC,QAAQ,CAAC,CAAC;gBACxE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpB,CAAC;gBACD,gBAAgB,EAAE,CAAC;gBACnB,QAAQ,EAAE,CAAC;YACb,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,CAAC,GAAY,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,EAAE,QAAQ;YACpB,gBAAgB,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","debugId":"49217d9d-c552-53ec-93e8-6cbb59e2de04"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export { sanitizeFilename } from "./file-downloads/local-data.utils";
|
|
2
2
|
export { downloadAppContainerLogs } from "./file-downloads/app-container-logs";
|
|
3
|
-
export { getOrFetchReplay, getOrFetchReplayArchive, DownloadScope
|
|
3
|
+
export { getOrFetchReplay, getOrFetchReplayArchive, DownloadScope } from "./scripts/replays";
|
|
4
4
|
export { getOrFetchRecordedSession, getOrFetchRecordedSessionData, writeManifest, writeStructuredSessionData, type SessionsManifest, type WriteStructuredSessionOptions, } from "./file-downloads/sessions";
|
|
5
5
|
export { getOrFetchTestRunData, type TestRunDownloadScope, DOWNLOAD_SCOPES as TEST_RUN_DOWNLOAD_SCOPES, } from "./file-downloads/test-runs";
|
|
6
6
|
export { fetchAsset, checkIfAssetsOutdated } from "./scripts/replay-assets";
|
|
7
|
-
export { downloadFile, downloadAndExtractFile,
|
|
7
|
+
export { downloadFile, downloadAndExtractFile, streamDownloadAndExtractTar, type StreamDownloadAndExtractTarOptions, } from "./file-downloads/download-file";
|
|
8
8
|
export { getReplayDir } from "./scripts/replays";
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="
|
|
2
|
+
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="d169d7ae-ff26-586a-b6c3-6317d3e6bc42")}catch(e){}}();
|
|
3
3
|
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.getReplayDir = exports.
|
|
5
|
+
exports.getReplayDir = exports.streamDownloadAndExtractTar = exports.downloadAndExtractFile = exports.downloadFile = exports.checkIfAssetsOutdated = exports.fetchAsset = exports.TEST_RUN_DOWNLOAD_SCOPES = exports.getOrFetchTestRunData = exports.writeStructuredSessionData = exports.writeManifest = exports.getOrFetchRecordedSessionData = exports.getOrFetchRecordedSession = exports.getOrFetchReplayArchive = exports.getOrFetchReplay = exports.downloadAppContainerLogs = exports.sanitizeFilename = void 0;
|
|
6
6
|
var local_data_utils_1 = require("./file-downloads/local-data.utils");
|
|
7
7
|
Object.defineProperty(exports, "sanitizeFilename", { enumerable: true, get: function () { return local_data_utils_1.sanitizeFilename; } });
|
|
8
8
|
var app_container_logs_1 = require("./file-downloads/app-container-logs");
|
|
@@ -24,8 +24,8 @@ Object.defineProperty(exports, "checkIfAssetsOutdated", { enumerable: true, get:
|
|
|
24
24
|
var download_file_1 = require("./file-downloads/download-file");
|
|
25
25
|
Object.defineProperty(exports, "downloadFile", { enumerable: true, get: function () { return download_file_1.downloadFile; } });
|
|
26
26
|
Object.defineProperty(exports, "downloadAndExtractFile", { enumerable: true, get: function () { return download_file_1.downloadAndExtractFile; } });
|
|
27
|
-
Object.defineProperty(exports, "
|
|
27
|
+
Object.defineProperty(exports, "streamDownloadAndExtractTar", { enumerable: true, get: function () { return download_file_1.streamDownloadAndExtractTar; } });
|
|
28
28
|
var replays_2 = require("./scripts/replays");
|
|
29
29
|
Object.defineProperty(exports, "getReplayDir", { enumerable: true, get: function () { return replays_2.getReplayDir; } });
|
|
30
30
|
//# sourceMappingURL=index.js.map
|
|
31
|
-
//# debugId=
|
|
31
|
+
//# debugId=d169d7ae-ff26-586a-b6c3-6317d3e6bc42
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sources":["../src/index.ts"],"sourceRoot":"","names":[],"mappings":";;;;;AAAA,sEAAqE;AAA5D,oHAAA,gBAAgB,OAAA;AACzB,0EAA+E;AAAtE,8HAAA,wBAAwB,OAAA;AACjC,
|
|
1
|
+
{"version":3,"file":"index.js","sources":["../src/index.ts"],"sourceRoot":"","names":[],"mappings":";;;;;AAAA,sEAAqE;AAA5D,oHAAA,gBAAgB,OAAA;AACzB,0EAA+E;AAAtE,8HAAA,wBAAwB,OAAA;AACjC,6CAA6F;AAApF,2GAAA,gBAAgB,OAAA;AAAE,kHAAA,uBAAuB,OAAA;AAClD,sDAOmC;AANjC,qHAAA,yBAAyB,OAAA;AACzB,yHAAA,6BAA6B,OAAA;AAC7B,yGAAA,aAAa,OAAA;AACb,sHAAA,0BAA0B,OAAA;AAI5B,wDAIoC;AAHlC,kHAAA,qBAAqB,OAAA;AAErB,qHAAA,eAAe,OAA4B;AAE7C,yDAA4E;AAAnE,2GAAA,UAAU,OAAA;AAAE,sHAAA,qBAAqB,OAAA;AAC1C,gEAKwC;AAJtC,6GAAA,YAAY,OAAA;AACZ,uHAAA,sBAAsB,OAAA;AACtB,4HAAA,2BAA2B,OAAA;AAG7B,6CAAiD;AAAxC,uGAAA,YAAY,OAAA","debugId":"d169d7ae-ff26-586a-b6c3-6317d3e6bc42"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alwaysmeticulous/downloading-helpers",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.272.0",
|
|
4
4
|
"description": "Helper utilities for downloading files & scripts required to execute replays",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -56,5 +56,5 @@
|
|
|
56
56
|
"bugs": {
|
|
57
57
|
"url": "https://github.com/alwaysmeticulous/meticulous-sdk/issues"
|
|
58
58
|
},
|
|
59
|
-
"gitHead": "
|
|
59
|
+
"gitHead": "cc675dc8e2c5c5931e630af6566606dd335dd534"
|
|
60
60
|
}
|