@milaboratories/pl-drivers 1.10.16 → 1.10.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -45,32 +45,42 @@ class RemoteFileDownloader {
|
|
|
45
45
|
try {
|
|
46
46
|
await checkStatusCodeOk(statusCode, webBody, url);
|
|
47
47
|
ops.signal?.throwIfAborted();
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
48
|
+
let result = undefined;
|
|
49
|
+
const contentLength = Number(responseHeaders['content-length']);
|
|
50
|
+
if (Number.isNaN(contentLength) || contentLength === 0) {
|
|
51
|
+
// Some backend versions have a bug that they are not returning content-length header.
|
|
52
|
+
// In this case `content-length` header is returned as 0.
|
|
53
|
+
// We should not clip the result stream to 0 bytes in such case.
|
|
54
|
+
result = await handler(webBody, 0);
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
// Some backend versions have a bug where they return more data than requested in range.
|
|
58
|
+
// So we have to manually normalize the stream to the expected size.
|
|
59
|
+
const size = ops.range ? ops.range.to - ops.range.from : contentLength;
|
|
60
|
+
const normalizedStream = webBody.pipeThrough(new (class extends web.TransformStream {
|
|
61
|
+
constructor(sizeBytes, recordOffByOne) {
|
|
62
|
+
super({
|
|
63
|
+
transform(chunk, controller) {
|
|
64
|
+
const truncatedChunk = chunk.slice(0, sizeBytes);
|
|
65
|
+
controller.enqueue(truncatedChunk);
|
|
66
|
+
sizeBytes -= truncatedChunk.length;
|
|
67
|
+
if (!sizeBytes)
|
|
68
|
+
controller.terminate();
|
|
69
|
+
},
|
|
70
|
+
flush(controller) {
|
|
71
|
+
// Some backend versions have a bug where they return 1 less byte than requested in range.
|
|
72
|
+
// We cannot always request one more byte because if this end byte is the last byte of the file,
|
|
73
|
+
// the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
|
|
74
|
+
if (sizeBytes === 1) {
|
|
75
|
+
recordOffByOne();
|
|
76
|
+
controller.error(new OffByOneError());
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
})(size, () => this.offByOneServers.push(urlOrigin)));
|
|
82
|
+
result = await handler(normalizedStream, size);
|
|
83
|
+
}
|
|
74
84
|
handlerSuccess = true;
|
|
75
85
|
return result;
|
|
76
86
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n headers,\n signal: ops.signal,\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n // Some backend versions have a bug where they return more data than requested in range.\n
|
|
1
|
+
{"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n headers,\n signal: ops.signal,\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n let result: T | undefined = undefined;\n\n const contentLength = Number(responseHeaders['content-length']);\n if (Number.isNaN(contentLength) || contentLength === 0) {\n // Some backend versions have a bug that they are not returning content-length header.\n // In this case `content-length` header is returned as 0.\n // We should not clip the result stream to 0 bytes in such case.\n result = await handler(webBody, 0);\n } else {\n // Some backend versions have a bug where they return more data than requested in range.\n // So we have to manually normalize the stream to the expected size.\n const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n constructor(sizeBytes: number, recordOffByOne: () => void) {\n super({\n transform(chunk: Uint8Array, controller) {\n const truncatedChunk = chunk.slice(0, sizeBytes);\n controller.enqueue(truncatedChunk);\n sizeBytes -= truncatedChunk.length;\n if (!sizeBytes) controller.terminate();\n },\n flush(controller) {\n // Some backend versions have a bug where they return 1 less byte than requested in range.\n // We cannot always request one more byte because if this end byte is the last byte of the file,\n // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n if (sizeBytes === 1) {\n recordOffByOne();\n controller.error(new OffByOneError());\n }\n },\n });\n }\n })(size, () => this.offByOneServers.push(urlOrigin)));\n result = await handler(normalizedStream, size);\n }\n\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && !webBody.locked) {\n try {\n await webBody.cancel();\n } catch {\n // Ignore cleanup errors\n }\n }\n throw error;\n }\n }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n const beginning = (await text(webBody)).substring(0, 1000);\n\n if (400 <= statusCode && statusCode < 500) {\n throw new NetworkError400(\n `Http error: statusCode: ${statusCode} `\n + `url: ${url.toString()}, beginning of body: ${beginning}`);\n }\n\n throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":";;;;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;;;;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/helpers/download.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAEtF,gFAAgF;AAChF,qBAAa,eAAgB,SAAQ,KAAK;IACxC,IAAI,SAAqB;CAC1B;AAED;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,IAAI,SAAmB;CACxB;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,qBAAa,oBAAoB;aAGH,UAAU,EAAE,UAAU;IAFlD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpB,UAAU,EAAE,UAAU;IAE5C,WAAW,CAAC,CAAC,EACjB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC;CA0Ed"}
|
package/dist/helpers/download.js
CHANGED
|
@@ -43,32 +43,42 @@ class RemoteFileDownloader {
|
|
|
43
43
|
try {
|
|
44
44
|
await checkStatusCodeOk(statusCode, webBody, url);
|
|
45
45
|
ops.signal?.throwIfAborted();
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
46
|
+
let result = undefined;
|
|
47
|
+
const contentLength = Number(responseHeaders['content-length']);
|
|
48
|
+
if (Number.isNaN(contentLength) || contentLength === 0) {
|
|
49
|
+
// Some backend versions have a bug that they are not returning content-length header.
|
|
50
|
+
// In this case `content-length` header is returned as 0.
|
|
51
|
+
// We should not clip the result stream to 0 bytes in such case.
|
|
52
|
+
result = await handler(webBody, 0);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
// Some backend versions have a bug where they return more data than requested in range.
|
|
56
|
+
// So we have to manually normalize the stream to the expected size.
|
|
57
|
+
const size = ops.range ? ops.range.to - ops.range.from : contentLength;
|
|
58
|
+
const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
|
|
59
|
+
constructor(sizeBytes, recordOffByOne) {
|
|
60
|
+
super({
|
|
61
|
+
transform(chunk, controller) {
|
|
62
|
+
const truncatedChunk = chunk.slice(0, sizeBytes);
|
|
63
|
+
controller.enqueue(truncatedChunk);
|
|
64
|
+
sizeBytes -= truncatedChunk.length;
|
|
65
|
+
if (!sizeBytes)
|
|
66
|
+
controller.terminate();
|
|
67
|
+
},
|
|
68
|
+
flush(controller) {
|
|
69
|
+
// Some backend versions have a bug where they return 1 less byte than requested in range.
|
|
70
|
+
// We cannot always request one more byte because if this end byte is the last byte of the file,
|
|
71
|
+
// the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
|
|
72
|
+
if (sizeBytes === 1) {
|
|
73
|
+
recordOffByOne();
|
|
74
|
+
controller.error(new OffByOneError());
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
})(size, () => this.offByOneServers.push(urlOrigin)));
|
|
80
|
+
result = await handler(normalizedStream, size);
|
|
81
|
+
}
|
|
72
82
|
handlerSuccess = true;
|
|
73
83
|
return result;
|
|
74
84
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n headers,\n signal: ops.signal,\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n // Some backend versions have a bug where they return more data than requested in range.\n
|
|
1
|
+
{"version":3,"file":"download.js","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class NetworkError400 extends Error {\n name = 'NetworkError400';\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n headers,\n signal: ops.signal,\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n let result: T | undefined = undefined;\n\n const contentLength = Number(responseHeaders['content-length']);\n if (Number.isNaN(contentLength) || contentLength === 0) {\n // Some backend versions have a bug that they are not returning content-length header.\n // In this case `content-length` header is returned as 0.\n // We should not clip the result stream to 0 bytes in such case.\n result = await handler(webBody, 0);\n } else {\n // Some backend versions have a bug where they return more data than requested in range.\n // So we have to manually normalize the stream to the expected size.\n const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n constructor(sizeBytes: number, recordOffByOne: () => void) {\n super({\n transform(chunk: Uint8Array, controller) {\n const truncatedChunk = chunk.slice(0, sizeBytes);\n controller.enqueue(truncatedChunk);\n sizeBytes -= truncatedChunk.length;\n if (!sizeBytes) controller.terminate();\n },\n flush(controller) {\n // Some backend versions have a bug where they return 1 less byte than requested in range.\n // We cannot always request one more byte because if this end byte is the last byte of the file,\n // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n if (sizeBytes === 1) {\n recordOffByOne();\n controller.error(new OffByOneError());\n }\n },\n });\n }\n })(size, () => this.offByOneServers.push(urlOrigin)));\n result = await handler(normalizedStream, size);\n }\n\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && !webBody.locked) {\n try {\n await webBody.cancel();\n } catch {\n // Ignore cleanup errors\n }\n }\n throw error;\n }\n }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n const beginning = (await text(webBody)).substring(0, 1000);\n\n if (400 <= statusCode && statusCode < 500) {\n throw new NetworkError400(\n `Http error: statusCode: ${statusCode} `\n + `url: ${url.toString()}, beginning of body: ${beginning}`);\n }\n\n throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);\n }\n}\n"],"names":[],"mappings":";;;;;AAYA;AACM,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;AACnB,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAc,eAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;QAE1D,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG,EAAE;AACzC,YAAA,MAAM,IAAI,eAAe,CACvB,CAAA,wBAAA,EAA2B,UAAU,CAAA,CAAA;kBACnC,CAAA,KAAA,EAAQ,GAAG,CAAC,QAAQ,EAAE,wBAAwB,SAAS,CAAA,CAAE,CAAC;QAChE;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,CAAA,wBAAA,EAA2B,UAAU,CAAA,MAAA,EAAS,GAAG,CAAC,QAAQ,EAAE,CAAA,CAAE,CAAC;IACjF;AACF;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-drivers",
|
|
3
|
-
"version": "1.10.
|
|
3
|
+
"version": "1.10.17",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=20"
|
|
6
6
|
},
|
|
@@ -33,10 +33,10 @@
|
|
|
33
33
|
"zod": "~3.23.8",
|
|
34
34
|
"@milaboratories/ts-helpers": "1.4.7",
|
|
35
35
|
"@milaboratories/helpers": "1.8.0",
|
|
36
|
-
"@milaboratories/
|
|
36
|
+
"@milaboratories/pl-model-common": "1.19.19",
|
|
37
37
|
"@milaboratories/pl-client": "2.12.2",
|
|
38
|
-
"@milaboratories/
|
|
39
|
-
"@milaboratories/pl-
|
|
38
|
+
"@milaboratories/computable": "2.6.8",
|
|
39
|
+
"@milaboratories/pl-tree": "1.7.13"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@types/decompress": "^4.2.7",
|
|
@@ -46,9 +46,9 @@
|
|
|
46
46
|
"eslint": "^9.25.1",
|
|
47
47
|
"typescript": "~5.6.3",
|
|
48
48
|
"vitest": "^2.1.9",
|
|
49
|
-
"@milaboratories/eslint-config": "1.0.4",
|
|
50
|
-
"@milaboratories/build-configs": "1.0.8",
|
|
51
49
|
"@milaboratories/ts-builder": "1.0.5",
|
|
50
|
+
"@milaboratories/build-configs": "1.0.8",
|
|
51
|
+
"@milaboratories/eslint-config": "1.0.4",
|
|
52
52
|
"@milaboratories/ts-configs": "1.0.6"
|
|
53
53
|
},
|
|
54
54
|
"scripts": {
|
package/src/helpers/download.ts
CHANGED
|
@@ -62,31 +62,41 @@ export class RemoteFileDownloader {
|
|
|
62
62
|
await checkStatusCodeOk(statusCode, webBody, url);
|
|
63
63
|
ops.signal?.throwIfAborted();
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
65
|
+
let result: T | undefined = undefined;
|
|
66
|
+
|
|
67
|
+
const contentLength = Number(responseHeaders['content-length']);
|
|
68
|
+
if (Number.isNaN(contentLength) || contentLength === 0) {
|
|
69
|
+
// Some backend versions have a bug that they are not returning content-length header.
|
|
70
|
+
// In this case `content-length` header is returned as 0.
|
|
71
|
+
// We should not clip the result stream to 0 bytes in such case.
|
|
72
|
+
result = await handler(webBody, 0);
|
|
73
|
+
} else {
|
|
74
|
+
// Some backend versions have a bug where they return more data than requested in range.
|
|
75
|
+
// So we have to manually normalize the stream to the expected size.
|
|
76
|
+
const size = ops.range ? ops.range.to - ops.range.from : contentLength;
|
|
77
|
+
const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {
|
|
78
|
+
constructor(sizeBytes: number, recordOffByOne: () => void) {
|
|
79
|
+
super({
|
|
80
|
+
transform(chunk: Uint8Array, controller) {
|
|
81
|
+
const truncatedChunk = chunk.slice(0, sizeBytes);
|
|
82
|
+
controller.enqueue(truncatedChunk);
|
|
83
|
+
sizeBytes -= truncatedChunk.length;
|
|
84
|
+
if (!sizeBytes) controller.terminate();
|
|
85
|
+
},
|
|
86
|
+
flush(controller) {
|
|
87
|
+
// Some backend versions have a bug where they return 1 less byte than requested in range.
|
|
88
|
+
// We cannot always request one more byte because if this end byte is the last byte of the file,
|
|
89
|
+
// the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
|
|
90
|
+
if (sizeBytes === 1) {
|
|
91
|
+
recordOffByOne();
|
|
92
|
+
controller.error(new OffByOneError());
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
})(size, () => this.offByOneServers.push(urlOrigin)));
|
|
98
|
+
result = await handler(normalizedStream, size);
|
|
99
|
+
}
|
|
90
100
|
|
|
91
101
|
handlerSuccess = true;
|
|
92
102
|
return result;
|