@devrev/ts-adaas 1.13.4-beta.0 → 1.13.4-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +26 -19
- package/dist/uploader/uploader.d.ts +9 -6
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -3
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +6 -7
- package/dist/uploader/uploader.helpers.test.js +9 -7
- package/dist/uploader/uploader.interfaces.d.ts +11 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +58 -46
- package/dist/uploader/uploader.test.js +30 -21
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAmHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA0BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA2E5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6LvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAkGvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAmD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAsHzC"}
|
|
@@ -230,11 +230,11 @@ class WorkerAdapter {
|
|
|
230
230
|
break;
|
|
231
231
|
}
|
|
232
232
|
if (!fileToLoad.completed) {
|
|
233
|
-
const
|
|
233
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
234
234
|
artifactId: fileToLoad.id,
|
|
235
235
|
isGzipped: true,
|
|
236
|
-
})
|
|
237
|
-
if (
|
|
236
|
+
});
|
|
237
|
+
if (transformerFileError) {
|
|
238
238
|
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
239
239
|
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
240
240
|
error: {
|
|
@@ -242,6 +242,7 @@ class WorkerAdapter {
|
|
|
242
242
|
},
|
|
243
243
|
});
|
|
244
244
|
}
|
|
245
|
+
const transformerFile = response;
|
|
245
246
|
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
246
247
|
const { report, rateLimit } = await this.loadItem({
|
|
247
248
|
item: transformerFile[i],
|
|
@@ -276,10 +277,11 @@ class WorkerAdapter {
|
|
|
276
277
|
var _a;
|
|
277
278
|
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
278
279
|
if (statsFileArtifactId) {
|
|
279
|
-
const
|
|
280
|
+
const { response, error: statsFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
280
281
|
artifactId: statsFileArtifactId,
|
|
281
|
-
})
|
|
282
|
-
|
|
282
|
+
});
|
|
283
|
+
const statsFile = response;
|
|
284
|
+
if (statsFileError || statsFile.length === 0) {
|
|
283
285
|
return [];
|
|
284
286
|
}
|
|
285
287
|
const filesToLoad = (0, worker_adapter_helpers_1.getFilesToLoad)({
|
|
@@ -310,11 +312,12 @@ class WorkerAdapter {
|
|
|
310
312
|
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
311
313
|
outerloop: for (const fileToLoad of filesToLoad) {
|
|
312
314
|
if (!fileToLoad.completed) {
|
|
313
|
-
const
|
|
315
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
314
316
|
artifactId: fileToLoad.id,
|
|
315
317
|
isGzipped: true,
|
|
316
|
-
})
|
|
317
|
-
|
|
318
|
+
});
|
|
319
|
+
const transformerFile = response;
|
|
320
|
+
if (transformerFileError) {
|
|
318
321
|
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
319
322
|
break outerloop;
|
|
320
323
|
}
|
|
@@ -513,9 +516,10 @@ class WorkerAdapter {
|
|
|
513
516
|
? parseInt(httpStream.headers['content-length'])
|
|
514
517
|
: undefined;
|
|
515
518
|
// Get upload URL
|
|
516
|
-
const
|
|
517
|
-
if (
|
|
518
|
-
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment
|
|
519
|
+
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
520
|
+
if (artifactUrlError) {
|
|
521
|
+
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
522
|
+
(0, logger_1.serializeError)(artifactUrlError));
|
|
519
523
|
this.destroyHttpStream(httpStream);
|
|
520
524
|
return;
|
|
521
525
|
}
|
|
@@ -524,21 +528,24 @@ class WorkerAdapter {
|
|
|
524
528
|
return;
|
|
525
529
|
}
|
|
526
530
|
// Stream attachment
|
|
527
|
-
const
|
|
528
|
-
if (
|
|
529
|
-
console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment
|
|
531
|
+
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
532
|
+
if (uploadedArtifactError) {
|
|
533
|
+
console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
534
|
+
(0, logger_1.serializeError)(uploadedArtifactError));
|
|
530
535
|
this.destroyHttpStream(httpStream);
|
|
531
536
|
return;
|
|
532
537
|
}
|
|
533
538
|
// Confirm attachment upload
|
|
534
|
-
const
|
|
535
|
-
if (
|
|
536
|
-
console.warn('Error while confirming upload for attachment ID ' +
|
|
539
|
+
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
540
|
+
if (confirmArtifactUploadError) {
|
|
541
|
+
console.warn('Error while confirming upload for attachment ID ' +
|
|
542
|
+
attachment.id +
|
|
543
|
+
'.', confirmArtifactUploadError);
|
|
537
544
|
return;
|
|
538
545
|
}
|
|
539
546
|
const ssorAttachment = {
|
|
540
547
|
id: {
|
|
541
|
-
devrev:
|
|
548
|
+
devrev: artifactUrlResponse.artifact_id,
|
|
542
549
|
external: attachment.id,
|
|
543
550
|
},
|
|
544
551
|
parent_id: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
2
|
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
|
-
import { ArtifactToUpload, UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
|
|
3
|
+
import { ArtifactToUpload, UploadResponse, UploaderFactoryInterface, UploaderResult } from './uploader.interfaces';
|
|
4
4
|
export declare class Uploader {
|
|
5
5
|
private isLocalDevelopment?;
|
|
6
6
|
private devrevApiEndpoint;
|
|
@@ -22,27 +22,30 @@ export declare class Uploader {
|
|
|
22
22
|
* @param {number} [fileSize] - Optional file size in bytes
|
|
23
23
|
* @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
|
|
24
24
|
*/
|
|
25
|
-
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<ArtifactToUpload
|
|
25
|
+
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<UploaderResult<ArtifactToUpload>>;
|
|
26
26
|
/**
|
|
27
27
|
* Uploads an artifact file to the provided upload URL using multipart form data.
|
|
28
28
|
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
29
29
|
* @param {Buffer} file - The file buffer to upload
|
|
30
30
|
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
31
31
|
*/
|
|
32
|
-
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<AxiosResponse
|
|
32
|
+
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
|
|
33
33
|
/**
|
|
34
34
|
* Streams an artifact file from an axios response to the upload URL.
|
|
35
35
|
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
36
36
|
* @param {AxiosResponse} fileStream - The axios response stream containing the file data
|
|
37
37
|
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
38
38
|
*/
|
|
39
|
-
streamArtifact(artifact: ArtifactToUpload, fileStream: AxiosResponse): Promise<AxiosResponse
|
|
39
|
+
streamArtifact(artifact: ArtifactToUpload, fileStream: AxiosResponse): Promise<UploaderResult<AxiosResponse>>;
|
|
40
40
|
/**
|
|
41
41
|
* Confirms that an artifact upload has been completed successfully.
|
|
42
42
|
* @param {string} artifactId - The ID of the artifact to confirm
|
|
43
43
|
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
44
44
|
*/
|
|
45
|
-
confirmArtifactUpload(artifactId: string): Promise<
|
|
45
|
+
confirmArtifactUpload(artifactId: string): Promise<{
|
|
46
|
+
response?: AxiosResponse;
|
|
47
|
+
error?: unknown;
|
|
48
|
+
}>;
|
|
46
49
|
/**
|
|
47
50
|
* Destroys a stream to prevent resource leaks.
|
|
48
51
|
* @param {any} fileStream - The axios response stream to destroy
|
|
@@ -84,6 +87,6 @@ export declare class Uploader {
|
|
|
84
87
|
getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
|
|
85
88
|
artifactId: string;
|
|
86
89
|
isGzipped?: boolean;
|
|
87
|
-
}): Promise<object[] | object
|
|
90
|
+
}): Promise<UploaderResult<object[] | object>>;
|
|
88
91
|
}
|
|
89
92
|
//# sourceMappingURL=uploader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAUxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA6D1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAqB5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IA8BzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IAsDF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,21 +1,22 @@
|
|
|
1
|
+
import { UploaderResult } from './uploader.interfaces';
|
|
1
2
|
/**
|
|
2
3
|
* Compresses a JSONL string using gzip compression.
|
|
3
4
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
4
5
|
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
5
6
|
*/
|
|
6
|
-
export declare function compressGzip(jsonlObject: string): Buffer
|
|
7
|
+
export declare function compressGzip(jsonlObject: string): UploaderResult<Buffer>;
|
|
7
8
|
/**
|
|
8
9
|
* Decompresses a gzipped buffer to a JSONL string.
|
|
9
10
|
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
10
11
|
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
11
12
|
*/
|
|
12
|
-
export declare function decompressGzip(gzippedJsonlObject: Buffer): string
|
|
13
|
+
export declare function decompressGzip(gzippedJsonlObject: Buffer): UploaderResult<string>;
|
|
13
14
|
/**
|
|
14
15
|
* Parses a JSONL string into an array of objects.
|
|
15
16
|
* @param {string} jsonlObject - The JSONL string to parse
|
|
16
17
|
* @returns {object[] | null} The parsed array of objects or null on error
|
|
17
18
|
*/
|
|
18
|
-
export declare function parseJsonl(jsonlObject: string): object[]
|
|
19
|
+
export declare function parseJsonl(jsonlObject: string): UploaderResult<object[]>;
|
|
19
20
|
/**
|
|
20
21
|
* Downloads fetched objects to the local file system (for local development).
|
|
21
22
|
* @param {string} itemType - The type of items being downloaded
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -52,10 +52,10 @@ const constants_1 = require("../common/constants");
|
|
|
52
52
|
*/
|
|
53
53
|
function compressGzip(jsonlObject) {
|
|
54
54
|
try {
|
|
55
|
-
return zlib_1.default.gzipSync(jsonlObject);
|
|
55
|
+
return { response: zlib_1.default.gzipSync(jsonlObject) };
|
|
56
56
|
}
|
|
57
57
|
catch (error) {
|
|
58
|
-
|
|
58
|
+
return { error };
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
/**
|
|
@@ -66,10 +66,10 @@ function compressGzip(jsonlObject) {
|
|
|
66
66
|
function decompressGzip(gzippedJsonlObject) {
|
|
67
67
|
try {
|
|
68
68
|
const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
|
|
69
|
-
return jsonlObject.toString();
|
|
69
|
+
return { response: jsonlObject.toString() };
|
|
70
70
|
}
|
|
71
71
|
catch (error) {
|
|
72
|
-
|
|
72
|
+
return { error };
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
/**
|
|
@@ -79,12 +79,11 @@ function decompressGzip(gzippedJsonlObject) {
|
|
|
79
79
|
*/
|
|
80
80
|
function parseJsonl(jsonlObject) {
|
|
81
81
|
try {
|
|
82
|
-
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
82
|
+
return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
|
|
83
83
|
}
|
|
84
84
|
catch (error) {
|
|
85
|
-
|
|
85
|
+
return { error };
|
|
86
86
|
}
|
|
87
|
-
return null;
|
|
88
87
|
}
|
|
89
88
|
/**
|
|
90
89
|
* Downloads fetched objects to the local file system (for local development).
|
|
@@ -31,8 +31,10 @@ describe('uploader.helpers', () => {
|
|
|
31
31
|
// Act
|
|
32
32
|
const result = (0, uploader_helpers_1.compressGzip)(input);
|
|
33
33
|
// Assert
|
|
34
|
-
expect(result).toBeInstanceOf(Buffer);
|
|
35
|
-
const decompressed = zlib_1.default
|
|
34
|
+
expect(result.response).toBeInstanceOf(Buffer);
|
|
35
|
+
const decompressed = zlib_1.default
|
|
36
|
+
.gunzipSync(result.response)
|
|
37
|
+
.toString();
|
|
36
38
|
expect(decompressed).toBe(input);
|
|
37
39
|
});
|
|
38
40
|
it('[edge] should return undefined and log error when compression fails', () => {
|
|
@@ -45,7 +47,7 @@ describe('uploader.helpers', () => {
|
|
|
45
47
|
// Act
|
|
46
48
|
const result = (0, uploader_helpers_1.compressGzip)('test data');
|
|
47
49
|
// Assert
|
|
48
|
-
expect(result).toBeUndefined();
|
|
50
|
+
expect(result.response).toBeUndefined();
|
|
49
51
|
gzipSyncSpy.mockRestore();
|
|
50
52
|
});
|
|
51
53
|
});
|
|
@@ -57,7 +59,7 @@ describe('uploader.helpers', () => {
|
|
|
57
59
|
// Act
|
|
58
60
|
const result = (0, uploader_helpers_1.decompressGzip)(compressed);
|
|
59
61
|
// Assert
|
|
60
|
-
expect(result).toBe(originalString);
|
|
62
|
+
expect(result.response).toBe(originalString);
|
|
61
63
|
});
|
|
62
64
|
it('[edge] should return undefined and log error when decompression fails', () => {
|
|
63
65
|
// Arrange
|
|
@@ -65,7 +67,7 @@ describe('uploader.helpers', () => {
|
|
|
65
67
|
// Act
|
|
66
68
|
const result = (0, uploader_helpers_1.decompressGzip)(invalidGzip);
|
|
67
69
|
// Assert
|
|
68
|
-
expect(result).toBeUndefined();
|
|
70
|
+
expect(result.response).toBeUndefined();
|
|
69
71
|
});
|
|
70
72
|
});
|
|
71
73
|
describe(uploader_helpers_1.parseJsonl.name, () => {
|
|
@@ -79,7 +81,7 @@ describe('uploader.helpers', () => {
|
|
|
79
81
|
// Act
|
|
80
82
|
const result = (0, uploader_helpers_1.parseJsonl)(jsonlString);
|
|
81
83
|
// Assert
|
|
82
|
-
expect(result).toEqual(data);
|
|
84
|
+
expect(result.response).toEqual(data);
|
|
83
85
|
});
|
|
84
86
|
it('[edge] should return null and log error when parsing fails', () => {
|
|
85
87
|
// Arrange
|
|
@@ -87,7 +89,7 @@ describe('uploader.helpers', () => {
|
|
|
87
89
|
// Act
|
|
88
90
|
const result = (0, uploader_helpers_1.parseJsonl)(invalidJsonl);
|
|
89
91
|
// Assert
|
|
90
|
-
expect(result).
|
|
92
|
+
expect(result.response).toBeUndefined();
|
|
91
93
|
});
|
|
92
94
|
});
|
|
93
95
|
describe(uploader_helpers_1.downloadToLocal.name, () => {
|
|
@@ -6,6 +6,17 @@ export interface UploaderFactoryInterface {
|
|
|
6
6
|
event: AirdropEvent;
|
|
7
7
|
options?: WorkerAdapterOptions;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Generic result type for uploader operations that can either succeed with a response or fail with an error.
|
|
11
|
+
* @template T The type of the successful response data
|
|
12
|
+
*/
|
|
13
|
+
export type UploaderResult<T> = {
|
|
14
|
+
response: T;
|
|
15
|
+
error?: never;
|
|
16
|
+
} | {
|
|
17
|
+
response?: never;
|
|
18
|
+
error: unknown;
|
|
19
|
+
};
|
|
9
20
|
/**
|
|
10
21
|
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
11
22
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
@@ -31,33 +31,34 @@ class Uploader {
|
|
|
31
31
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
32
32
|
}
|
|
33
33
|
// Compress the fetched objects to a gzipped jsonl object
|
|
34
|
-
const file = (0, uploader_helpers_1.compressGzip)(js_jsonl_1.jsonl.stringify(fetchedObjects));
|
|
35
|
-
if (
|
|
34
|
+
const { response: file, error: fileError } = (0, uploader_helpers_1.compressGzip)(js_jsonl_1.jsonl.stringify(fetchedObjects));
|
|
35
|
+
if (fileError) {
|
|
36
36
|
return {
|
|
37
|
-
error: new Error('Error while compressing jsonl object.'),
|
|
37
|
+
error: new Error('Error while compressing jsonl object. ' + (0, logger_1.serializeError)(fileError)),
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
40
|
const filename = itemType + '.jsonl.gz';
|
|
41
41
|
const fileType = 'application/x-gzip';
|
|
42
42
|
// Get upload url
|
|
43
|
-
const preparedArtifact = await this.getArtifactUploadUrl(filename, fileType);
|
|
44
|
-
if (
|
|
43
|
+
const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType);
|
|
44
|
+
if (preparedArtifactError) {
|
|
45
45
|
return {
|
|
46
46
|
error: new Error('Error while getting artifact upload URL.'),
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
49
|
// Upload prepared artifact to the given url
|
|
50
|
-
const
|
|
51
|
-
if (
|
|
50
|
+
const { error: uploadItemError } = await this.uploadArtifact(preparedArtifact, file);
|
|
51
|
+
if (uploadItemError) {
|
|
52
52
|
return {
|
|
53
53
|
error: new Error('Error while uploading artifact.'),
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
// Confirm upload
|
|
57
|
-
const
|
|
58
|
-
if (
|
|
57
|
+
const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
58
|
+
if (confirmArtifactUploadError) {
|
|
59
59
|
return {
|
|
60
|
-
error: new Error('Error while confirming artifact upload.'
|
|
60
|
+
error: new Error('Error while confirming artifact upload. ' +
|
|
61
|
+
JSON.stringify(confirmArtifactUploadError)),
|
|
61
62
|
};
|
|
62
63
|
}
|
|
63
64
|
// Return the artifact information to the platform
|
|
@@ -87,10 +88,10 @@ class Uploader {
|
|
|
87
88
|
file_size: fileSize,
|
|
88
89
|
},
|
|
89
90
|
});
|
|
90
|
-
return response.data;
|
|
91
|
+
return { response: response.data };
|
|
91
92
|
}
|
|
92
93
|
catch (error) {
|
|
93
|
-
|
|
94
|
+
return { error };
|
|
94
95
|
}
|
|
95
96
|
}
|
|
96
97
|
/**
|
|
@@ -109,10 +110,10 @@ class Uploader {
|
|
|
109
110
|
const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
|
|
110
111
|
headers: Object.assign({}, formData.getHeaders()),
|
|
111
112
|
});
|
|
112
|
-
return response;
|
|
113
|
+
return { response };
|
|
113
114
|
}
|
|
114
115
|
catch (error) {
|
|
115
|
-
|
|
116
|
+
return { error };
|
|
116
117
|
}
|
|
117
118
|
}
|
|
118
119
|
/**
|
|
@@ -140,12 +141,11 @@ class Uploader {
|
|
|
140
141
|
validateStatus: (status) => status >= 200 && status < 400,
|
|
141
142
|
});
|
|
142
143
|
this.destroyStream(fileStream);
|
|
143
|
-
return response;
|
|
144
|
+
return { response };
|
|
144
145
|
}
|
|
145
146
|
catch (error) {
|
|
146
|
-
console.error('Error while streaming artifact.', (0, logger_1.serializeError)(error));
|
|
147
147
|
this.destroyStream(fileStream);
|
|
148
|
-
return;
|
|
148
|
+
return { error };
|
|
149
149
|
}
|
|
150
150
|
}
|
|
151
151
|
/**
|
|
@@ -162,10 +162,19 @@ class Uploader {
|
|
|
162
162
|
}, {
|
|
163
163
|
headers: Object.assign({}, this.defaultHeaders),
|
|
164
164
|
});
|
|
165
|
-
return response
|
|
165
|
+
// If response exists and the status is 2xx, return the response
|
|
166
|
+
if ((response === null || response === void 0 ? void 0 : response.status) >= 200 && (response === null || response === void 0 ? void 0 : response.status) < 300) {
|
|
167
|
+
return { response };
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
return {
|
|
171
|
+
error: new Error('Error while confirming artifact upload. ' +
|
|
172
|
+
JSON.stringify(response)),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
166
175
|
}
|
|
167
176
|
catch (error) {
|
|
168
|
-
|
|
177
|
+
return { error: (0, logger_1.serializeError)(error) };
|
|
169
178
|
}
|
|
170
179
|
}
|
|
171
180
|
/**
|
|
@@ -196,31 +205,34 @@ class Uploader {
|
|
|
196
205
|
*/
|
|
197
206
|
async getAttachmentsFromArtifactId({ artifact, }) {
|
|
198
207
|
// Get the URL of the attachments metadata artifact
|
|
199
|
-
const artifactUrl = await this.getArtifactDownloadUrl(artifact);
|
|
200
|
-
if (
|
|
208
|
+
const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifact);
|
|
209
|
+
if (artifactUrlError) {
|
|
201
210
|
return {
|
|
202
|
-
error: new Error('Error while getting artifact download URL.'
|
|
211
|
+
error: new Error('Error while getting artifact download URL. ' +
|
|
212
|
+
(0, logger_1.serializeError)(artifactUrlError)),
|
|
203
213
|
};
|
|
204
214
|
}
|
|
205
215
|
// Download artifact from the URL
|
|
206
|
-
const gzippedJsonlObject = await this.downloadArtifact(artifactUrl);
|
|
207
|
-
if (
|
|
216
|
+
const { response: gzippedJsonlObject, error: gzippedJsonlObjectError } = await this.downloadArtifact(artifactUrl);
|
|
217
|
+
if (gzippedJsonlObjectError) {
|
|
208
218
|
return {
|
|
209
|
-
error: new Error('Error while downloading gzipped jsonl object.'
|
|
219
|
+
error: new Error('Error while downloading gzipped jsonl object.' +
|
|
220
|
+
(0, logger_1.serializeError)(gzippedJsonlObjectError)),
|
|
210
221
|
};
|
|
211
222
|
}
|
|
212
223
|
// Decompress the gzipped jsonl object
|
|
213
|
-
const jsonlObject = (0, uploader_helpers_1.decompressGzip)(gzippedJsonlObject);
|
|
214
|
-
if (
|
|
224
|
+
const { response: jsonlObject, error: jsonlObjectError } = (0, uploader_helpers_1.decompressGzip)(gzippedJsonlObject);
|
|
225
|
+
if (jsonlObjectError) {
|
|
215
226
|
return {
|
|
216
|
-
error: new Error('Error while decompressing gzipped jsonl object.'
|
|
227
|
+
error: new Error('Error while decompressing gzipped jsonl object.' +
|
|
228
|
+
(0, logger_1.serializeError)(jsonlObjectError)),
|
|
217
229
|
};
|
|
218
230
|
}
|
|
219
231
|
// Parse the jsonl object to get the attachment metadata
|
|
220
|
-
const jsonObject = (0, uploader_helpers_1.parseJsonl)(jsonlObject);
|
|
221
|
-
if (
|
|
232
|
+
const { response: jsonObject, error: jsonObjectError } = (0, uploader_helpers_1.parseJsonl)(jsonlObject);
|
|
233
|
+
if (jsonObjectError) {
|
|
222
234
|
return {
|
|
223
|
-
error: new Error('Error while parsing jsonl object.'),
|
|
235
|
+
error: new Error('Error while parsing jsonl object.' + (0, logger_1.serializeError)(jsonObjectError)),
|
|
224
236
|
};
|
|
225
237
|
}
|
|
226
238
|
return { attachments: jsonObject };
|
|
@@ -240,10 +252,10 @@ class Uploader {
|
|
|
240
252
|
artifact_id: artifactId,
|
|
241
253
|
},
|
|
242
254
|
});
|
|
243
|
-
return response.data.download_url;
|
|
255
|
+
return { response: response.data.download_url };
|
|
244
256
|
}
|
|
245
257
|
catch (error) {
|
|
246
|
-
|
|
258
|
+
return { error };
|
|
247
259
|
}
|
|
248
260
|
}
|
|
249
261
|
/**
|
|
@@ -256,10 +268,10 @@ class Uploader {
|
|
|
256
268
|
const response = await axios_client_internal_1.axiosClient.get(artifactUrl, {
|
|
257
269
|
responseType: 'arraybuffer',
|
|
258
270
|
});
|
|
259
|
-
return response.data;
|
|
271
|
+
return { response: response.data };
|
|
260
272
|
}
|
|
261
273
|
catch (error) {
|
|
262
|
-
|
|
274
|
+
return { error };
|
|
263
275
|
}
|
|
264
276
|
}
|
|
265
277
|
/**
|
|
@@ -270,24 +282,24 @@ class Uploader {
|
|
|
270
282
|
* @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
|
|
271
283
|
*/
|
|
272
284
|
async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
|
|
273
|
-
const artifactUrl = await this.getArtifactDownloadUrl(artifactId);
|
|
274
|
-
if (
|
|
275
|
-
return;
|
|
285
|
+
const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifactId);
|
|
286
|
+
if (artifactUrlError) {
|
|
287
|
+
return { error: artifactUrlError };
|
|
276
288
|
}
|
|
277
|
-
const artifact = await this.downloadArtifact(artifactUrl);
|
|
278
|
-
if (
|
|
279
|
-
return;
|
|
289
|
+
const { response: artifact, error: artifactError } = await this.downloadArtifact(artifactUrl);
|
|
290
|
+
if (artifactError) {
|
|
291
|
+
return { error: artifactError };
|
|
280
292
|
}
|
|
281
293
|
if (isGzipped) {
|
|
282
|
-
const decompressedArtifact = (0, uploader_helpers_1.decompressGzip)(artifact);
|
|
283
|
-
if (
|
|
284
|
-
return;
|
|
294
|
+
const { response: decompressedArtifact, error: decompressedArtifactError, } = (0, uploader_helpers_1.decompressGzip)(artifact);
|
|
295
|
+
if (decompressedArtifactError) {
|
|
296
|
+
return { error: decompressedArtifactError };
|
|
285
297
|
}
|
|
286
298
|
const jsonlObject = Buffer.from(decompressedArtifact).toString('utf-8');
|
|
287
|
-
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
299
|
+
return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
|
|
288
300
|
}
|
|
289
301
|
const jsonlObject = Buffer.from(artifact).toString('utf-8');
|
|
290
|
-
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
302
|
+
return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
|
|
291
303
|
}
|
|
292
304
|
}
|
|
293
305
|
exports.Uploader = Uploader;
|
|
@@ -118,7 +118,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
118
118
|
// Arrange
|
|
119
119
|
const itemType = 'tasks';
|
|
120
120
|
const fetchedObjects = [{ id: 1 }];
|
|
121
|
-
mockedCompressGzip.mockReturnValueOnce(
|
|
121
|
+
mockedCompressGzip.mockReturnValueOnce({ error: 'Mock error' });
|
|
122
122
|
// Act
|
|
123
123
|
const result = await uploader.upload(itemType, fetchedObjects);
|
|
124
124
|
// Assert
|
|
@@ -139,7 +139,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
139
139
|
// Act
|
|
140
140
|
const result = await uploader.getArtifactUploadUrl(filename, fileType, fileSize);
|
|
141
141
|
// Assert
|
|
142
|
-
expect(result).toEqual(expectedArtifact);
|
|
142
|
+
expect(result.response).toEqual(expectedArtifact);
|
|
143
143
|
expect(mockedAxiosClient.get).toHaveBeenCalledWith(expect.stringContaining('/internal/airdrop.artifacts.upload-url'), expect.objectContaining({
|
|
144
144
|
params: expect.objectContaining({
|
|
145
145
|
file_type: fileType,
|
|
@@ -156,7 +156,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
156
156
|
// Act
|
|
157
157
|
const result = await uploader.getArtifactUploadUrl(filename, fileType);
|
|
158
158
|
// Assert
|
|
159
|
-
expect(result).toBeUndefined();
|
|
159
|
+
expect(result.response).toBeUndefined();
|
|
160
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
160
161
|
expect(mockedAxiosClient.get).toHaveBeenCalled();
|
|
161
162
|
});
|
|
162
163
|
});
|
|
@@ -170,7 +171,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
170
171
|
// Act
|
|
171
172
|
const result = await uploader.uploadArtifact(artifact, file);
|
|
172
173
|
// Assert
|
|
173
|
-
expect(result).toBe(mockResponse);
|
|
174
|
+
expect(result.response).toBe(mockResponse);
|
|
174
175
|
expect(mockedAxiosClient.post).toHaveBeenCalledWith(artifact.upload_url, expect.any(form_data_1.default), expect.objectContaining({
|
|
175
176
|
headers: expect.any(Object),
|
|
176
177
|
}));
|
|
@@ -203,7 +204,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
203
204
|
// Act
|
|
204
205
|
const result = await uploader.uploadArtifact(artifact, file);
|
|
205
206
|
// Assert
|
|
206
|
-
expect(result).toBeUndefined();
|
|
207
|
+
expect(result.response).toBeUndefined();
|
|
208
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
207
209
|
});
|
|
208
210
|
});
|
|
209
211
|
describe(uploader_1.Uploader.prototype.confirmArtifactUpload.name, () => {
|
|
@@ -215,7 +217,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
215
217
|
// Act
|
|
216
218
|
const result = await uploader.confirmArtifactUpload(artifactId);
|
|
217
219
|
// Assert
|
|
218
|
-
expect(result).toBe(mockResponse);
|
|
220
|
+
expect(result.response).toBe(mockResponse);
|
|
219
221
|
expect(mockedAxiosClient.post).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
|
|
220
222
|
artifact_id: artifactId,
|
|
221
223
|
}), expect.any(Object));
|
|
@@ -227,7 +229,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
227
229
|
// Act
|
|
228
230
|
const result = await uploader.confirmArtifactUpload(artifactId);
|
|
229
231
|
// Assert
|
|
230
|
-
expect(result).toBeUndefined();
|
|
232
|
+
expect(result.response).toBeUndefined();
|
|
233
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
231
234
|
});
|
|
232
235
|
});
|
|
233
236
|
describe(uploader_1.Uploader.prototype.streamArtifact.name, () => {
|
|
@@ -244,7 +247,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
244
247
|
// Act
|
|
245
248
|
const result = await uploader.streamArtifact(artifact, fileStream);
|
|
246
249
|
// Assert
|
|
247
|
-
expect(result).toBe(mockResponse);
|
|
250
|
+
expect(result.response).toBe(mockResponse);
|
|
248
251
|
expect(axios_client_internal_1.axiosClient.post).toHaveBeenCalledWith(artifact.upload_url, expect.any(form_data_1.default), expect.objectContaining({
|
|
249
252
|
headers: expect.any(Object),
|
|
250
253
|
maxRedirects: 0,
|
|
@@ -315,7 +318,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
315
318
|
// Act
|
|
316
319
|
const result = await uploader.streamArtifact(artifact, fileStream);
|
|
317
320
|
// Assert
|
|
318
|
-
expect(result).toBeUndefined();
|
|
321
|
+
expect(result.response).toBeUndefined();
|
|
322
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
319
323
|
expect(destroyFn).toHaveBeenCalled();
|
|
320
324
|
});
|
|
321
325
|
});
|
|
@@ -405,7 +409,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
405
409
|
// Act
|
|
406
410
|
const result = await getArtifactDownloadUrl(artifactId);
|
|
407
411
|
// Assert
|
|
408
|
-
expect(result).toBe(expectedDownloadUrl);
|
|
412
|
+
expect(result.response).toBe(expectedDownloadUrl);
|
|
413
|
+
expect(result.error).toBeUndefined();
|
|
409
414
|
expect(mockedAxiosClient.get).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
|
|
410
415
|
params: expect.objectContaining({
|
|
411
416
|
artifact_id: artifactId,
|
|
@@ -420,7 +425,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
420
425
|
// Act
|
|
421
426
|
const result = await getArtifactDownloadUrl(artifactId);
|
|
422
427
|
// Assert
|
|
423
|
-
expect(result).toBeUndefined();
|
|
428
|
+
expect(result.response).toBeUndefined();
|
|
429
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
424
430
|
});
|
|
425
431
|
});
|
|
426
432
|
describe(uploader_1.Uploader.prototype.getAttachmentsFromArtifactId.name, () => {
|
|
@@ -447,7 +453,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
447
453
|
it('should return error when getArtifactDownloadUrl fails', async () => {
|
|
448
454
|
// Arrange
|
|
449
455
|
const artifactId = 'art_123';
|
|
450
|
-
(0, test_helpers_1.spyOnPrivateMethod)(uploader, 'getArtifactDownloadUrl').mockResolvedValueOnce(
|
|
456
|
+
(0, test_helpers_1.spyOnPrivateMethod)(uploader, 'getArtifactDownloadUrl').mockResolvedValueOnce({ error: new Error('API error') });
|
|
451
457
|
// Act
|
|
452
458
|
const result = await uploader.getAttachmentsFromArtifactId({
|
|
453
459
|
artifact: artifactId,
|
|
@@ -520,7 +526,7 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
520
526
|
isGzipped: false,
|
|
521
527
|
});
|
|
522
528
|
// Assert
|
|
523
|
-
expect(result).toEqual(mockData);
|
|
529
|
+
expect(result.response).toEqual(mockData);
|
|
524
530
|
});
|
|
525
531
|
it('should return parsed data when downloading gzipped artifact', async () => {
|
|
526
532
|
// Arrange
|
|
@@ -540,18 +546,19 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
540
546
|
isGzipped: true,
|
|
541
547
|
});
|
|
542
548
|
// Assert
|
|
543
|
-
expect(result).toEqual(mockData);
|
|
549
|
+
expect(result.response).toEqual(mockData);
|
|
544
550
|
});
|
|
545
|
-
it('[edge] should return
|
|
551
|
+
it('[edge] should return error when getArtifactDownloadUrl fails', async () => {
|
|
546
552
|
// Arrange
|
|
547
553
|
const artifactId = 'art_123';
|
|
548
|
-
(0, test_helpers_1.spyOnPrivateMethod)(uploader, 'getArtifactDownloadUrl').mockResolvedValueOnce(
|
|
554
|
+
(0, test_helpers_1.spyOnPrivateMethod)(uploader, 'getArtifactDownloadUrl').mockResolvedValueOnce({ error: new Error('API error') });
|
|
549
555
|
// Act
|
|
550
556
|
const result = await uploader.getJsonObjectByArtifactId({ artifactId });
|
|
551
557
|
// Assert
|
|
552
|
-
expect(result).toBeUndefined();
|
|
558
|
+
expect(result.response).toBeUndefined();
|
|
559
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
553
560
|
});
|
|
554
|
-
it('[edge] should return
|
|
561
|
+
it('[edge] should return error when downloadArtifact fails', async () => {
|
|
555
562
|
// Arrange
|
|
556
563
|
const artifactId = 'art_123';
|
|
557
564
|
mockedAxiosClient.get.mockResolvedValueOnce((0, test_helpers_1.createDownloadUrlResponse)());
|
|
@@ -559,9 +566,10 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
559
566
|
// Act
|
|
560
567
|
const result = await uploader.getJsonObjectByArtifactId({ artifactId });
|
|
561
568
|
// Assert
|
|
562
|
-
expect(result).toBeUndefined();
|
|
569
|
+
expect(result.response).toBeUndefined();
|
|
570
|
+
expect(result.error).toBeInstanceOf(Error);
|
|
563
571
|
});
|
|
564
|
-
it('[edge] should return
|
|
572
|
+
it('[edge] should return error when decompression fails for gzipped artifact', async () => {
|
|
565
573
|
// Arrange
|
|
566
574
|
const artifactId = 'art_123';
|
|
567
575
|
mockedAxiosClient.get.mockResolvedValueOnce((0, test_helpers_1.createDownloadUrlResponse)());
|
|
@@ -574,7 +582,8 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
574
582
|
isGzipped: true,
|
|
575
583
|
});
|
|
576
584
|
// Assert
|
|
577
|
-
expect(result).toBeUndefined();
|
|
585
|
+
expect(result.response).toBeUndefined();
|
|
586
|
+
expect(result.error).toBeDefined();
|
|
578
587
|
});
|
|
579
588
|
});
|
|
580
589
|
});
|
package/package.json
CHANGED