@devrev/ts-adaas 1.13.0 → 1.13.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/install-initial-domain-mapping.test.js +1 -1
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +10 -15
- package/dist/http/axios-client-internal.test.js +126 -161
- package/dist/tests/dummy-connector/data-extraction.d.ts +2 -0
- package/dist/tests/dummy-connector/data-extraction.d.ts.map +1 -0
- package/dist/tests/dummy-connector/data-extraction.js +43 -0
- package/dist/tests/dummy-connector/data-extraction.test.d.ts +2 -0
- package/dist/tests/dummy-connector/data-extraction.test.d.ts.map +1 -0
- package/dist/tests/dummy-connector/data-extraction.test.js +24 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.d.ts +2 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.d.ts.map +1 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.js +26 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.test.d.ts +2 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.test.d.ts.map +1 -0
- package/dist/tests/dummy-connector/external-sync-units-extraction.test.js +21 -0
- package/dist/tests/dummy-connector/extraction.d.ts +4 -0
- package/dist/tests/dummy-connector/extraction.d.ts.map +1 -0
- package/dist/tests/dummy-connector/extraction.js +19 -0
- package/dist/tests/dummy-connector/metadata-extraction.d.ts +2 -0
- package/dist/tests/dummy-connector/metadata-extraction.d.ts.map +1 -0
- package/dist/tests/dummy-connector/metadata-extraction.js +23 -0
- package/dist/tests/dummy-connector/metadata-extraction.test.d.ts +2 -0
- package/dist/tests/dummy-connector/metadata-extraction.test.d.ts.map +1 -0
- package/dist/tests/dummy-connector/metadata-extraction.test.js +54 -0
- package/dist/tests/jest.setup.d.ts +3 -0
- package/dist/tests/jest.setup.d.ts.map +1 -0
- package/dist/tests/jest.setup.js +14 -0
- package/dist/tests/mock-server/mock-server.d.ts +79 -0
- package/dist/tests/mock-server/mock-server.d.ts.map +1 -0
- package/dist/tests/mock-server/mock-server.interfaces.d.ts +53 -0
- package/dist/tests/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/tests/mock-server/mock-server.interfaces.js +4 -0
- package/dist/tests/mock-server/mock-server.js +237 -0
- package/dist/tests/test-helpers.d.ts.map +1 -1
- package/dist/tests/test-helpers.js +4 -3
- package/dist/tests/timeout-handling/extraction.d.ts.map +1 -1
- package/dist/tests/timeout-handling/no-timeout.d.ts +2 -0
- package/dist/tests/timeout-handling/no-timeout.d.ts.map +1 -0
- package/dist/tests/timeout-handling/{timeout-1.js → no-timeout.js} +1 -1
- package/dist/tests/timeout-handling/no-timeout.test.d.ts +2 -0
- package/dist/tests/timeout-handling/no-timeout.test.d.ts.map +1 -0
- package/dist/tests/timeout-handling/no-timeout.test.js +24 -0
- package/dist/tests/timeout-handling/timeout-blocked.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-blocked.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-blocked.js +23 -0
- package/dist/tests/timeout-handling/timeout-blocked.test.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-blocked.test.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-blocked.test.js +25 -0
- package/dist/tests/timeout-handling/timeout-graceful.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-graceful.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-graceful.js +17 -0
- package/dist/tests/timeout-handling/timeout-graceful.test.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-graceful.test.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-graceful.test.js +25 -0
- package/dist/tests/timeout-handling/timeout-unblocked.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-unblocked.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-unblocked.js +26 -0
- package/dist/tests/timeout-handling/timeout-unblocked.test.d.ts +2 -0
- package/dist/tests/timeout-handling/timeout-unblocked.test.d.ts.map +1 -0
- package/dist/tests/timeout-handling/timeout-unblocked.test.js +25 -0
- package/dist/uploader/uploader.d.ts +72 -8
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.js +73 -8
- package/dist/workers/spawn.d.ts.map +1 -1
- package/dist/workers/spawn.js +28 -0
- package/dist/workers/worker-adapter.d.ts.map +1 -1
- package/dist/workers/worker-adapter.js +7 -4
- package/package.json +3 -8
- package/dist/tests/mock-server.d.ts +0 -21
- package/dist/tests/mock-server.d.ts.map +0 -1
- package/dist/tests/mock-server.js +0 -280
- package/dist/tests/timeout-handling/timeout-1.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-1.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-1.test.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-1.test.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-1.test.js +0 -45
- package/dist/tests/timeout-handling/timeout-2.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-2.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-2.js +0 -44
- package/dist/tests/timeout-handling/timeout-2.test.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-2.test.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-2.test.js +0 -46
- package/dist/tests/timeout-handling/timeout-3a.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-3a.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-3a.js +0 -37
- package/dist/tests/timeout-handling/timeout-3a.test.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-3a.test.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-3a.test.js +0 -46
- package/dist/tests/timeout-handling/timeout-3b.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-3b.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-3b.js +0 -38
- package/dist/tests/timeout-handling/timeout-3b.test.d.ts +0 -2
- package/dist/tests/timeout-handling/timeout-3b.test.d.ts.map +0 -1
- package/dist/tests/timeout-handling/timeout-3b.test.js +0 -46
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timeout-graceful.test.d.ts","sourceRoot":"","sources":["../../../src/tests/timeout-handling/timeout-graceful.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
const jest_setup_1 = require("../jest.setup");
|
|
8
|
+
const test_helpers_1 = require("../test-helpers");
|
|
9
|
+
const extraction_2 = __importDefault(require("./extraction"));
|
|
10
|
+
jest.setTimeout(10000);
|
|
11
|
+
describe('Timeout graceful', () => {
|
|
12
|
+
let event;
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
event = (0, test_helpers_1.createEvent)({
|
|
15
|
+
eventType: extraction_1.EventType.StartExtractingData,
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
it('should emit progress event when timeout is reached but event loop is not blocked', async () => {
|
|
19
|
+
await (0, extraction_2.default)([event], __dirname + '/timeout-graceful');
|
|
20
|
+
const lastRequest = jest_setup_1.mockServer.getLastRequest();
|
|
21
|
+
expect(lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.url).toContain('/callback_url');
|
|
22
|
+
expect(lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.method).toBe('POST');
|
|
23
|
+
expect((lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.body).event_type).toBe(extraction_1.ExtractorEventType.DataExtractionProgress);
|
|
24
|
+
});
|
|
25
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timeout-unblocked.d.ts","sourceRoot":"","sources":["../../../src/tests/timeout-handling/timeout-unblocked.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const helpers_1 = require("../../common/helpers");
|
|
4
|
+
const index_1 = require("../../index");
|
|
5
|
+
(0, index_1.processTask)({
|
|
6
|
+
task: async ({ adapter }) => {
|
|
7
|
+
// CPU-intensive nested loops that yield control after logging
|
|
8
|
+
// This allows the event loop to process timeout messages
|
|
9
|
+
let result = 0;
|
|
10
|
+
for (let i = 0; i < 100000; i++) {
|
|
11
|
+
for (let j = 0; j < 10000; j++) {
|
|
12
|
+
result += Math.sqrt(i * j) * Math.sin(i + j);
|
|
13
|
+
result = Math.abs(result) % 1000000;
|
|
14
|
+
}
|
|
15
|
+
// Log every 1000 iterations and yield control to event loop
|
|
16
|
+
if (i % 1000 === 0) {
|
|
17
|
+
console.log(`timeout-unblocked iteration ${i}`);
|
|
18
|
+
await (0, helpers_1.sleep)(0);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
await adapter.emit(index_1.ExtractorEventType.DataExtractionDone);
|
|
22
|
+
},
|
|
23
|
+
onTimeout: async ({ adapter }) => {
|
|
24
|
+
await adapter.emit(index_1.ExtractorEventType.DataExtractionProgress);
|
|
25
|
+
},
|
|
26
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timeout-unblocked.test.d.ts","sourceRoot":"","sources":["../../../src/tests/timeout-handling/timeout-unblocked.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
const jest_setup_1 = require("../jest.setup");
|
|
8
|
+
const test_helpers_1 = require("../test-helpers");
|
|
9
|
+
const extraction_2 = __importDefault(require("./extraction"));
|
|
10
|
+
jest.setTimeout(10000); // 10 seconds
|
|
11
|
+
describe('Timeout unblocked', () => {
|
|
12
|
+
let event;
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
event = (0, test_helpers_1.createEvent)({
|
|
15
|
+
eventType: extraction_1.EventType.StartExtractingData,
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
it('should emit progress event when timeout is reached and event loop can process messages', async () => {
|
|
19
|
+
await (0, extraction_2.default)([event], __dirname + '/timeout-unblocked');
|
|
20
|
+
const lastRequest = jest_setup_1.mockServer.getLastRequest();
|
|
21
|
+
expect(lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.url).toContain('/callback_url');
|
|
22
|
+
expect(lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.method).toBe('POST');
|
|
23
|
+
expect((lastRequest === null || lastRequest === void 0 ? void 0 : lastRequest.body).event_type).toBe(extraction_1.ExtractorEventType.DataExtractionProgress);
|
|
24
|
+
});
|
|
25
|
+
});
|
|
@@ -2,7 +2,6 @@ import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
|
2
2
|
import { AxiosResponse } from 'axios';
|
|
3
3
|
import { ArtifactToUpload, UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
|
|
4
4
|
export declare class Uploader {
|
|
5
|
-
private event;
|
|
6
5
|
private isLocalDevelopment?;
|
|
7
6
|
private devrevApiEndpoint;
|
|
8
7
|
private devrevApiToken;
|
|
@@ -10,24 +9,51 @@ export declare class Uploader {
|
|
|
10
9
|
private defaultHeaders;
|
|
11
10
|
constructor({ event, options }: UploaderFactoryInterface);
|
|
12
11
|
/**
|
|
13
|
-
* Uploads the fetched objects to the DevRev platform.
|
|
14
|
-
* The fetched objects are uploaded to the platform and the artifact information is returned.
|
|
15
|
-
* @param {string} filename - The name of the file to be uploaded
|
|
12
|
+
* Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform.
|
|
16
13
|
* @param {string} itemType - The type of the item to be uploaded
|
|
17
|
-
* @param {object[] | object} fetchedObjects - The
|
|
18
|
-
* @returns {Promise<UploadResponse>} - The response object containing the artifact information
|
|
19
|
-
* or error information if there was an error
|
|
14
|
+
* @param {object[] | object} fetchedObjects - The objects to be uploaded
|
|
15
|
+
* @returns {Promise<UploadResponse>} - The response object containing the artifact information or error information if there was an error
|
|
20
16
|
*/
|
|
21
17
|
upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
|
|
22
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Gets the upload URL for an artifact from the DevRev API.
|
|
20
|
+
* @param {string} filename - The name of the file to upload
|
|
21
|
+
* @param {string} fileType - The MIME type of the file
|
|
22
|
+
* @param {number} [fileSize] - Optional file size in bytes
|
|
23
|
+
* @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
|
|
24
|
+
*/
|
|
25
|
+
getArtifactUploadUrl(filename: string, fileType: string, fileSize?: number): Promise<ArtifactToUpload | void>;
|
|
26
|
+
/**
|
|
27
|
+
* Uploads an artifact file to the provided upload URL using multipart form data.
|
|
28
|
+
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
29
|
+
* @param {Buffer} file - The file buffer to upload
|
|
30
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
31
|
+
*/
|
|
23
32
|
uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<AxiosResponse | void>;
|
|
33
|
+
/**
|
|
34
|
+
* Streams an artifact file from an axios response to the upload URL.
|
|
35
|
+
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
36
|
+
* @param {AxiosResponse} fileStream - The axios response stream containing the file data
|
|
37
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
38
|
+
*/
|
|
24
39
|
streamArtifact(artifact: ArtifactToUpload, fileStream: AxiosResponse): Promise<AxiosResponse | void>;
|
|
40
|
+
/**
|
|
41
|
+
* Confirms that an artifact upload has been completed successfully.
|
|
42
|
+
* @param {string} artifactId - The ID of the artifact to confirm
|
|
43
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
44
|
+
*/
|
|
25
45
|
confirmArtifactUpload(artifactId: string): Promise<AxiosResponse | void>;
|
|
26
46
|
/**
|
|
27
47
|
* Destroys a stream to prevent resource leaks.
|
|
28
48
|
* @param {any} fileStream - The axios response stream to destroy
|
|
29
49
|
*/
|
|
30
50
|
private destroyStream;
|
|
51
|
+
/**
|
|
52
|
+
* Retrieves attachment metadata from an artifact by downloading and parsing it.
|
|
53
|
+
* @param {object} param0 - Configuration object
|
|
54
|
+
* @param {string} param0.artifact - The artifact ID to download attachments from
|
|
55
|
+
* @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object
|
|
56
|
+
*/
|
|
31
57
|
getAttachmentsFromArtifactId({ artifact, }: {
|
|
32
58
|
artifact: string;
|
|
33
59
|
}): Promise<{
|
|
@@ -36,15 +62,53 @@ export declare class Uploader {
|
|
|
36
62
|
message: string;
|
|
37
63
|
};
|
|
38
64
|
}>;
|
|
65
|
+
/**
|
|
66
|
+
* Gets the download URL for an artifact from the DevRev API.
|
|
67
|
+
* @param {string} artifactId - The ID of the artifact to download
|
|
68
|
+
* @returns {Promise<string | void>} The download URL or undefined on error
|
|
69
|
+
*/
|
|
39
70
|
private getArtifactDownloadUrl;
|
|
71
|
+
/**
|
|
72
|
+
* Downloads an artifact file from the given URL.
|
|
73
|
+
* @param {string} artifactUrl - The URL to download the artifact from
|
|
74
|
+
* @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
|
|
75
|
+
*/
|
|
40
76
|
private downloadArtifact;
|
|
77
|
+
/**
|
|
78
|
+
* Compresses a JSONL string using gzip compression.
|
|
79
|
+
* @param {string} jsonlObject - The JSONL string to compress
|
|
80
|
+
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
81
|
+
*/
|
|
41
82
|
private compressGzip;
|
|
83
|
+
/**
|
|
84
|
+
* Decompresses a gzipped buffer to a JSONL string.
|
|
85
|
+
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
86
|
+
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
87
|
+
*/
|
|
42
88
|
private decompressGzip;
|
|
89
|
+
/**
|
|
90
|
+
* Parses a JSONL string into an array of objects.
|
|
91
|
+
* @param {string} jsonlObject - The JSONL string to parse
|
|
92
|
+
* @returns {object[] | null} The parsed array of objects or null on error
|
|
93
|
+
*/
|
|
43
94
|
private parseJsonl;
|
|
95
|
+
/**
|
|
96
|
+
* Retrieves and parses JSON objects from an artifact by artifact ID.
|
|
97
|
+
* @param {object} param0 - Configuration object
|
|
98
|
+
* @param {string} param0.artifactId - The artifact ID to download and parse
|
|
99
|
+
* @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped
|
|
100
|
+
* @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
|
|
101
|
+
*/
|
|
44
102
|
getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
|
|
45
103
|
artifactId: string;
|
|
46
104
|
isGzipped?: boolean;
|
|
47
105
|
}): Promise<object[] | object | void>;
|
|
106
|
+
/**
|
|
107
|
+
* Downloads fetched objects to the local file system (for local development).
|
|
108
|
+
* @param {string} itemType - The type of items being downloaded
|
|
109
|
+
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
110
|
+
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
111
|
+
*/
|
|
48
112
|
private downloadToLocal;
|
|
49
113
|
}
|
|
50
114
|
//# sourceMappingURL=uploader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAUxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwD1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAwBnC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmBhC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA+BhC;;;;OAIG;IACG,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IAqCF;;;;OAIG;YACW,sBAAsB;IAyBpC;;;;OAIG;YACW,gBAAgB;IAe9B;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAQpB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAStB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IASlB;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;IAyBrC;;;;;OAKG;YACW,eAAe;CAgC9B"}
|
|
@@ -47,7 +47,6 @@ const helpers_1 = require("../common/helpers");
|
|
|
47
47
|
const logger_1 = require("../logger/logger");
|
|
48
48
|
class Uploader {
|
|
49
49
|
constructor({ event, options }) {
|
|
50
|
-
this.event = event;
|
|
51
50
|
this.devrevApiEndpoint = event.execution_metadata.devrev_endpoint;
|
|
52
51
|
this.devrevApiToken = event.context.secrets.service_account_token;
|
|
53
52
|
this.requestId = event.payload.event_context.request_id;
|
|
@@ -57,13 +56,10 @@ class Uploader {
|
|
|
57
56
|
};
|
|
58
57
|
}
|
|
59
58
|
/**
|
|
60
|
-
* Uploads the fetched objects to the DevRev platform.
|
|
61
|
-
* The fetched objects are uploaded to the platform and the artifact information is returned.
|
|
62
|
-
* @param {string} filename - The name of the file to be uploaded
|
|
59
|
+
* Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform.
|
|
63
60
|
* @param {string} itemType - The type of the item to be uploaded
|
|
64
|
-
* @param {object[] | object} fetchedObjects - The
|
|
65
|
-
* @returns {Promise<UploadResponse>} - The response object containing the artifact information
|
|
66
|
-
* or error information if there was an error
|
|
61
|
+
* @param {object[] | object} fetchedObjects - The objects to be uploaded
|
|
62
|
+
* @returns {Promise<UploadResponse>} - The response object containing the artifact information or error information if there was an error
|
|
67
63
|
*/
|
|
68
64
|
async upload(itemType, fetchedObjects) {
|
|
69
65
|
if (this.isLocalDevelopment) {
|
|
@@ -107,7 +103,14 @@ class Uploader {
|
|
|
107
103
|
};
|
|
108
104
|
return { artifact };
|
|
109
105
|
}
|
|
110
|
-
|
|
106
|
+
/**
|
|
107
|
+
* Gets the upload URL for an artifact from the DevRev API.
|
|
108
|
+
* @param {string} filename - The name of the file to upload
|
|
109
|
+
* @param {string} fileType - The MIME type of the file
|
|
110
|
+
* @param {number} [fileSize] - Optional file size in bytes
|
|
111
|
+
* @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
|
|
112
|
+
*/
|
|
113
|
+
async getArtifactUploadUrl(filename, fileType, fileSize) {
|
|
111
114
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
|
112
115
|
try {
|
|
113
116
|
const response = await axios_client_internal_1.axiosClient.get(url, {
|
|
@@ -116,6 +119,7 @@ class Uploader {
|
|
|
116
119
|
request_id: this.requestId,
|
|
117
120
|
file_type: fileType,
|
|
118
121
|
file_name: (0, helpers_1.truncateFilename)(filename),
|
|
122
|
+
file_size: fileSize,
|
|
119
123
|
},
|
|
120
124
|
});
|
|
121
125
|
return response.data;
|
|
@@ -124,6 +128,12 @@ class Uploader {
|
|
|
124
128
|
console.error('Error while getting artifact upload URL.', (0, logger_1.serializeError)(error));
|
|
125
129
|
}
|
|
126
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Uploads an artifact file to the provided upload URL using multipart form data.
|
|
133
|
+
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
134
|
+
* @param {Buffer} file - The file buffer to upload
|
|
135
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
136
|
+
*/
|
|
127
137
|
async uploadArtifact(artifact, file) {
|
|
128
138
|
const formData = new form_data_1.default();
|
|
129
139
|
for (const field in artifact.form_data) {
|
|
@@ -140,6 +150,12 @@ class Uploader {
|
|
|
140
150
|
console.error('Error while uploading artifact.', (0, logger_1.serializeError)(error));
|
|
141
151
|
}
|
|
142
152
|
}
|
|
153
|
+
/**
|
|
154
|
+
* Streams an artifact file from an axios response to the upload URL.
|
|
155
|
+
* @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
|
|
156
|
+
* @param {AxiosResponse} fileStream - The axios response stream containing the file data
|
|
157
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
158
|
+
*/
|
|
143
159
|
async streamArtifact(artifact, fileStream) {
|
|
144
160
|
const formData = new form_data_1.default();
|
|
145
161
|
for (const field in artifact.form_data) {
|
|
@@ -167,6 +183,11 @@ class Uploader {
|
|
|
167
183
|
return;
|
|
168
184
|
}
|
|
169
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Confirms that an artifact upload has been completed successfully.
|
|
188
|
+
* @param {string} artifactId - The ID of the artifact to confirm
|
|
189
|
+
* @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
|
|
190
|
+
*/
|
|
170
191
|
async confirmArtifactUpload(artifactId) {
|
|
171
192
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.confirm-upload`;
|
|
172
193
|
try {
|
|
@@ -202,6 +223,12 @@ class Uploader {
|
|
|
202
223
|
console.warn('Error while destroying stream:', (0, logger_1.serializeError)(error));
|
|
203
224
|
}
|
|
204
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* Retrieves attachment metadata from an artifact by downloading and parsing it.
|
|
228
|
+
* @param {object} param0 - Configuration object
|
|
229
|
+
* @param {string} param0.artifact - The artifact ID to download attachments from
|
|
230
|
+
* @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object
|
|
231
|
+
*/
|
|
205
232
|
async getAttachmentsFromArtifactId({ artifact, }) {
|
|
206
233
|
// Get the URL of the attachments metadata artifact
|
|
207
234
|
const artifactUrl = await this.getArtifactDownloadUrl(artifact);
|
|
@@ -233,6 +260,11 @@ class Uploader {
|
|
|
233
260
|
}
|
|
234
261
|
return { attachments: jsonObject };
|
|
235
262
|
}
|
|
263
|
+
/**
|
|
264
|
+
* Gets the download URL for an artifact from the DevRev API.
|
|
265
|
+
* @param {string} artifactId - The ID of the artifact to download
|
|
266
|
+
* @returns {Promise<string | void>} The download URL or undefined on error
|
|
267
|
+
*/
|
|
236
268
|
async getArtifactDownloadUrl(artifactId) {
|
|
237
269
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.download-url`;
|
|
238
270
|
try {
|
|
@@ -249,6 +281,11 @@ class Uploader {
|
|
|
249
281
|
console.error('Error while getting artifact download URL.', (0, logger_1.serializeError)(error));
|
|
250
282
|
}
|
|
251
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* Downloads an artifact file from the given URL.
|
|
286
|
+
* @param {string} artifactUrl - The URL to download the artifact from
|
|
287
|
+
* @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
|
|
288
|
+
*/
|
|
252
289
|
async downloadArtifact(artifactUrl) {
|
|
253
290
|
try {
|
|
254
291
|
const response = await axios_client_internal_1.axiosClient.get(artifactUrl, {
|
|
@@ -260,6 +297,11 @@ class Uploader {
|
|
|
260
297
|
console.error('Error while downloading artifact from URL.', (0, logger_1.serializeError)(error));
|
|
261
298
|
}
|
|
262
299
|
}
|
|
300
|
+
/**
|
|
301
|
+
* Compresses a JSONL string using gzip compression.
|
|
302
|
+
* @param {string} jsonlObject - The JSONL string to compress
|
|
303
|
+
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
304
|
+
*/
|
|
263
305
|
compressGzip(jsonlObject) {
|
|
264
306
|
try {
|
|
265
307
|
return zlib_1.default.gzipSync(jsonlObject);
|
|
@@ -268,6 +310,11 @@ class Uploader {
|
|
|
268
310
|
console.error('Error while compressing jsonl object.', error);
|
|
269
311
|
}
|
|
270
312
|
}
|
|
313
|
+
/**
|
|
314
|
+
* Decompresses a gzipped buffer to a JSONL string.
|
|
315
|
+
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
316
|
+
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
317
|
+
*/
|
|
271
318
|
decompressGzip(gzippedJsonlObject) {
|
|
272
319
|
try {
|
|
273
320
|
const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
|
|
@@ -277,6 +324,11 @@ class Uploader {
|
|
|
277
324
|
console.error('Error while decompressing gzipped jsonl object.', error);
|
|
278
325
|
}
|
|
279
326
|
}
|
|
327
|
+
/**
|
|
328
|
+
* Parses a JSONL string into an array of objects.
|
|
329
|
+
* @param {string} jsonlObject - The JSONL string to parse
|
|
330
|
+
* @returns {object[] | null} The parsed array of objects or null on error
|
|
331
|
+
*/
|
|
280
332
|
parseJsonl(jsonlObject) {
|
|
281
333
|
try {
|
|
282
334
|
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
@@ -286,6 +338,13 @@ class Uploader {
|
|
|
286
338
|
}
|
|
287
339
|
return null;
|
|
288
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Retrieves and parses JSON objects from an artifact by artifact ID.
|
|
343
|
+
* @param {object} param0 - Configuration object
|
|
344
|
+
* @param {string} param0.artifactId - The artifact ID to download and parse
|
|
345
|
+
* @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped
|
|
346
|
+
* @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
|
|
347
|
+
*/
|
|
289
348
|
async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
|
|
290
349
|
const artifactUrl = await this.getArtifactDownloadUrl(artifactId);
|
|
291
350
|
if (!artifactUrl) {
|
|
@@ -306,6 +365,12 @@ class Uploader {
|
|
|
306
365
|
const jsonlObject = Buffer.from(artifact).toString('utf-8');
|
|
307
366
|
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
308
367
|
}
|
|
368
|
+
/**
|
|
369
|
+
* Downloads fetched objects to the local file system (for local development).
|
|
370
|
+
* @param {string} itemType - The type of items being downloaded
|
|
371
|
+
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
372
|
+
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
373
|
+
*/
|
|
309
374
|
async downloadToLocal(itemType, fetchedObjects) {
|
|
310
375
|
console.log(`Downloading ${itemType} to local file system.`);
|
|
311
376
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAoC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAmIvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
|
package/dist/workers/spawn.js
CHANGED
|
@@ -15,6 +15,7 @@ const extraction_1 = require("../types/extraction");
|
|
|
15
15
|
const workers_1 = require("../types/workers");
|
|
16
16
|
const constants_1 = require("../common/constants");
|
|
17
17
|
const create_worker_1 = require("./create-worker");
|
|
18
|
+
const types_1 = require("../types");
|
|
18
19
|
function getWorkerPath({ event, workerBasePath, }) {
|
|
19
20
|
let path = null;
|
|
20
21
|
switch (event.payload.event_type) {
|
|
@@ -111,6 +112,33 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
|
|
|
111
112
|
}
|
|
112
113
|
}
|
|
113
114
|
else {
|
|
115
|
+
// If no script is found for the delete event type, return `Done`
|
|
116
|
+
switch (event.payload.event_type) {
|
|
117
|
+
case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
|
|
118
|
+
await (0, control_protocol_1.emit)({
|
|
119
|
+
event,
|
|
120
|
+
eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionDone,
|
|
121
|
+
});
|
|
122
|
+
return;
|
|
123
|
+
case extraction_1.EventType.StartDeletingLoaderAttachmentState:
|
|
124
|
+
await (0, control_protocol_1.emit)({
|
|
125
|
+
event,
|
|
126
|
+
eventType: types_1.LoaderEventType.LoaderAttachmentStateDeletionDone,
|
|
127
|
+
});
|
|
128
|
+
return;
|
|
129
|
+
case extraction_1.EventType.StartDeletingExtractorState:
|
|
130
|
+
await (0, control_protocol_1.emit)({
|
|
131
|
+
event,
|
|
132
|
+
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
|
|
133
|
+
});
|
|
134
|
+
return;
|
|
135
|
+
case extraction_1.EventType.StartDeletingLoaderState:
|
|
136
|
+
await (0, control_protocol_1.emit)({
|
|
137
|
+
event,
|
|
138
|
+
eventType: types_1.LoaderEventType.LoaderStateDeletionDone,
|
|
139
|
+
});
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
114
142
|
console.error('Script was not found for event type: ' + event.payload.event_type + '.');
|
|
115
143
|
try {
|
|
116
144
|
await (0, control_protocol_1.emit)({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../src/workers/worker-adapter.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,QAAQ,EAAkB,MAAM,iCAAiC,CAAC;AAE3E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAiFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAgHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IAuBK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwE5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6LvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../src/workers/worker-adapter.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,QAAQ,EAAkB,MAAM,iCAAiC,CAAC;AAE3E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAiFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAgHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IAuBK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwE5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6LvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4FvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAmD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAsHzC"}
|
|
@@ -494,7 +494,7 @@ class WorkerAdapter {
|
|
|
494
494
|
}
|
|
495
495
|
}
|
|
496
496
|
async processAttachment(attachment, stream) {
|
|
497
|
-
var _a
|
|
497
|
+
var _a;
|
|
498
498
|
const { httpStream, delay, error } = await stream({
|
|
499
499
|
item: attachment,
|
|
500
500
|
event: this.event,
|
|
@@ -506,9 +506,12 @@ class WorkerAdapter {
|
|
|
506
506
|
return { delay };
|
|
507
507
|
}
|
|
508
508
|
if (httpStream) {
|
|
509
|
-
const fileType =
|
|
509
|
+
const fileType = httpStream.headers['content-type'] || 'application/octet-stream';
|
|
510
|
+
const fileSize = httpStream.headers['content-length']
|
|
511
|
+
? parseInt(httpStream.headers['content-length'])
|
|
512
|
+
: undefined;
|
|
510
513
|
// Get upload URL
|
|
511
|
-
const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType);
|
|
514
|
+
const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
512
515
|
if (!preparedArtifact) {
|
|
513
516
|
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment.`);
|
|
514
517
|
this.destroyHttpStream(httpStream);
|
|
@@ -553,7 +556,7 @@ class WorkerAdapter {
|
|
|
553
556
|
else if (attachment.inline === false) {
|
|
554
557
|
ssorAttachment.inline = false;
|
|
555
558
|
}
|
|
556
|
-
await ((
|
|
559
|
+
await ((_a = this.getRepo('ssor_attachment')) === null || _a === void 0 ? void 0 : _a.push([ssorAttachment]));
|
|
557
560
|
}
|
|
558
561
|
return;
|
|
559
562
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/ts-adaas",
|
|
3
|
-
"version": "1.13.0",
|
|
3
|
+
"version": "1.13.1-beta.0",
|
|
4
4
|
"description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -27,24 +27,18 @@
|
|
|
27
27
|
"devDependencies": {
|
|
28
28
|
"@microsoft/api-extractor": "^7.52.11",
|
|
29
29
|
"@microsoft/api-extractor-model": "^7.30.7",
|
|
30
|
+
"@types/express": "^5.0.3",
|
|
30
31
|
"@types/jest": "^29.5.14",
|
|
31
|
-
"@types/lambda-log": "^3.0.3",
|
|
32
32
|
"@types/node": "^22.18.0",
|
|
33
33
|
"@types/yargs": "^17.0.33",
|
|
34
34
|
"@typescript-eslint/eslint-plugin": "^8.46.0",
|
|
35
35
|
"@typescript-eslint/parser": "^8.46.0",
|
|
36
|
-
"axios-mock-adapter": "^2.1.0",
|
|
37
36
|
"eslint": "9.32.0",
|
|
38
37
|
"eslint-config-prettier": "^9.1.2",
|
|
39
|
-
"eslint-plugin-import": "^2.28.1",
|
|
40
38
|
"eslint-plugin-prettier": "4.0.0",
|
|
41
|
-
"eslint-plugin-simple-import-sort": "^10.0.0",
|
|
42
|
-
"eslint-plugin-sort-keys-fix": "^1.1.2",
|
|
43
|
-
"eslint-plugin-unused-imports": "^4.1.4",
|
|
44
39
|
"jest": "^29.7.0",
|
|
45
40
|
"jiti": "^2.6.1",
|
|
46
41
|
"prettier": "^2.8.3",
|
|
47
|
-
"prettier-plugin-organize-imports": "^3.2.2",
|
|
48
42
|
"ts-jest": "^29.4.4",
|
|
49
43
|
"typescript": "^5.3.3",
|
|
50
44
|
"typescript-eslint": "^8.46.1"
|
|
@@ -53,6 +47,7 @@
|
|
|
53
47
|
"@devrev/typescript-sdk": "^1.1.59",
|
|
54
48
|
"axios": "^1.12.1",
|
|
55
49
|
"axios-retry": "^4.5.0",
|
|
50
|
+
"express": "^5.2.1",
|
|
56
51
|
"form-data": "^4.0.4",
|
|
57
52
|
"js-jsonl": "^1.1.1",
|
|
58
53
|
"ts-node": "^10.9.2",
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
export declare class MockServer {
|
|
2
|
-
private server;
|
|
3
|
-
private port;
|
|
4
|
-
private consoleLogs;
|
|
5
|
-
private requests;
|
|
6
|
-
constructor(port?: number);
|
|
7
|
-
private getDefaultWorkerState;
|
|
8
|
-
private handleRequest;
|
|
9
|
-
start(): Promise<void>;
|
|
10
|
-
stop(): Promise<void>;
|
|
11
|
-
getBaseUrl(): string;
|
|
12
|
-
getPort(): number;
|
|
13
|
-
getConsoleLogs(): string[];
|
|
14
|
-
getRequests(): {
|
|
15
|
-
method: string;
|
|
16
|
-
url: string;
|
|
17
|
-
body?: any;
|
|
18
|
-
}[];
|
|
19
|
-
clearRequests(): void;
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=mock-server.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/tests/mock-server.ts"],"names":[],"mappings":"AAGA,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,WAAW,CAAgB;IAEnC,OAAO,CAAC,QAAQ,CAA0D;gBAE9D,IAAI,CAAC,EAAE,MAAM;IAIzB,OAAO,CAAC,qBAAqB;IAe7B,OAAO,CAAC,aAAa;IA+Nf,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,UAAU,IAAI,MAAM;IAIpB,OAAO,IAAI,MAAM;IAIjB,cAAc,IAAI,MAAM,EAAE;IAI1B,WAAW;gBA/RuB,MAAM;aAAO,MAAM;eAAS,GAAG;;IAmSjE,aAAa;CAGd"}
|