@devrev/ts-adaas 1.8.1-beta.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +33 -40
- package/dist/common/control-protocol.js +2 -2
- package/dist/common/helpers.test.js +29 -29
- package/dist/common/install-initial-domain-mapping.js +4 -4
- package/dist/common/install-initial-domain-mapping.test.js +17 -27
- package/dist/deprecated/uploader/index.js +2 -2
- package/dist/http/axios-client-internal.d.ts +2 -0
- package/dist/http/axios-client-internal.js +60 -0
- package/dist/http/axios-client-internal.test.js +189 -0
- package/dist/http/axios-client.d.ts +0 -1
- package/dist/http/axios-client.js +5 -15
- package/dist/logger/logger.test.js +4 -4
- package/dist/mappers/mappers.js +4 -4
- package/dist/repo/repo.test.js +51 -37
- package/dist/state/state.js +8 -4
- package/dist/tests/timeout-handling/timeout-1.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-2.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3a.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3b.test.js +1 -1
- package/dist/types/extraction.test.js +51 -5
- package/dist/uploader/uploader.js +7 -7
- package/dist/uploader/uploader.test.js +18 -21
- package/dist/workers/create-worker.test.js +64 -2
- package/dist/workers/default-workers/attachments-extraction.js +7 -3
- package/dist/workers/worker-adapter.js +5 -2
- package/dist/workers/worker-adapter.test.js +84 -77
- package/package.json +3 -2
- package/dist/http/axios-client.test.js +0 -129
- /package/dist/http/{axios-client.test.d.ts → axios-client-internal.test.d.ts} +0 -0
|
@@ -9,7 +9,6 @@
|
|
|
9
9
|
* 1. Network errors (where no response is received).
|
|
10
10
|
* 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT).
|
|
11
11
|
* 3. All 5xx server errors.
|
|
12
|
-
* 4. 429 Too Many Requests (respects Retry-After header when present).
|
|
13
12
|
*
|
|
14
13
|
* Retry Strategy:
|
|
15
14
|
* - A maximum of 5 retries are attempted.
|
|
@@ -36,25 +35,16 @@ exports.axiosClient = axiosClient;
|
|
|
36
35
|
(0, axios_retry_1.default)(axiosClient, {
|
|
37
36
|
retries: 5,
|
|
38
37
|
retryDelay: (retryCount, error) => {
|
|
39
|
-
var _a, _b, _c
|
|
40
|
-
//
|
|
41
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
42
|
-
const retryAfter = (_b = error.response.headers) === null || _b === void 0 ? void 0 : _b['retry-after'];
|
|
43
|
-
if (retryAfter) {
|
|
44
|
-
const delay = parseInt(retryAfter, 10) * 1000;
|
|
45
|
-
console.warn(`Request to ${(_c = error.config) === null || _c === void 0 ? void 0 : _c.url} failed with 429 Too Many Requests. Method ${(_d = error.config) === null || _d === void 0 ? void 0 : _d.method}. Retry count: ${retryCount}. Retrying after ${Math.round(delay / 1000)}s as specified by Retry-After header.`);
|
|
46
|
-
return delay;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
// Default exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
|
|
38
|
+
var _a, _b, _c;
|
|
39
|
+
// exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
|
|
50
40
|
const delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
51
|
-
console.warn(`Request to ${(
|
|
41
|
+
console.warn(`Request to ${(_a = error.config) === null || _a === void 0 ? void 0 : _a.url} failed with response status code ${(_b = error.response) === null || _b === void 0 ? void 0 : _b.status}. Method ${(_c = error.config) === null || _c === void 0 ? void 0 : _c.method}. Retry count: ${retryCount}. Retrying in ${Math.round(delay / 1000)}s.`);
|
|
52
42
|
return delay;
|
|
53
43
|
},
|
|
54
44
|
retryCondition: (error) => {
|
|
55
45
|
var _a, _b, _c;
|
|
56
|
-
return (axios_retry_1.default.isNetworkOrIdempotentRequestError(error)
|
|
57
|
-
((_a = error.response) === null || _a === void 0 ? void 0 : _a.status)
|
|
46
|
+
return ((axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
47
|
+
((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== 429) ||
|
|
58
48
|
((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.status) !== null && _c !== void 0 ? _c : 0) >= 500);
|
|
59
49
|
},
|
|
60
50
|
onMaxRetryTimesExceeded(error) {
|
|
@@ -14,7 +14,7 @@ jest.mock('node:worker_threads', () => ({
|
|
|
14
14
|
isMainThread: true,
|
|
15
15
|
parentPort: null,
|
|
16
16
|
}));
|
|
17
|
-
describe(
|
|
17
|
+
describe(logger_1.Logger.name, () => {
|
|
18
18
|
let mockEvent;
|
|
19
19
|
let mockOptions;
|
|
20
20
|
beforeEach(() => {
|
|
@@ -137,7 +137,7 @@ describe('Logger', () => {
|
|
|
137
137
|
mockOptions.isLocalDevelopment = false;
|
|
138
138
|
logger = new logger_1.Logger({ event: mockEvent, options: mockOptions });
|
|
139
139
|
});
|
|
140
|
-
it('should handle empty string message', () => {
|
|
140
|
+
it('[edge] should handle empty string message', () => {
|
|
141
141
|
logger.info('');
|
|
142
142
|
expect(mockConsoleInfo).toHaveBeenCalledTimes(1);
|
|
143
143
|
const callArgs = mockConsoleInfo.mock.calls[0][0];
|
|
@@ -146,7 +146,7 @@ describe('Logger', () => {
|
|
|
146
146
|
expect(logObject.dev_oid).toBe(mockEvent.payload.event_context.dev_oid);
|
|
147
147
|
expect(logObject.request_id).toBe(mockEvent.payload.event_context.request_id);
|
|
148
148
|
});
|
|
149
|
-
it('should handle null and undefined values', () => {
|
|
149
|
+
it('[edge] should handle null and undefined values', () => {
|
|
150
150
|
logger.info('test', null, undefined);
|
|
151
151
|
expect(mockConsoleInfo).toHaveBeenCalledTimes(1);
|
|
152
152
|
const callArgs = mockConsoleInfo.mock.calls[0][0];
|
|
@@ -155,7 +155,7 @@ describe('Logger', () => {
|
|
|
155
155
|
expect(logObject.message).toBe('test null undefined');
|
|
156
156
|
expect(logObject.dev_oid).toBe(mockEvent.payload.event_context.dev_oid);
|
|
157
157
|
});
|
|
158
|
-
it('should handle complex nested objects', () => {
|
|
158
|
+
it('[edge] should handle complex nested objects', () => {
|
|
159
159
|
const complexObject = {
|
|
160
160
|
level1: {
|
|
161
161
|
level2: {
|
package/dist/mappers/mappers.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Mappers = void 0;
|
|
4
|
-
const
|
|
4
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
5
5
|
class Mappers {
|
|
6
6
|
constructor({ event }) {
|
|
7
7
|
this.devrevApiEndpoint = event.execution_metadata.devrev_endpoint;
|
|
@@ -9,7 +9,7 @@ class Mappers {
|
|
|
9
9
|
}
|
|
10
10
|
async getByTargetId(params) {
|
|
11
11
|
const { sync_unit, target } = params;
|
|
12
|
-
return
|
|
12
|
+
return axios_client_internal_1.axiosClient.get(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-target`, {
|
|
13
13
|
headers: {
|
|
14
14
|
Authorization: this.devrevApiToken,
|
|
15
15
|
},
|
|
@@ -17,14 +17,14 @@ class Mappers {
|
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
19
|
async create(params) {
|
|
20
|
-
return
|
|
20
|
+
return axios_client_internal_1.axiosClient.post(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.create`, params, {
|
|
21
21
|
headers: {
|
|
22
22
|
Authorization: this.devrevApiToken,
|
|
23
23
|
},
|
|
24
24
|
});
|
|
25
25
|
}
|
|
26
26
|
async update(params) {
|
|
27
|
-
return
|
|
27
|
+
return axios_client_internal_1.axiosClient.post(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.update`, params, {
|
|
28
28
|
headers: {
|
|
29
29
|
Authorization: this.devrevApiToken,
|
|
30
30
|
},
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -5,7 +5,7 @@ const test_helpers_1 = require("../tests/test-helpers");
|
|
|
5
5
|
const types_1 = require("../types");
|
|
6
6
|
const repo_1 = require("./repo");
|
|
7
7
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
8
|
-
describe(
|
|
8
|
+
describe(repo_1.Repo.name, () => {
|
|
9
9
|
let repo;
|
|
10
10
|
let normalize;
|
|
11
11
|
beforeEach(() => {
|
|
@@ -20,18 +20,14 @@ describe('Repo class push method', () => {
|
|
|
20
20
|
afterEach(() => {
|
|
21
21
|
jest.clearAllMocks();
|
|
22
22
|
});
|
|
23
|
-
it('should
|
|
24
|
-
await repo.push([]);
|
|
25
|
-
expect(repo.getItems()).toEqual([]);
|
|
26
|
-
});
|
|
27
|
-
it('should normalize and push 10 items if array is not empty', async () => {
|
|
23
|
+
it('should normalize and push items when array contains items', async () => {
|
|
28
24
|
const items = (0, test_helpers_1.createItems)(10);
|
|
29
25
|
await repo.push(items);
|
|
30
26
|
expect(normalize).toHaveBeenCalledTimes(10);
|
|
31
27
|
const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
|
|
32
28
|
expect(repo.getItems()).toEqual(normalizedItems);
|
|
33
29
|
});
|
|
34
|
-
it('should not normalize items
|
|
30
|
+
it('should not normalize items when normalize function is not provided', async () => {
|
|
35
31
|
repo = new repo_1.Repo({
|
|
36
32
|
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
37
33
|
itemType: 'test_item_type',
|
|
@@ -42,46 +38,56 @@ describe('Repo class push method', () => {
|
|
|
42
38
|
await repo.push(items);
|
|
43
39
|
expect(normalize).not.toHaveBeenCalled();
|
|
44
40
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
expect(normalize).not.toHaveBeenCalled();
|
|
41
|
+
it('[edge] should not push items when items array is empty', async () => {
|
|
42
|
+
await repo.push([]);
|
|
43
|
+
expect(repo.getItems()).toEqual([]);
|
|
44
|
+
});
|
|
45
|
+
it('should not normalize items when item type is external_domain_metadata', async () => {
|
|
46
|
+
repo = new repo_1.Repo({
|
|
47
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
48
|
+
itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA,
|
|
49
|
+
normalize,
|
|
50
|
+
onUpload: jest.fn(),
|
|
51
|
+
options: {},
|
|
57
52
|
});
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
53
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
54
|
+
await repo.push(items);
|
|
55
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
it('should not normalize items when item type is ssor_attachment', async () => {
|
|
58
|
+
repo = new repo_1.Repo({
|
|
59
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
60
|
+
itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT,
|
|
61
|
+
normalize,
|
|
62
|
+
onUpload: jest.fn(),
|
|
63
|
+
options: {},
|
|
69
64
|
});
|
|
65
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
66
|
+
await repo.push(items);
|
|
67
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
70
68
|
});
|
|
71
69
|
it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
|
|
72
70
|
const items = (0, test_helpers_1.createItems)(2005);
|
|
73
71
|
await repo.push(items);
|
|
74
72
|
expect(repo.getItems().length).toBe(5);
|
|
75
73
|
});
|
|
76
|
-
it('should
|
|
77
|
-
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
74
|
+
it('should normalize all items when pushing 4005 items with batch size of 2000', async () => {
|
|
78
75
|
const items = (0, test_helpers_1.createItems)(4005);
|
|
79
76
|
await repo.push(items);
|
|
80
77
|
expect(normalize).toHaveBeenCalledTimes(4005);
|
|
81
|
-
|
|
82
|
-
|
|
78
|
+
});
|
|
79
|
+
it('should upload 2 batches when pushing 4005 items with batch size of 2000', async () => {
|
|
80
|
+
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
81
|
+
const items = (0, test_helpers_1.createItems)(4005);
|
|
82
|
+
await repo.push(items);
|
|
83
|
+
expect(uploadSpy).toHaveBeenCalledTimes(2);
|
|
83
84
|
uploadSpy.mockRestore();
|
|
84
85
|
});
|
|
86
|
+
it('should leave 5 items in array after pushing 4005 items with batch size of 2000', async () => {
|
|
87
|
+
const items = (0, test_helpers_1.createItems)(4005);
|
|
88
|
+
await repo.push(items);
|
|
89
|
+
expect(repo.getItems().length).toBe(5);
|
|
90
|
+
});
|
|
85
91
|
describe('should take batch size into account', () => {
|
|
86
92
|
beforeEach(() => {
|
|
87
93
|
repo = new repo_1.Repo({
|
|
@@ -104,14 +110,22 @@ describe('Repo class push method', () => {
|
|
|
104
110
|
await repo.push(items);
|
|
105
111
|
expect(repo.getItems().length).toBe(5);
|
|
106
112
|
});
|
|
107
|
-
it('should
|
|
108
|
-
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
113
|
+
it('should normalize all items when pushing 205 items with batch size of 50', async () => {
|
|
109
114
|
const items = (0, test_helpers_1.createItems)(205);
|
|
110
115
|
await repo.push(items);
|
|
111
116
|
expect(normalize).toHaveBeenCalledTimes(205);
|
|
112
|
-
|
|
117
|
+
});
|
|
118
|
+
it('should upload 4 batches when pushing 205 items with batch size of 50', async () => {
|
|
119
|
+
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
120
|
+
const items = (0, test_helpers_1.createItems)(205);
|
|
121
|
+
await repo.push(items);
|
|
113
122
|
expect(uploadSpy).toHaveBeenCalledTimes(4);
|
|
114
123
|
uploadSpy.mockRestore();
|
|
115
124
|
});
|
|
125
|
+
it('should leave 5 items in array after pushing 205 items with batch size of 50', async () => {
|
|
126
|
+
const items = (0, test_helpers_1.createItems)(205);
|
|
127
|
+
await repo.push(items);
|
|
128
|
+
expect(repo.getItems().length).toBe(5);
|
|
129
|
+
});
|
|
116
130
|
});
|
|
117
131
|
});
|
package/dist/state/state.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
exports.State = void 0;
|
|
4
7
|
exports.createAdapterState = createAdapterState;
|
|
5
|
-
const
|
|
8
|
+
const axios_1 = __importDefault(require("axios"));
|
|
9
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
6
10
|
const extraction_1 = require("../types/extraction");
|
|
7
11
|
const constants_1 = require("../common/constants");
|
|
8
12
|
const logger_1 = require("../logger/logger");
|
|
@@ -88,7 +92,7 @@ class State {
|
|
|
88
92
|
*/
|
|
89
93
|
async postState(state) {
|
|
90
94
|
try {
|
|
91
|
-
await
|
|
95
|
+
await axios_client_internal_1.axiosClient.post(this.workerUrl + '.update', {
|
|
92
96
|
state: JSON.stringify(state || this.state),
|
|
93
97
|
}, {
|
|
94
98
|
headers: {
|
|
@@ -118,7 +122,7 @@ class State {
|
|
|
118
122
|
this.event.payload.event_context.sync_unit_id +
|
|
119
123
|
'.');
|
|
120
124
|
try {
|
|
121
|
-
const response = await
|
|
125
|
+
const response = await axios_client_internal_1.axiosClient.get(this.workerUrl + '.get', {
|
|
122
126
|
headers: {
|
|
123
127
|
Authorization: this.devrevToken,
|
|
124
128
|
},
|
|
@@ -132,7 +136,7 @@ class State {
|
|
|
132
136
|
return this.state;
|
|
133
137
|
}
|
|
134
138
|
catch (error) {
|
|
135
|
-
if (
|
|
139
|
+
if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
136
140
|
const state = Object.assign(Object.assign({}, initialState), this.initialSdkState);
|
|
137
141
|
this.state = state;
|
|
138
142
|
console.log('State not found, returning initial state. Current state', (0, logger_1.getPrintableState)(this.state));
|
|
@@ -7,7 +7,7 @@ const extraction_1 = require("../../types/extraction");
|
|
|
7
7
|
const test_helpers_1 = require("../test-helpers");
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
|
-
describe('timeout-1', () => {
|
|
10
|
+
describe('timeout-1 extraction', () => {
|
|
11
11
|
let mockServer;
|
|
12
12
|
beforeAll(async () => {
|
|
13
13
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-2', () => {
|
|
11
|
+
describe('timeout-2 extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-3a', () => {
|
|
11
|
+
describe('timeout-3a extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-3b', () => {
|
|
11
|
+
describe('timeout-3b extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -2,22 +2,68 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const extraction_1 = require("./extraction");
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
-
|
|
5
|
+
// Test the EventContext interface and related extraction types
|
|
6
|
+
describe('ExtractionTypes', () => {
|
|
6
7
|
const baseEvent = (0, test_helpers_1.createEvent)({ eventType: extraction_1.EventType.ExtractionDataStart });
|
|
7
|
-
it('should
|
|
8
|
+
it('should create event context without optional fields', () => {
|
|
8
9
|
const event = Object.assign({}, baseEvent);
|
|
9
10
|
// If this compiles, the test passes
|
|
10
11
|
expect(event).toBeDefined();
|
|
12
|
+
expect(event.payload.event_context).toBeDefined();
|
|
11
13
|
});
|
|
12
|
-
it('should
|
|
14
|
+
it('should create event context with all optional fields', () => {
|
|
13
15
|
const event = Object.assign({}, baseEvent);
|
|
14
16
|
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: '2024-01-01T00:00:00Z', initial_sync_scope: extraction_1.InitialSyncScope.TIME_SCOPED, reset_extract_from: true });
|
|
15
|
-
// Test with all optionals present
|
|
16
17
|
expect(event).toBeDefined();
|
|
18
|
+
expect(event.payload.event_context.extract_from).toBe('2024-01-01T00:00:00Z');
|
|
19
|
+
expect(event.payload.event_context.initial_sync_scope).toBe(extraction_1.InitialSyncScope.TIME_SCOPED);
|
|
20
|
+
expect(event.payload.event_context.reset_extract_from).toBe(true);
|
|
17
21
|
});
|
|
18
|
-
it('should
|
|
22
|
+
it('should create event context with partial optional fields', () => {
|
|
19
23
|
const event = Object.assign({}, baseEvent);
|
|
20
24
|
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: '2024-01-01T00:00:00Z' });
|
|
21
25
|
expect(event).toBeDefined();
|
|
26
|
+
expect(event.payload.event_context.extract_from).toBe('2024-01-01T00:00:00Z');
|
|
27
|
+
});
|
|
28
|
+
it('should handle different InitialSyncScope values', () => {
|
|
29
|
+
const event = Object.assign({}, baseEvent);
|
|
30
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { initial_sync_scope: extraction_1.InitialSyncScope.FULL_HISTORY });
|
|
31
|
+
expect(event.payload.event_context.initial_sync_scope).toBe(extraction_1.InitialSyncScope.FULL_HISTORY);
|
|
32
|
+
});
|
|
33
|
+
it('[edge] should handle null event context gracefully', () => {
|
|
34
|
+
const event = Object.assign({}, baseEvent);
|
|
35
|
+
event.payload.event_context = null;
|
|
36
|
+
expect(event.payload.event_context).toBeNull();
|
|
37
|
+
});
|
|
38
|
+
it('[edge] should handle undefined optional fields', () => {
|
|
39
|
+
const event = Object.assign({}, baseEvent);
|
|
40
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: undefined, initial_sync_scope: undefined, reset_extract_from: undefined });
|
|
41
|
+
expect(event.payload.event_context.extract_from).toBeUndefined();
|
|
42
|
+
expect(event.payload.event_context.initial_sync_scope).toBeUndefined();
|
|
43
|
+
expect(event.payload.event_context.reset_extract_from).toBeUndefined();
|
|
44
|
+
});
|
|
45
|
+
it('[edge] should handle invalid date format in extract_from', () => {
|
|
46
|
+
const event = Object.assign({}, baseEvent);
|
|
47
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: 'invalid-date-format' });
|
|
48
|
+
expect(event.payload.event_context.extract_from).toBe('invalid-date-format');
|
|
49
|
+
// Note: Type validation would typically happen at runtime, not compile time
|
|
50
|
+
});
|
|
51
|
+
it('[edge] should handle explicit boolean values for reset_extract_from', () => {
|
|
52
|
+
const eventWithTrue = (0, test_helpers_1.createEvent)({
|
|
53
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
54
|
+
eventContextOverrides: {
|
|
55
|
+
reset_extract_from: true
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
const eventWithFalse = (0, test_helpers_1.createEvent)({
|
|
59
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
60
|
+
eventContextOverrides: {
|
|
61
|
+
reset_extract_from: false
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
expect(eventWithTrue.payload.event_context.reset_extract_from).toBe(true);
|
|
65
|
+
expect(eventWithFalse.payload.event_context.reset_extract_from).toBe(false);
|
|
66
|
+
expect(typeof eventWithTrue.payload.event_context.reset_extract_from).toBe('boolean');
|
|
67
|
+
expect(typeof eventWithFalse.payload.event_context.reset_extract_from).toBe('boolean');
|
|
22
68
|
});
|
|
23
69
|
});
|
|
@@ -38,7 +38,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.Uploader = void 0;
|
|
40
40
|
const fs_1 = __importStar(require("fs"));
|
|
41
|
-
const
|
|
41
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
42
42
|
const zlib_1 = __importDefault(require("zlib"));
|
|
43
43
|
const js_jsonl_1 = require("js-jsonl");
|
|
44
44
|
const form_data_1 = __importDefault(require("form-data"));
|
|
@@ -110,7 +110,7 @@ class Uploader {
|
|
|
110
110
|
async getArtifactUploadUrl(filename, fileType) {
|
|
111
111
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
|
|
112
112
|
try {
|
|
113
|
-
const response = await
|
|
113
|
+
const response = await axios_client_internal_1.axiosClient.get(url, {
|
|
114
114
|
headers: Object.assign({}, this.defaultHeaders),
|
|
115
115
|
params: {
|
|
116
116
|
request_id: this.requestId,
|
|
@@ -131,7 +131,7 @@ class Uploader {
|
|
|
131
131
|
}
|
|
132
132
|
formData.append('file', file);
|
|
133
133
|
try {
|
|
134
|
-
const response = await
|
|
134
|
+
const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
|
|
135
135
|
headers: Object.assign({}, formData.getHeaders()),
|
|
136
136
|
});
|
|
137
137
|
return response;
|
|
@@ -151,7 +151,7 @@ class Uploader {
|
|
|
151
151
|
return;
|
|
152
152
|
}
|
|
153
153
|
try {
|
|
154
|
-
const response = await
|
|
154
|
+
const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
|
|
155
155
|
headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStream.headers['content-length']
|
|
156
156
|
? {
|
|
157
157
|
'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
|
|
@@ -170,7 +170,7 @@ class Uploader {
|
|
|
170
170
|
async confirmArtifactUpload(artifactId) {
|
|
171
171
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.confirm-upload`;
|
|
172
172
|
try {
|
|
173
|
-
const response = await
|
|
173
|
+
const response = await axios_client_internal_1.axiosClient.post(url, {
|
|
174
174
|
request_id: this.requestId,
|
|
175
175
|
artifact_id: artifactId,
|
|
176
176
|
}, {
|
|
@@ -216,7 +216,7 @@ class Uploader {
|
|
|
216
216
|
async getArtifactDownloadUrl(artifactId) {
|
|
217
217
|
const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.download-url`;
|
|
218
218
|
try {
|
|
219
|
-
const response = await
|
|
219
|
+
const response = await axios_client_internal_1.axiosClient.get(url, {
|
|
220
220
|
headers: Object.assign({}, this.defaultHeaders),
|
|
221
221
|
params: {
|
|
222
222
|
request_id: this.requestId,
|
|
@@ -231,7 +231,7 @@ class Uploader {
|
|
|
231
231
|
}
|
|
232
232
|
async downloadArtifact(artifactUrl) {
|
|
233
233
|
try {
|
|
234
|
-
const response = await
|
|
234
|
+
const response = await axios_client_internal_1.axiosClient.get(artifactUrl, {
|
|
235
235
|
responseType: 'arraybuffer',
|
|
236
236
|
});
|
|
237
237
|
return response.data;
|
|
@@ -3,9 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
4
4
|
const types_1 = require("../types");
|
|
5
5
|
const uploader_1 = require("./uploader");
|
|
6
|
-
const
|
|
7
|
-
jest.mock('../http/axios-client', () => {
|
|
8
|
-
const originalModule = jest.requireActual('../http/axios-client');
|
|
6
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
7
|
+
jest.mock('../http/axios-client-internal', () => {
|
|
8
|
+
const originalModule = jest.requireActual('../http/axios-client-internal');
|
|
9
9
|
return Object.assign(Object.assign({}, originalModule), { axiosClient: {
|
|
10
10
|
get: jest.fn(),
|
|
11
11
|
post: jest.fn(),
|
|
@@ -27,23 +27,20 @@ const getArtifactUploadUrlMockResponse = {
|
|
|
27
27
|
form_data: [],
|
|
28
28
|
},
|
|
29
29
|
};
|
|
30
|
-
describe(
|
|
30
|
+
describe(uploader_1.Uploader.name, () => {
|
|
31
31
|
const mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
|
|
32
32
|
let uploader;
|
|
33
|
-
let consoleWarnSpy;
|
|
34
33
|
beforeEach(() => {
|
|
35
34
|
uploader = new uploader_1.Uploader({ event: mockEvent });
|
|
36
|
-
consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
37
35
|
});
|
|
38
36
|
afterEach(() => {
|
|
39
37
|
jest.clearAllMocks();
|
|
40
|
-
consoleWarnSpy.mockRestore();
|
|
41
38
|
});
|
|
42
39
|
it('should upload the file to the DevRev platform and return the artifact information', async () => {
|
|
43
40
|
// Mock successful response from getArtifactUploadUrl
|
|
44
|
-
|
|
41
|
+
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(getArtifactUploadUrlMockResponse);
|
|
45
42
|
// Mock successful response from confirmArtifactUpload and uploadArtifact
|
|
46
|
-
|
|
43
|
+
axios_client_internal_1.axiosClient.post.mockResolvedValue(getSuccessResponse());
|
|
47
44
|
const entity = 'entity';
|
|
48
45
|
const fetchedObjects = [{ key: 'value' }];
|
|
49
46
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
@@ -55,40 +52,40 @@ describe('Uploader Class Tests', () => {
|
|
|
55
52
|
},
|
|
56
53
|
});
|
|
57
54
|
});
|
|
58
|
-
it('should handle failure
|
|
55
|
+
it('[edge] should handle failure when getting artifact upload URL', async () => {
|
|
59
56
|
var _a;
|
|
60
57
|
// Mock unsuccessful response for getArtifactUploadUrl
|
|
61
|
-
|
|
58
|
+
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(undefined);
|
|
62
59
|
const entity = 'entity';
|
|
63
60
|
const fetchedObjects = [{ key: 'value' }];
|
|
64
61
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
65
62
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
66
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
63
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
67
64
|
});
|
|
68
|
-
it('should handle failure
|
|
65
|
+
it('[edge] should handle failure when uploading artifact', async () => {
|
|
69
66
|
var _a;
|
|
70
67
|
// Mock successful response for getArtifactUploadUrl
|
|
71
|
-
|
|
68
|
+
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(getArtifactUploadUrlMockResponse);
|
|
72
69
|
// Mock unsuccessful response for uploadArtifact
|
|
73
|
-
|
|
70
|
+
axios_client_internal_1.axiosClient.post.mockResolvedValueOnce(undefined);
|
|
74
71
|
const entity = 'entity';
|
|
75
72
|
const fetchedObjects = [{ key: 'value' }];
|
|
76
73
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
77
74
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
78
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
75
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
79
76
|
});
|
|
80
|
-
it('should handle failure
|
|
77
|
+
it('[edge] should handle failure when confirming artifact upload', async () => {
|
|
81
78
|
var _a;
|
|
82
79
|
// Mock successful response for getArtifactUploadUrl
|
|
83
|
-
|
|
80
|
+
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(getArtifactUploadUrlMockResponse);
|
|
84
81
|
// Mock successful response from uploadArtifact
|
|
85
|
-
|
|
82
|
+
axios_client_internal_1.axiosClient.post.mockResolvedValueOnce(getSuccessResponse());
|
|
86
83
|
// Mock unsuccessful response from confirmArtifactUpload
|
|
87
|
-
|
|
84
|
+
axios_client_internal_1.axiosClient.post.mockResolvedValueOnce(undefined);
|
|
88
85
|
const entity = 'entity';
|
|
89
86
|
const fetchedObjects = [{ key: 'value' }];
|
|
90
87
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
91
88
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
92
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
89
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
93
90
|
});
|
|
94
91
|
});
|
|
@@ -4,8 +4,8 @@ const worker_threads_1 = require("worker_threads");
|
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
5
|
const extraction_1 = require("../types/extraction");
|
|
6
6
|
const create_worker_1 = require("./create-worker");
|
|
7
|
-
describe(
|
|
8
|
-
it('should
|
|
7
|
+
describe(create_worker_1.createWorker.name, () => {
|
|
8
|
+
it('should create a Worker instance when valid parameters are provided', async () => {
|
|
9
9
|
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
10
10
|
const worker = worker_threads_1.isMainThread
|
|
11
11
|
? await (0, create_worker_1.createWorker)({
|
|
@@ -22,4 +22,66 @@ describe('createWorker function', () => {
|
|
|
22
22
|
await worker.terminate();
|
|
23
23
|
}
|
|
24
24
|
});
|
|
25
|
+
it('should throw error when not in main thread', async () => {
|
|
26
|
+
const originalIsMainThread = worker_threads_1.isMainThread;
|
|
27
|
+
worker_threads_1.isMainThread = false;
|
|
28
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
29
|
+
await expect((0, create_worker_1.createWorker)({
|
|
30
|
+
event: (0, test_helpers_1.createEvent)({
|
|
31
|
+
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
32
|
+
}),
|
|
33
|
+
initialState: {},
|
|
34
|
+
workerPath,
|
|
35
|
+
})).rejects.toThrow('Worker threads can not start more worker threads.');
|
|
36
|
+
// Restore original value
|
|
37
|
+
worker_threads_1.isMainThread = originalIsMainThread;
|
|
38
|
+
});
|
|
39
|
+
it('[edge] should handle worker creation with minimal valid data', async () => {
|
|
40
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
41
|
+
if (worker_threads_1.isMainThread) {
|
|
42
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
43
|
+
event: (0, test_helpers_1.createEvent)({
|
|
44
|
+
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
45
|
+
}),
|
|
46
|
+
initialState: {},
|
|
47
|
+
workerPath,
|
|
48
|
+
});
|
|
49
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
50
|
+
await worker.terminate();
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
it('[edge] should handle worker creation with complex initial state', async () => {
|
|
54
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
55
|
+
const complexState = {
|
|
56
|
+
nested: {
|
|
57
|
+
data: [1, 2, 3],
|
|
58
|
+
config: { enabled: true }
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
if (worker_threads_1.isMainThread) {
|
|
62
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
63
|
+
event: (0, test_helpers_1.createEvent)({
|
|
64
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
65
|
+
}),
|
|
66
|
+
initialState: complexState,
|
|
67
|
+
workerPath,
|
|
68
|
+
});
|
|
69
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
70
|
+
await worker.terminate();
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
it('[edge] should handle different event types', async () => {
|
|
74
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
75
|
+
if (worker_threads_1.isMainThread) {
|
|
76
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
77
|
+
event: (0, test_helpers_1.createEvent)({
|
|
78
|
+
eventType: extraction_1.EventType.ExtractionMetadataStart,
|
|
79
|
+
}),
|
|
80
|
+
initialState: {},
|
|
81
|
+
workerPath,
|
|
82
|
+
});
|
|
83
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
84
|
+
await worker.terminate();
|
|
85
|
+
}
|
|
86
|
+
});
|
|
25
87
|
});
|