@devrev/ts-adaas 1.9.0-beta.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +33 -40
- package/dist/common/constants.d.ts +6 -1
- package/dist/common/constants.js +20 -9
- package/dist/common/control-protocol.js +2 -2
- package/dist/common/helpers.test.js +29 -29
- package/dist/common/install-initial-domain-mapping.js +4 -4
- package/dist/common/install-initial-domain-mapping.test.js +17 -27
- package/dist/deprecated/uploader/index.js +2 -2
- package/dist/http/axios-client-internal.d.ts +2 -0
- package/dist/http/axios-client-internal.js +60 -0
- package/dist/http/axios-client-internal.test.d.ts +1 -0
- package/dist/http/axios-client-internal.test.js +189 -0
- package/dist/logger/logger.js +1 -1
- package/dist/logger/logger.test.js +12 -12
- package/dist/mappers/mappers.js +4 -4
- package/dist/repo/repo.test.js +51 -37
- package/dist/state/state.d.ts +12 -8
- package/dist/state/state.interfaces.d.ts +18 -0
- package/dist/state/state.interfaces.js +19 -0
- package/dist/state/state.js +85 -67
- package/dist/state/state.test.d.ts +1 -0
- package/dist/state/state.test.js +223 -0
- package/dist/tests/test-helpers.js +8 -0
- package/dist/tests/timeout-handling/timeout-1.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-2.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3a.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3b.test.js +1 -1
- package/dist/types/common.d.ts +11 -0
- package/dist/types/common.js +13 -1
- package/dist/types/extraction.d.ts +32 -9
- package/dist/types/extraction.js +1 -11
- package/dist/types/extraction.test.js +51 -5
- package/dist/types/index.d.ts +2 -2
- package/dist/types/index.js +2 -2
- package/dist/uploader/uploader.js +7 -7
- package/dist/uploader/uploader.test.js +18 -21
- package/dist/workers/create-worker.test.js +64 -2
- package/dist/workers/default-workers/attachments-extraction.js +7 -3
- package/dist/workers/worker-adapter.js +10 -3
- package/dist/workers/worker-adapter.test.js +140 -71
- package/package.json +3 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
|
|
4
|
-
describe(
|
|
4
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
5
5
|
let mockAdapter;
|
|
6
6
|
let mockStream;
|
|
7
7
|
let mockAttachments;
|
|
@@ -50,7 +50,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
50
50
|
jest.clearAllMocks();
|
|
51
51
|
jest.restoreAllMocks();
|
|
52
52
|
});
|
|
53
|
-
describe(
|
|
53
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.constructor.name, () => {
|
|
54
54
|
it('should initialize with default values', () => {
|
|
55
55
|
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
56
56
|
adapter: mockAdapter,
|
|
@@ -82,7 +82,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
82
82
|
expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
|
|
83
83
|
});
|
|
84
84
|
});
|
|
85
|
-
describe(
|
|
85
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.streamAll.name, () => {
|
|
86
86
|
it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
|
|
87
87
|
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined;
|
|
88
88
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
@@ -116,7 +116,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
116
116
|
const result = await pool.streamAll();
|
|
117
117
|
expect(result).toEqual({});
|
|
118
118
|
expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
|
|
119
|
-
expect(console.log).toHaveBeenCalledWith('Starting download of 0 attachments, streaming 10 at once.');
|
|
120
119
|
});
|
|
121
120
|
it('should return delay when rate limit is hit', async () => {
|
|
122
121
|
const delayResponse = { delay: 5000 };
|
|
@@ -130,7 +129,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
130
129
|
expect(result).toEqual({ delay: 5000 });
|
|
131
130
|
});
|
|
132
131
|
});
|
|
133
|
-
describe(
|
|
132
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
|
|
134
133
|
it('should skip already processed attachments', async () => {
|
|
135
134
|
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = ['attachment-1'];
|
|
136
135
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
@@ -140,7 +139,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
140
139
|
stream: mockStream
|
|
141
140
|
});
|
|
142
141
|
await pool.streamAll();
|
|
143
|
-
expect(console.log).toHaveBeenCalledWith('Attachment with ID attachment-1 has already been processed. Skipping.');
|
|
144
142
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
|
|
145
143
|
});
|
|
146
144
|
it('should add successfully processed attachment IDs to the list', async () => {
|
|
@@ -156,9 +154,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
156
154
|
'attachment-2',
|
|
157
155
|
'attachment-3'
|
|
158
156
|
]);
|
|
159
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-1');
|
|
160
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-2');
|
|
161
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-3');
|
|
162
157
|
});
|
|
163
158
|
it('should handle processing errors gracefully', async () => {
|
|
164
159
|
const error = new Error('Processing failed');
|
|
@@ -206,40 +201,38 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
206
201
|
expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
|
|
207
202
|
});
|
|
208
203
|
});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
stream: mockStream
|
|
216
|
-
});
|
|
217
|
-
const result = await pool.streamAll();
|
|
218
|
-
expect(result).toEqual({});
|
|
219
|
-
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
204
|
+
it('[edge] should handle single attachment', async () => {
|
|
205
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
206
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
207
|
+
adapter: mockAdapter,
|
|
208
|
+
attachments: [mockAttachments[0]],
|
|
209
|
+
stream: mockStream
|
|
220
210
|
});
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
211
|
+
const result = await pool.streamAll();
|
|
212
|
+
expect(result).toEqual({});
|
|
213
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
214
|
+
});
|
|
215
|
+
it('[edge] should handle batch size larger than attachments array', async () => {
|
|
216
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
217
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
218
|
+
adapter: mockAdapter,
|
|
219
|
+
attachments: mockAttachments,
|
|
220
|
+
batchSize: 100,
|
|
221
|
+
stream: mockStream
|
|
231
222
|
});
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
223
|
+
await pool.streamAll();
|
|
224
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
225
|
+
});
|
|
226
|
+
it('[edge] should handle batch size of 1', async () => {
|
|
227
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
228
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
229
|
+
adapter: mockAdapter,
|
|
230
|
+
attachments: mockAttachments,
|
|
231
|
+
batchSize: 1,
|
|
232
|
+
stream: mockStream
|
|
242
233
|
});
|
|
234
|
+
await pool.streamAll();
|
|
235
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
243
236
|
});
|
|
244
237
|
describe('concurrency behavior', () => {
|
|
245
238
|
it('should process attachments concurrently within batch size', async () => {
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { EventType } from '../types/extraction';
|
|
2
|
-
export declare const STATELESS_EVENT_TYPES: EventType[];
|
|
3
2
|
export declare const ALLOWED_EXTRACTION_EVENT_TYPES: EventType[];
|
|
4
3
|
export declare const ALLOWED_LOADING_EVENT_TYPES: EventType[];
|
|
5
4
|
export declare const ALLOWED_EVENT_TYPES: EventType[];
|
|
5
|
+
export declare const STATELESS_EXTRACTION_EVENT_TYPES: EventType[];
|
|
6
|
+
export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
|
|
7
|
+
export declare const STATELESS_EVENT_TYPES: EventType[];
|
|
8
|
+
export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
|
|
9
|
+
export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
|
|
10
|
+
export declare const STATEFUL_EVENT_TYPES: EventType[];
|
|
6
11
|
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
7
12
|
export declare const MAX_DEVREV_ARTIFACT_SIZE = 262144000;
|
|
8
13
|
export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
|
package/dist/common/constants.js
CHANGED
|
@@ -1,16 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DEFAULT_SLEEP_DELAY_MS = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.
|
|
3
|
+
exports.DEFAULT_SLEEP_DELAY_MS = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
4
4
|
const extraction_1 = require("../types/extraction");
|
|
5
5
|
const helpers_1 = require("./helpers");
|
|
6
|
-
exports.STATELESS_EVENT_TYPES = [
|
|
7
|
-
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
8
|
-
extraction_1.EventType.ExtractionMetadataStart,
|
|
9
|
-
extraction_1.EventType.ExtractionDataDelete,
|
|
10
|
-
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
11
|
-
extraction_1.EventType.StartDeletingLoaderState,
|
|
12
|
-
extraction_1.EventType.StartDeletingLoaderAttachmentState,
|
|
13
|
-
];
|
|
14
6
|
exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
|
|
15
7
|
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
16
8
|
extraction_1.EventType.ExtractionMetadataStart,
|
|
@@ -31,6 +23,25 @@ exports.ALLOWED_EVENT_TYPES = [
|
|
|
31
23
|
...exports.ALLOWED_EXTRACTION_EVENT_TYPES,
|
|
32
24
|
...exports.ALLOWED_LOADING_EVENT_TYPES,
|
|
33
25
|
];
|
|
26
|
+
exports.STATELESS_EXTRACTION_EVENT_TYPES = [
|
|
27
|
+
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
28
|
+
extraction_1.EventType.ExtractionDataDelete,
|
|
29
|
+
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
30
|
+
];
|
|
31
|
+
exports.STATELESS_LOADING_EVENT_TYPES = [
|
|
32
|
+
extraction_1.EventType.StartDeletingLoaderState,
|
|
33
|
+
extraction_1.EventType.StartDeletingLoaderAttachmentState,
|
|
34
|
+
];
|
|
35
|
+
exports.STATELESS_EVENT_TYPES = [
|
|
36
|
+
...exports.STATELESS_EXTRACTION_EVENT_TYPES,
|
|
37
|
+
...exports.STATELESS_LOADING_EVENT_TYPES,
|
|
38
|
+
];
|
|
39
|
+
exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES.filter((eventType) => !exports.STATELESS_EXTRACTION_EVENT_TYPES.includes(eventType));
|
|
40
|
+
exports.STATEFUL_LOADING_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES.filter((eventType) => !exports.STATELESS_LOADING_EVENT_TYPES.includes(eventType));
|
|
41
|
+
exports.STATEFUL_EVENT_TYPES = [
|
|
42
|
+
...exports.STATEFUL_EXTRACTION_EVENT_TYPES,
|
|
43
|
+
...exports.STATEFUL_LOADING_EVENT_TYPES,
|
|
44
|
+
];
|
|
34
45
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
35
46
|
exports.MAX_DEVREV_ARTIFACT_SIZE = 262144000; // 250MB
|
|
36
47
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.emit = void 0;
|
|
4
|
-
const
|
|
4
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
5
5
|
const constants_1 = require("./constants");
|
|
6
6
|
const emit = async ({ event, eventType, data, }) => {
|
|
7
7
|
const newEvent = {
|
|
@@ -13,7 +13,7 @@ const emit = async ({ event, eventType, data, }) => {
|
|
|
13
13
|
},
|
|
14
14
|
};
|
|
15
15
|
console.info('Emitting event', newEvent);
|
|
16
|
-
return
|
|
16
|
+
return axios_client_internal_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
|
|
17
17
|
headers: {
|
|
18
18
|
Accept: 'application/json, text/plain, */*',
|
|
19
19
|
Authorization: event.context.secrets.service_account_token,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const helpers_1 = require("./helpers");
|
|
4
|
-
describe(
|
|
4
|
+
describe(helpers_1.getFilesToLoad.name, () => {
|
|
5
5
|
let statsFile;
|
|
6
6
|
beforeEach(() => {
|
|
7
7
|
statsFile = [
|
|
@@ -49,34 +49,7 @@ describe('getFilesToLoad', () => {
|
|
|
49
49
|
},
|
|
50
50
|
];
|
|
51
51
|
});
|
|
52
|
-
it('should
|
|
53
|
-
statsFile = [];
|
|
54
|
-
const itemTypesToLoad = [];
|
|
55
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
56
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
57
|
-
statsFile,
|
|
58
|
-
});
|
|
59
|
-
expect(result).toEqual([]);
|
|
60
|
-
});
|
|
61
|
-
it('should return an empty array if itemTypesToLoad is empty', () => {
|
|
62
|
-
const itemTypesToLoad = [];
|
|
63
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
64
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
65
|
-
statsFile,
|
|
66
|
-
});
|
|
67
|
-
expect(result).toEqual([]);
|
|
68
|
-
});
|
|
69
|
-
it('should return an empty array if statsFile has no matching items', () => {
|
|
70
|
-
const itemTypesToLoad = [
|
|
71
|
-
{ itemType: 'users', create: jest.fn(), update: jest.fn() },
|
|
72
|
-
];
|
|
73
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
74
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
75
|
-
statsFile,
|
|
76
|
-
});
|
|
77
|
-
expect(result).toEqual([]);
|
|
78
|
-
});
|
|
79
|
-
it('should filter out files not in itemTypesToLoad and order them by itemTypesToLoad', () => {
|
|
52
|
+
it('should filter files by supported item types and order them correctly', () => {
|
|
80
53
|
const itemTypesToLoad = [
|
|
81
54
|
{ itemType: 'attachments', create: jest.fn(), update: jest.fn() },
|
|
82
55
|
{ itemType: 'issues', create: jest.fn(), update: jest.fn() },
|
|
@@ -155,4 +128,31 @@ describe('getFilesToLoad', () => {
|
|
|
155
128
|
},
|
|
156
129
|
]);
|
|
157
130
|
});
|
|
131
|
+
it('[edge] should return an empty array when statsFile is empty', () => {
|
|
132
|
+
statsFile = [];
|
|
133
|
+
const itemTypesToLoad = [];
|
|
134
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
135
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
136
|
+
statsFile,
|
|
137
|
+
});
|
|
138
|
+
expect(result).toEqual([]);
|
|
139
|
+
});
|
|
140
|
+
it('[edge] should return an empty array when itemTypesToLoad is empty', () => {
|
|
141
|
+
const itemTypesToLoad = [];
|
|
142
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
143
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
144
|
+
statsFile,
|
|
145
|
+
});
|
|
146
|
+
expect(result).toEqual([]);
|
|
147
|
+
});
|
|
148
|
+
it('[edge] should return an empty array when statsFile has no matching items', () => {
|
|
149
|
+
const itemTypesToLoad = [
|
|
150
|
+
{ itemType: 'users', create: jest.fn(), update: jest.fn() },
|
|
151
|
+
];
|
|
152
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
153
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
154
|
+
statsFile,
|
|
155
|
+
});
|
|
156
|
+
expect(result).toEqual([]);
|
|
157
|
+
});
|
|
158
158
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.installInitialDomainMapping = installInitialDomainMapping;
|
|
4
|
-
const
|
|
4
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
5
5
|
const logger_1 = require("../logger/logger");
|
|
6
6
|
async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
7
7
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
@@ -13,7 +13,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
13
13
|
return;
|
|
14
14
|
}
|
|
15
15
|
// Get snap-in details
|
|
16
|
-
const snapInResponse = await
|
|
16
|
+
const snapInResponse = await axios_client_internal_1.axiosClient.get(devrevEndpoint + '/internal/snap-ins.get', {
|
|
17
17
|
headers: {
|
|
18
18
|
Authorization: devrevToken,
|
|
19
19
|
},
|
|
@@ -33,7 +33,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
33
33
|
if (startingRecipeBlueprint &&
|
|
34
34
|
Object.keys(startingRecipeBlueprint).length !== 0) {
|
|
35
35
|
try {
|
|
36
|
-
const recipeBlueprintResponse = await
|
|
36
|
+
const recipeBlueprintResponse = await axios_client_internal_1.axiosClient.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
|
|
37
37
|
headers: {
|
|
38
38
|
Authorization: devrevToken,
|
|
39
39
|
},
|
|
@@ -47,7 +47,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
47
47
|
}
|
|
48
48
|
// Install the initial domain mappings
|
|
49
49
|
const additionalMappings = initialDomainMappingJson.additional_mappings || {};
|
|
50
|
-
const initialDomainMappingInstallResponse = await
|
|
50
|
+
const initialDomainMappingInstallResponse = await axios_client_internal_1.axiosClient.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
|
|
51
51
|
starting_recipe_blueprint: recipeBlueprintId,
|
|
52
52
|
})), additionalMappings), {
|
|
53
53
|
headers: {
|
|
@@ -5,16 +5,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
const axios_1 = __importDefault(require("axios"));
|
|
7
7
|
const install_initial_domain_mapping_1 = require("./install-initial-domain-mapping");
|
|
8
|
-
const
|
|
8
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
9
9
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
10
10
|
const extraction_1 = require("../types/extraction");
|
|
11
11
|
// Mock dependencies
|
|
12
12
|
jest.mock('axios', () => (Object.assign(Object.assign({}, jest.requireActual('axios')), { isAxiosError: jest.fn() })));
|
|
13
|
-
jest.mock('../http/axios-client')
|
|
13
|
+
jest.mock('../http/axios-client-internal', () => {
|
|
14
|
+
const originalModule = jest.requireActual('../http/axios-client-internal');
|
|
15
|
+
return Object.assign(Object.assign({}, originalModule), { axiosClient: {
|
|
16
|
+
get: jest.fn(),
|
|
17
|
+
post: jest.fn(),
|
|
18
|
+
} });
|
|
19
|
+
});
|
|
14
20
|
jest.mock('../logger/logger');
|
|
15
|
-
const mockAxiosClient =
|
|
21
|
+
const mockAxiosClient = axios_client_internal_1.axiosClient;
|
|
16
22
|
const mockIsAxiosError = axios_1.default.isAxiosError;
|
|
17
|
-
describe(
|
|
23
|
+
describe(install_initial_domain_mapping_1.installInitialDomainMapping.name, () => {
|
|
18
24
|
// Create mock objects
|
|
19
25
|
const mockEvent = (0, test_helpers_1.createEvent)({ eventType: extraction_1.EventType.ExtractionDataStart });
|
|
20
26
|
const mockInitialDomainMapping = {
|
|
@@ -36,18 +42,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
36
42
|
};
|
|
37
43
|
const mockEndpoint = 'test_devrev_endpoint';
|
|
38
44
|
const mockToken = 'test_token';
|
|
39
|
-
let mockConsoleLog;
|
|
40
|
-
let mockConsoleWarn;
|
|
41
|
-
let mockConsoleError;
|
|
42
|
-
// Before each test, create a fresh spy.
|
|
43
|
-
beforeEach(() => {
|
|
44
|
-
// Re-initialize the spy and its mock implementation
|
|
45
|
-
mockConsoleLog = jest.spyOn(console, 'log').mockImplementation(() => { });
|
|
46
|
-
mockConsoleWarn = jest.spyOn(console, 'warn').mockImplementation(() => { });
|
|
47
|
-
mockConsoleError = jest
|
|
48
|
-
.spyOn(console, 'error')
|
|
49
|
-
.mockImplementation(() => { });
|
|
50
|
-
});
|
|
51
45
|
// After each test, clear all mocks to prevent state from leaking.
|
|
52
46
|
afterEach(() => {
|
|
53
47
|
jest.clearAllMocks();
|
|
@@ -103,8 +97,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
103
97
|
Authorization: mockToken,
|
|
104
98
|
},
|
|
105
99
|
});
|
|
106
|
-
expect(mockConsoleLog).toHaveBeenCalledWith('Successfully created recipe blueprint with id: recipe-blueprint-123');
|
|
107
|
-
expect(mockConsoleLog).toHaveBeenCalledWith(`Successfully installed initial domain mapping ${JSON.stringify(mockDomainMappingResponse.data)}`);
|
|
108
100
|
});
|
|
109
101
|
it('should successfully install without recipe blueprint when not provided', async () => {
|
|
110
102
|
const mappingWithoutBlueprint = {
|
|
@@ -147,19 +139,17 @@ describe('installInitialDomainMapping', () => {
|
|
|
147
139
|
// Should only make one POST request (no recipe blueprint creation for empty object)
|
|
148
140
|
expect(mockAxiosClient.post).toHaveBeenCalledTimes(1);
|
|
149
141
|
});
|
|
150
|
-
it('should return early with warning when
|
|
142
|
+
it('[edge] should return early with warning when initial domain mapping is null', async () => {
|
|
151
143
|
await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, null);
|
|
152
|
-
expect(mockConsoleWarn).toHaveBeenCalledWith('No initial domain mapping found.');
|
|
153
144
|
expect(mockAxiosClient.get).not.toHaveBeenCalled();
|
|
154
145
|
expect(mockAxiosClient.post).not.toHaveBeenCalled();
|
|
155
146
|
});
|
|
156
|
-
it('should return early with warning when
|
|
147
|
+
it('[edge] should return early with warning when initial domain mapping is undefined', async () => {
|
|
157
148
|
await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, undefined);
|
|
158
|
-
expect(mockConsoleWarn).toHaveBeenCalledWith('No initial domain mapping found.');
|
|
159
149
|
expect(mockAxiosClient.get).not.toHaveBeenCalled();
|
|
160
150
|
expect(mockAxiosClient.post).not.toHaveBeenCalled();
|
|
161
151
|
});
|
|
162
|
-
it('should throw error when import slug is missing', async () => {
|
|
152
|
+
it('[edge] should throw error when import slug is missing', async () => {
|
|
163
153
|
const snapInResponseWithoutImport = {
|
|
164
154
|
data: {
|
|
165
155
|
snap_in: {
|
|
@@ -169,9 +159,9 @@ describe('installInitialDomainMapping', () => {
|
|
|
169
159
|
},
|
|
170
160
|
};
|
|
171
161
|
mockAxiosClient.get.mockResolvedValueOnce(snapInResponseWithoutImport);
|
|
172
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
162
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
173
163
|
});
|
|
174
|
-
it('should throw error when snap-in slug is missing', async () => {
|
|
164
|
+
it('[edge] should throw error when snap-in slug is missing', async () => {
|
|
175
165
|
const snapInResponseWithoutSlug = {
|
|
176
166
|
data: {
|
|
177
167
|
snap_in: {
|
|
@@ -181,7 +171,7 @@ describe('installInitialDomainMapping', () => {
|
|
|
181
171
|
},
|
|
182
172
|
};
|
|
183
173
|
mockAxiosClient.get.mockResolvedValueOnce(snapInResponseWithoutSlug);
|
|
184
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
174
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
185
175
|
});
|
|
186
176
|
it('should handle the error during recipe blueprint creation', async () => {
|
|
187
177
|
mockAxiosClient.get.mockResolvedValueOnce(mockSnapInResponse);
|
|
@@ -210,6 +200,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
210
200
|
mockAxiosClient.post.mockResolvedValueOnce(mockRecipeBlueprintResponse);
|
|
211
201
|
const domainMappingError = new Error('Domain mapping installation failed');
|
|
212
202
|
mockAxiosClient.post.mockRejectedValueOnce(domainMappingError);
|
|
213
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
203
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
214
204
|
});
|
|
215
205
|
});
|
|
@@ -34,7 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.Uploader = void 0;
|
|
37
|
-
const
|
|
37
|
+
const axios_client_internal_1 = require("../../http/axios-client-internal");
|
|
38
38
|
const typescript_sdk_1 = require("@devrev/typescript-sdk");
|
|
39
39
|
const fs_1 = __importStar(require("fs"));
|
|
40
40
|
const helpers_1 = require("../common/helpers");
|
|
@@ -118,7 +118,7 @@ class Uploader {
|
|
|
118
118
|
) {
|
|
119
119
|
const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
|
|
120
120
|
try {
|
|
121
|
-
const response = await
|
|
121
|
+
const response = await axios_client_internal_1.axiosClient.post(preparedArtifact.url, formData, {
|
|
122
122
|
headers: {
|
|
123
123
|
'Content-Type': 'multipart/form-data',
|
|
124
124
|
},
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.axiosClient = void 0;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
9
|
+
const axiosClient = axios_1.default.create();
|
|
10
|
+
exports.axiosClient = axiosClient;
|
|
11
|
+
(0, axios_retry_1.default)(axiosClient, {
|
|
12
|
+
retries: 5,
|
|
13
|
+
retryDelay: (retryCount, error) => {
|
|
14
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
|
|
15
|
+
// If the response status is 429, retry after the time specified in the Retry-After header
|
|
16
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
17
|
+
const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
|
|
18
|
+
((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
|
|
19
|
+
// Since DevRev API returns the retry-after header in seconds and axios-retry expects milliseconds, we need to convert it to milliseconds
|
|
20
|
+
const delay = parseInt(retryAfter, 10) * 1000;
|
|
21
|
+
const delayInSeconds = Math.round(delay / 1000);
|
|
22
|
+
console.warn(`Retrying ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.method} request to ${(_g = error.config) === null || _g === void 0 ? void 0 : _g.url} in ${delayInSeconds}s due to 429 Too Many Requests.`);
|
|
23
|
+
return delay;
|
|
24
|
+
}
|
|
25
|
+
// Default exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
|
|
26
|
+
// This will retry requests after 2, 4, 8, 16 and 32 seconds
|
|
27
|
+
const delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
28
|
+
const delayInSeconds = Math.round(delay / 1000);
|
|
29
|
+
console.warn(`Retrying ${(_h = error.config) === null || _h === void 0 ? void 0 : _h.method} request to ${(_j = error.config) === null || _j === void 0 ? void 0 : _j.url} in ${delayInSeconds}s due to ${(_k = error.response) === null || _k === void 0 ? void 0 : _k.status} error.`);
|
|
30
|
+
return delay;
|
|
31
|
+
},
|
|
32
|
+
retryCondition: (error) => {
|
|
33
|
+
var _a, _b, _c, _d, _e, _f;
|
|
34
|
+
const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
|
|
35
|
+
((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
|
|
36
|
+
// 5xx errors
|
|
37
|
+
if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
// 429 errors when retry-after header is present
|
|
41
|
+
else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
|
|
42
|
+
error.response.status === 429 &&
|
|
43
|
+
retryAfter &&
|
|
44
|
+
!isNaN(Number(retryAfter)) &&
|
|
45
|
+
Number(retryAfter) >= 0) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// all other errors
|
|
49
|
+
else {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
onMaxRetryTimesExceeded(error) {
|
|
54
|
+
var _a, _b, _c, _d, _e, _f;
|
|
55
|
+
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
56
|
+
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
57
|
+
(_e = error.request) === null || _e === void 0 ? true : delete _e._header;
|
|
58
|
+
console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
|
|
59
|
+
},
|
|
60
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|