@devrev/ts-adaas 1.8.1-beta.1 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +33 -40
- package/dist/common/helpers.test.js +29 -29
- package/dist/common/install-initial-domain-mapping.test.js +8 -24
- package/dist/logger/logger.test.js +4 -4
- package/dist/repo/repo.test.js +51 -37
- package/dist/tests/timeout-handling/timeout-1.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-2.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3a.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3b.test.js +1 -1
- package/dist/types/extraction.test.js +51 -5
- package/dist/uploader/uploader.test.js +7 -10
- package/dist/workers/create-worker.test.js +64 -2
- package/dist/workers/worker-adapter.test.js +34 -97
- package/package.json +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
|
|
4
|
-
describe(
|
|
4
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
|
|
5
5
|
let mockAdapter;
|
|
6
6
|
let mockStream;
|
|
7
7
|
let mockAttachments;
|
|
@@ -50,7 +50,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
50
50
|
jest.clearAllMocks();
|
|
51
51
|
jest.restoreAllMocks();
|
|
52
52
|
});
|
|
53
|
-
describe(
|
|
53
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.constructor.name, () => {
|
|
54
54
|
it('should initialize with default values', () => {
|
|
55
55
|
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
56
56
|
adapter: mockAdapter,
|
|
@@ -82,7 +82,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
82
82
|
expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
|
|
83
83
|
});
|
|
84
84
|
});
|
|
85
|
-
describe(
|
|
85
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.streamAll.name, () => {
|
|
86
86
|
it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
|
|
87
87
|
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined;
|
|
88
88
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
@@ -116,7 +116,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
116
116
|
const result = await pool.streamAll();
|
|
117
117
|
expect(result).toEqual({});
|
|
118
118
|
expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
|
|
119
|
-
expect(console.log).toHaveBeenCalledWith('Starting download of 0 attachments, streaming 10 at once.');
|
|
120
119
|
});
|
|
121
120
|
it('should return delay when rate limit is hit', async () => {
|
|
122
121
|
const delayResponse = { delay: 5000 };
|
|
@@ -130,7 +129,7 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
130
129
|
expect(result).toEqual({ delay: 5000 });
|
|
131
130
|
});
|
|
132
131
|
});
|
|
133
|
-
describe(
|
|
132
|
+
describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
|
|
134
133
|
it('should skip already processed attachments', async () => {
|
|
135
134
|
mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = ['attachment-1'];
|
|
136
135
|
mockAdapter.processAttachment.mockResolvedValue({});
|
|
@@ -140,7 +139,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
140
139
|
stream: mockStream
|
|
141
140
|
});
|
|
142
141
|
await pool.streamAll();
|
|
143
|
-
expect(console.log).toHaveBeenCalledWith('Attachment with ID attachment-1 has already been processed. Skipping.');
|
|
144
142
|
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
|
|
145
143
|
});
|
|
146
144
|
it('should add successfully processed attachment IDs to the list', async () => {
|
|
@@ -156,9 +154,6 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
156
154
|
'attachment-2',
|
|
157
155
|
'attachment-3'
|
|
158
156
|
]);
|
|
159
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-1');
|
|
160
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-2');
|
|
161
|
-
expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-3');
|
|
162
157
|
});
|
|
163
158
|
it('should handle processing errors gracefully', async () => {
|
|
164
159
|
const error = new Error('Processing failed');
|
|
@@ -206,40 +201,38 @@ describe('AttachmentsStreamingPool', () => {
|
|
|
206
201
|
expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
|
|
207
202
|
});
|
|
208
203
|
});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
stream: mockStream
|
|
216
|
-
});
|
|
217
|
-
const result = await pool.streamAll();
|
|
218
|
-
expect(result).toEqual({});
|
|
219
|
-
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
204
|
+
it('[edge] should handle single attachment', async () => {
|
|
205
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
206
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
207
|
+
adapter: mockAdapter,
|
|
208
|
+
attachments: [mockAttachments[0]],
|
|
209
|
+
stream: mockStream
|
|
220
210
|
});
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
211
|
+
const result = await pool.streamAll();
|
|
212
|
+
expect(result).toEqual({});
|
|
213
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
214
|
+
});
|
|
215
|
+
it('[edge] should handle batch size larger than attachments array', async () => {
|
|
216
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
217
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
218
|
+
adapter: mockAdapter,
|
|
219
|
+
attachments: mockAttachments,
|
|
220
|
+
batchSize: 100,
|
|
221
|
+
stream: mockStream
|
|
231
222
|
});
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
223
|
+
await pool.streamAll();
|
|
224
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
225
|
+
});
|
|
226
|
+
it('[edge] should handle batch size of 1', async () => {
|
|
227
|
+
mockAdapter.processAttachment.mockResolvedValue({});
|
|
228
|
+
const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
229
|
+
adapter: mockAdapter,
|
|
230
|
+
attachments: mockAttachments,
|
|
231
|
+
batchSize: 1,
|
|
232
|
+
stream: mockStream
|
|
242
233
|
});
|
|
234
|
+
await pool.streamAll();
|
|
235
|
+
expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
243
236
|
});
|
|
244
237
|
describe('concurrency behavior', () => {
|
|
245
238
|
it('should process attachments concurrently within batch size', async () => {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const helpers_1 = require("./helpers");
|
|
4
|
-
describe(
|
|
4
|
+
describe(helpers_1.getFilesToLoad.name, () => {
|
|
5
5
|
let statsFile;
|
|
6
6
|
beforeEach(() => {
|
|
7
7
|
statsFile = [
|
|
@@ -49,34 +49,7 @@ describe('getFilesToLoad', () => {
|
|
|
49
49
|
},
|
|
50
50
|
];
|
|
51
51
|
});
|
|
52
|
-
it('should
|
|
53
|
-
statsFile = [];
|
|
54
|
-
const itemTypesToLoad = [];
|
|
55
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
56
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
57
|
-
statsFile,
|
|
58
|
-
});
|
|
59
|
-
expect(result).toEqual([]);
|
|
60
|
-
});
|
|
61
|
-
it('should return an empty array if itemTypesToLoad is empty', () => {
|
|
62
|
-
const itemTypesToLoad = [];
|
|
63
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
64
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
65
|
-
statsFile,
|
|
66
|
-
});
|
|
67
|
-
expect(result).toEqual([]);
|
|
68
|
-
});
|
|
69
|
-
it('should return an empty array if statsFile has no matching items', () => {
|
|
70
|
-
const itemTypesToLoad = [
|
|
71
|
-
{ itemType: 'users', create: jest.fn(), update: jest.fn() },
|
|
72
|
-
];
|
|
73
|
-
const result = (0, helpers_1.getFilesToLoad)({
|
|
74
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
75
|
-
statsFile,
|
|
76
|
-
});
|
|
77
|
-
expect(result).toEqual([]);
|
|
78
|
-
});
|
|
79
|
-
it('should filter out files not in itemTypesToLoad and order them by itemTypesToLoad', () => {
|
|
52
|
+
it('should filter files by supported item types and order them correctly', () => {
|
|
80
53
|
const itemTypesToLoad = [
|
|
81
54
|
{ itemType: 'attachments', create: jest.fn(), update: jest.fn() },
|
|
82
55
|
{ itemType: 'issues', create: jest.fn(), update: jest.fn() },
|
|
@@ -155,4 +128,31 @@ describe('getFilesToLoad', () => {
|
|
|
155
128
|
},
|
|
156
129
|
]);
|
|
157
130
|
});
|
|
131
|
+
it('[edge] should return an empty array when statsFile is empty', () => {
|
|
132
|
+
statsFile = [];
|
|
133
|
+
const itemTypesToLoad = [];
|
|
134
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
135
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
136
|
+
statsFile,
|
|
137
|
+
});
|
|
138
|
+
expect(result).toEqual([]);
|
|
139
|
+
});
|
|
140
|
+
it('[edge] should return an empty array when itemTypesToLoad is empty', () => {
|
|
141
|
+
const itemTypesToLoad = [];
|
|
142
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
143
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
144
|
+
statsFile,
|
|
145
|
+
});
|
|
146
|
+
expect(result).toEqual([]);
|
|
147
|
+
});
|
|
148
|
+
it('[edge] should return an empty array when statsFile has no matching items', () => {
|
|
149
|
+
const itemTypesToLoad = [
|
|
150
|
+
{ itemType: 'users', create: jest.fn(), update: jest.fn() },
|
|
151
|
+
];
|
|
152
|
+
const result = (0, helpers_1.getFilesToLoad)({
|
|
153
|
+
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
154
|
+
statsFile,
|
|
155
|
+
});
|
|
156
|
+
expect(result).toEqual([]);
|
|
157
|
+
});
|
|
158
158
|
});
|
|
@@ -20,7 +20,7 @@ jest.mock('../http/axios-client-internal', () => {
|
|
|
20
20
|
jest.mock('../logger/logger');
|
|
21
21
|
const mockAxiosClient = axios_client_internal_1.axiosClient;
|
|
22
22
|
const mockIsAxiosError = axios_1.default.isAxiosError;
|
|
23
|
-
describe(
|
|
23
|
+
describe(install_initial_domain_mapping_1.installInitialDomainMapping.name, () => {
|
|
24
24
|
// Create mock objects
|
|
25
25
|
const mockEvent = (0, test_helpers_1.createEvent)({ eventType: extraction_1.EventType.ExtractionDataStart });
|
|
26
26
|
const mockInitialDomainMapping = {
|
|
@@ -42,18 +42,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
42
42
|
};
|
|
43
43
|
const mockEndpoint = 'test_devrev_endpoint';
|
|
44
44
|
const mockToken = 'test_token';
|
|
45
|
-
let mockConsoleLog;
|
|
46
|
-
let mockConsoleWarn;
|
|
47
|
-
let mockConsoleError;
|
|
48
|
-
// Before each test, create a fresh spy.
|
|
49
|
-
beforeEach(() => {
|
|
50
|
-
// Re-initialize the spy and its mock implementation
|
|
51
|
-
mockConsoleLog = jest.spyOn(console, 'log').mockImplementation(() => { });
|
|
52
|
-
mockConsoleWarn = jest.spyOn(console, 'warn').mockImplementation(() => { });
|
|
53
|
-
mockConsoleError = jest
|
|
54
|
-
.spyOn(console, 'error')
|
|
55
|
-
.mockImplementation(() => { });
|
|
56
|
-
});
|
|
57
45
|
// After each test, clear all mocks to prevent state from leaking.
|
|
58
46
|
afterEach(() => {
|
|
59
47
|
jest.clearAllMocks();
|
|
@@ -109,8 +97,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
109
97
|
Authorization: mockToken,
|
|
110
98
|
},
|
|
111
99
|
});
|
|
112
|
-
expect(mockConsoleLog).toHaveBeenCalledWith('Successfully created recipe blueprint with id: recipe-blueprint-123');
|
|
113
|
-
expect(mockConsoleLog).toHaveBeenCalledWith(`Successfully installed initial domain mapping ${JSON.stringify(mockDomainMappingResponse.data)}`);
|
|
114
100
|
});
|
|
115
101
|
it('should successfully install without recipe blueprint when not provided', async () => {
|
|
116
102
|
const mappingWithoutBlueprint = {
|
|
@@ -153,19 +139,17 @@ describe('installInitialDomainMapping', () => {
|
|
|
153
139
|
// Should only make one POST request (no recipe blueprint creation for empty object)
|
|
154
140
|
expect(mockAxiosClient.post).toHaveBeenCalledTimes(1);
|
|
155
141
|
});
|
|
156
|
-
it('should return early with warning when
|
|
142
|
+
it('[edge] should return early with warning when initial domain mapping is null', async () => {
|
|
157
143
|
await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, null);
|
|
158
|
-
expect(mockConsoleWarn).toHaveBeenCalledWith('No initial domain mapping found.');
|
|
159
144
|
expect(mockAxiosClient.get).not.toHaveBeenCalled();
|
|
160
145
|
expect(mockAxiosClient.post).not.toHaveBeenCalled();
|
|
161
146
|
});
|
|
162
|
-
it('should return early with warning when
|
|
147
|
+
it('[edge] should return early with warning when initial domain mapping is undefined', async () => {
|
|
163
148
|
await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, undefined);
|
|
164
|
-
expect(mockConsoleWarn).toHaveBeenCalledWith('No initial domain mapping found.');
|
|
165
149
|
expect(mockAxiosClient.get).not.toHaveBeenCalled();
|
|
166
150
|
expect(mockAxiosClient.post).not.toHaveBeenCalled();
|
|
167
151
|
});
|
|
168
|
-
it('should throw error when import slug is missing', async () => {
|
|
152
|
+
it('[edge] should throw error when import slug is missing', async () => {
|
|
169
153
|
const snapInResponseWithoutImport = {
|
|
170
154
|
data: {
|
|
171
155
|
snap_in: {
|
|
@@ -175,9 +159,9 @@ describe('installInitialDomainMapping', () => {
|
|
|
175
159
|
},
|
|
176
160
|
};
|
|
177
161
|
mockAxiosClient.get.mockResolvedValueOnce(snapInResponseWithoutImport);
|
|
178
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
162
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
179
163
|
});
|
|
180
|
-
it('should throw error when snap-in slug is missing', async () => {
|
|
164
|
+
it('[edge] should throw error when snap-in slug is missing', async () => {
|
|
181
165
|
const snapInResponseWithoutSlug = {
|
|
182
166
|
data: {
|
|
183
167
|
snap_in: {
|
|
@@ -187,7 +171,7 @@ describe('installInitialDomainMapping', () => {
|
|
|
187
171
|
},
|
|
188
172
|
};
|
|
189
173
|
mockAxiosClient.get.mockResolvedValueOnce(snapInResponseWithoutSlug);
|
|
190
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
174
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
191
175
|
});
|
|
192
176
|
it('should handle the error during recipe blueprint creation', async () => {
|
|
193
177
|
mockAxiosClient.get.mockResolvedValueOnce(mockSnapInResponse);
|
|
@@ -216,6 +200,6 @@ describe('installInitialDomainMapping', () => {
|
|
|
216
200
|
mockAxiosClient.post.mockResolvedValueOnce(mockRecipeBlueprintResponse);
|
|
217
201
|
const domainMappingError = new Error('Domain mapping installation failed');
|
|
218
202
|
mockAxiosClient.post.mockRejectedValueOnce(domainMappingError);
|
|
219
|
-
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow(
|
|
203
|
+
await expect((0, install_initial_domain_mapping_1.installInitialDomainMapping)(mockEvent, mockInitialDomainMapping)).rejects.toThrow();
|
|
220
204
|
});
|
|
221
205
|
});
|
|
@@ -14,7 +14,7 @@ jest.mock('node:worker_threads', () => ({
|
|
|
14
14
|
isMainThread: true,
|
|
15
15
|
parentPort: null,
|
|
16
16
|
}));
|
|
17
|
-
describe(
|
|
17
|
+
describe(logger_1.Logger.name, () => {
|
|
18
18
|
let mockEvent;
|
|
19
19
|
let mockOptions;
|
|
20
20
|
beforeEach(() => {
|
|
@@ -137,7 +137,7 @@ describe('Logger', () => {
|
|
|
137
137
|
mockOptions.isLocalDevelopment = false;
|
|
138
138
|
logger = new logger_1.Logger({ event: mockEvent, options: mockOptions });
|
|
139
139
|
});
|
|
140
|
-
it('should handle empty string message', () => {
|
|
140
|
+
it('[edge] should handle empty string message', () => {
|
|
141
141
|
logger.info('');
|
|
142
142
|
expect(mockConsoleInfo).toHaveBeenCalledTimes(1);
|
|
143
143
|
const callArgs = mockConsoleInfo.mock.calls[0][0];
|
|
@@ -146,7 +146,7 @@ describe('Logger', () => {
|
|
|
146
146
|
expect(logObject.dev_oid).toBe(mockEvent.payload.event_context.dev_oid);
|
|
147
147
|
expect(logObject.request_id).toBe(mockEvent.payload.event_context.request_id);
|
|
148
148
|
});
|
|
149
|
-
it('should handle null and undefined values', () => {
|
|
149
|
+
it('[edge] should handle null and undefined values', () => {
|
|
150
150
|
logger.info('test', null, undefined);
|
|
151
151
|
expect(mockConsoleInfo).toHaveBeenCalledTimes(1);
|
|
152
152
|
const callArgs = mockConsoleInfo.mock.calls[0][0];
|
|
@@ -155,7 +155,7 @@ describe('Logger', () => {
|
|
|
155
155
|
expect(logObject.message).toBe('test null undefined');
|
|
156
156
|
expect(logObject.dev_oid).toBe(mockEvent.payload.event_context.dev_oid);
|
|
157
157
|
});
|
|
158
|
-
it('should handle complex nested objects', () => {
|
|
158
|
+
it('[edge] should handle complex nested objects', () => {
|
|
159
159
|
const complexObject = {
|
|
160
160
|
level1: {
|
|
161
161
|
level2: {
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -5,7 +5,7 @@ const test_helpers_1 = require("../tests/test-helpers");
|
|
|
5
5
|
const types_1 = require("../types");
|
|
6
6
|
const repo_1 = require("./repo");
|
|
7
7
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
8
|
-
describe(
|
|
8
|
+
describe(repo_1.Repo.name, () => {
|
|
9
9
|
let repo;
|
|
10
10
|
let normalize;
|
|
11
11
|
beforeEach(() => {
|
|
@@ -20,18 +20,14 @@ describe('Repo class push method', () => {
|
|
|
20
20
|
afterEach(() => {
|
|
21
21
|
jest.clearAllMocks();
|
|
22
22
|
});
|
|
23
|
-
it('should
|
|
24
|
-
await repo.push([]);
|
|
25
|
-
expect(repo.getItems()).toEqual([]);
|
|
26
|
-
});
|
|
27
|
-
it('should normalize and push 10 items if array is not empty', async () => {
|
|
23
|
+
it('should normalize and push items when array contains items', async () => {
|
|
28
24
|
const items = (0, test_helpers_1.createItems)(10);
|
|
29
25
|
await repo.push(items);
|
|
30
26
|
expect(normalize).toHaveBeenCalledTimes(10);
|
|
31
27
|
const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
|
|
32
28
|
expect(repo.getItems()).toEqual(normalizedItems);
|
|
33
29
|
});
|
|
34
|
-
it('should not normalize items
|
|
30
|
+
it('should not normalize items when normalize function is not provided', async () => {
|
|
35
31
|
repo = new repo_1.Repo({
|
|
36
32
|
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
37
33
|
itemType: 'test_item_type',
|
|
@@ -42,46 +38,56 @@ describe('Repo class push method', () => {
|
|
|
42
38
|
await repo.push(items);
|
|
43
39
|
expect(normalize).not.toHaveBeenCalled();
|
|
44
40
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
expect(normalize).not.toHaveBeenCalled();
|
|
41
|
+
it('[edge] should not push items when items array is empty', async () => {
|
|
42
|
+
await repo.push([]);
|
|
43
|
+
expect(repo.getItems()).toEqual([]);
|
|
44
|
+
});
|
|
45
|
+
it('should not normalize items when item type is external_domain_metadata', async () => {
|
|
46
|
+
repo = new repo_1.Repo({
|
|
47
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
48
|
+
itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA,
|
|
49
|
+
normalize,
|
|
50
|
+
onUpload: jest.fn(),
|
|
51
|
+
options: {},
|
|
57
52
|
});
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
53
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
54
|
+
await repo.push(items);
|
|
55
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
it('should not normalize items when item type is ssor_attachment', async () => {
|
|
58
|
+
repo = new repo_1.Repo({
|
|
59
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
60
|
+
itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT,
|
|
61
|
+
normalize,
|
|
62
|
+
onUpload: jest.fn(),
|
|
63
|
+
options: {},
|
|
69
64
|
});
|
|
65
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
66
|
+
await repo.push(items);
|
|
67
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
70
68
|
});
|
|
71
69
|
it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
|
|
72
70
|
const items = (0, test_helpers_1.createItems)(2005);
|
|
73
71
|
await repo.push(items);
|
|
74
72
|
expect(repo.getItems().length).toBe(5);
|
|
75
73
|
});
|
|
76
|
-
it('should
|
|
77
|
-
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
74
|
+
it('should normalize all items when pushing 4005 items with batch size of 2000', async () => {
|
|
78
75
|
const items = (0, test_helpers_1.createItems)(4005);
|
|
79
76
|
await repo.push(items);
|
|
80
77
|
expect(normalize).toHaveBeenCalledTimes(4005);
|
|
81
|
-
|
|
82
|
-
|
|
78
|
+
});
|
|
79
|
+
it('should upload 2 batches when pushing 4005 items with batch size of 2000', async () => {
|
|
80
|
+
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
81
|
+
const items = (0, test_helpers_1.createItems)(4005);
|
|
82
|
+
await repo.push(items);
|
|
83
|
+
expect(uploadSpy).toHaveBeenCalledTimes(2);
|
|
83
84
|
uploadSpy.mockRestore();
|
|
84
85
|
});
|
|
86
|
+
it('should leave 5 items in array after pushing 4005 items with batch size of 2000', async () => {
|
|
87
|
+
const items = (0, test_helpers_1.createItems)(4005);
|
|
88
|
+
await repo.push(items);
|
|
89
|
+
expect(repo.getItems().length).toBe(5);
|
|
90
|
+
});
|
|
85
91
|
describe('should take batch size into account', () => {
|
|
86
92
|
beforeEach(() => {
|
|
87
93
|
repo = new repo_1.Repo({
|
|
@@ -104,14 +110,22 @@ describe('Repo class push method', () => {
|
|
|
104
110
|
await repo.push(items);
|
|
105
111
|
expect(repo.getItems().length).toBe(5);
|
|
106
112
|
});
|
|
107
|
-
it('should
|
|
108
|
-
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
113
|
+
it('should normalize all items when pushing 205 items with batch size of 50', async () => {
|
|
109
114
|
const items = (0, test_helpers_1.createItems)(205);
|
|
110
115
|
await repo.push(items);
|
|
111
116
|
expect(normalize).toHaveBeenCalledTimes(205);
|
|
112
|
-
|
|
117
|
+
});
|
|
118
|
+
it('should upload 4 batches when pushing 205 items with batch size of 50', async () => {
|
|
119
|
+
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
120
|
+
const items = (0, test_helpers_1.createItems)(205);
|
|
121
|
+
await repo.push(items);
|
|
113
122
|
expect(uploadSpy).toHaveBeenCalledTimes(4);
|
|
114
123
|
uploadSpy.mockRestore();
|
|
115
124
|
});
|
|
125
|
+
it('should leave 5 items in array after pushing 205 items with batch size of 50', async () => {
|
|
126
|
+
const items = (0, test_helpers_1.createItems)(205);
|
|
127
|
+
await repo.push(items);
|
|
128
|
+
expect(repo.getItems().length).toBe(5);
|
|
129
|
+
});
|
|
116
130
|
});
|
|
117
131
|
});
|
|
@@ -7,7 +7,7 @@ const extraction_1 = require("../../types/extraction");
|
|
|
7
7
|
const test_helpers_1 = require("../test-helpers");
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
|
-
describe('timeout-1', () => {
|
|
10
|
+
describe('timeout-1 extraction', () => {
|
|
11
11
|
let mockServer;
|
|
12
12
|
beforeAll(async () => {
|
|
13
13
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-2', () => {
|
|
11
|
+
describe('timeout-2 extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-3a', () => {
|
|
11
|
+
describe('timeout-3a extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -8,7 +8,7 @@ const test_helpers_1 = require("../test-helpers");
|
|
|
8
8
|
const extraction_2 = __importDefault(require("./extraction"));
|
|
9
9
|
const mock_server_1 = require("../mock-server");
|
|
10
10
|
jest.setTimeout(15000);
|
|
11
|
-
describe('timeout-3b', () => {
|
|
11
|
+
describe('timeout-3b extraction', () => {
|
|
12
12
|
let mockServer;
|
|
13
13
|
beforeAll(async () => {
|
|
14
14
|
mockServer = new mock_server_1.MockServer(3001);
|
|
@@ -2,22 +2,68 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const extraction_1 = require("./extraction");
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
-
|
|
5
|
+
// Test the EventContext interface and related extraction types
|
|
6
|
+
describe('ExtractionTypes', () => {
|
|
6
7
|
const baseEvent = (0, test_helpers_1.createEvent)({ eventType: extraction_1.EventType.ExtractionDataStart });
|
|
7
|
-
it('should
|
|
8
|
+
it('should create event context without optional fields', () => {
|
|
8
9
|
const event = Object.assign({}, baseEvent);
|
|
9
10
|
// If this compiles, the test passes
|
|
10
11
|
expect(event).toBeDefined();
|
|
12
|
+
expect(event.payload.event_context).toBeDefined();
|
|
11
13
|
});
|
|
12
|
-
it('should
|
|
14
|
+
it('should create event context with all optional fields', () => {
|
|
13
15
|
const event = Object.assign({}, baseEvent);
|
|
14
16
|
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: '2024-01-01T00:00:00Z', initial_sync_scope: extraction_1.InitialSyncScope.TIME_SCOPED, reset_extract_from: true });
|
|
15
|
-
// Test with all optionals present
|
|
16
17
|
expect(event).toBeDefined();
|
|
18
|
+
expect(event.payload.event_context.extract_from).toBe('2024-01-01T00:00:00Z');
|
|
19
|
+
expect(event.payload.event_context.initial_sync_scope).toBe(extraction_1.InitialSyncScope.TIME_SCOPED);
|
|
20
|
+
expect(event.payload.event_context.reset_extract_from).toBe(true);
|
|
17
21
|
});
|
|
18
|
-
it('should
|
|
22
|
+
it('should create event context with partial optional fields', () => {
|
|
19
23
|
const event = Object.assign({}, baseEvent);
|
|
20
24
|
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: '2024-01-01T00:00:00Z' });
|
|
21
25
|
expect(event).toBeDefined();
|
|
26
|
+
expect(event.payload.event_context.extract_from).toBe('2024-01-01T00:00:00Z');
|
|
27
|
+
});
|
|
28
|
+
it('should handle different InitialSyncScope values', () => {
|
|
29
|
+
const event = Object.assign({}, baseEvent);
|
|
30
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { initial_sync_scope: extraction_1.InitialSyncScope.FULL_HISTORY });
|
|
31
|
+
expect(event.payload.event_context.initial_sync_scope).toBe(extraction_1.InitialSyncScope.FULL_HISTORY);
|
|
32
|
+
});
|
|
33
|
+
it('[edge] should handle null event context gracefully', () => {
|
|
34
|
+
const event = Object.assign({}, baseEvent);
|
|
35
|
+
event.payload.event_context = null;
|
|
36
|
+
expect(event.payload.event_context).toBeNull();
|
|
37
|
+
});
|
|
38
|
+
it('[edge] should handle undefined optional fields', () => {
|
|
39
|
+
const event = Object.assign({}, baseEvent);
|
|
40
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: undefined, initial_sync_scope: undefined, reset_extract_from: undefined });
|
|
41
|
+
expect(event.payload.event_context.extract_from).toBeUndefined();
|
|
42
|
+
expect(event.payload.event_context.initial_sync_scope).toBeUndefined();
|
|
43
|
+
expect(event.payload.event_context.reset_extract_from).toBeUndefined();
|
|
44
|
+
});
|
|
45
|
+
it('[edge] should handle invalid date format in extract_from', () => {
|
|
46
|
+
const event = Object.assign({}, baseEvent);
|
|
47
|
+
event.payload.event_context = Object.assign(Object.assign({}, baseEvent.payload.event_context), { extract_from: 'invalid-date-format' });
|
|
48
|
+
expect(event.payload.event_context.extract_from).toBe('invalid-date-format');
|
|
49
|
+
// Note: Type validation would typically happen at runtime, not compile time
|
|
50
|
+
});
|
|
51
|
+
it('[edge] should handle explicit boolean values for reset_extract_from', () => {
|
|
52
|
+
const eventWithTrue = (0, test_helpers_1.createEvent)({
|
|
53
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
54
|
+
eventContextOverrides: {
|
|
55
|
+
reset_extract_from: true
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
const eventWithFalse = (0, test_helpers_1.createEvent)({
|
|
59
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
60
|
+
eventContextOverrides: {
|
|
61
|
+
reset_extract_from: false
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
expect(eventWithTrue.payload.event_context.reset_extract_from).toBe(true);
|
|
65
|
+
expect(eventWithFalse.payload.event_context.reset_extract_from).toBe(false);
|
|
66
|
+
expect(typeof eventWithTrue.payload.event_context.reset_extract_from).toBe('boolean');
|
|
67
|
+
expect(typeof eventWithFalse.payload.event_context.reset_extract_from).toBe('boolean');
|
|
22
68
|
});
|
|
23
69
|
});
|
|
@@ -27,17 +27,14 @@ const getArtifactUploadUrlMockResponse = {
|
|
|
27
27
|
form_data: [],
|
|
28
28
|
},
|
|
29
29
|
};
|
|
30
|
-
describe(
|
|
30
|
+
describe(uploader_1.Uploader.name, () => {
|
|
31
31
|
const mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
|
|
32
32
|
let uploader;
|
|
33
|
-
let consoleWarnSpy;
|
|
34
33
|
beforeEach(() => {
|
|
35
34
|
uploader = new uploader_1.Uploader({ event: mockEvent });
|
|
36
|
-
consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
37
35
|
});
|
|
38
36
|
afterEach(() => {
|
|
39
37
|
jest.clearAllMocks();
|
|
40
|
-
consoleWarnSpy.mockRestore();
|
|
41
38
|
});
|
|
42
39
|
it('should upload the file to the DevRev platform and return the artifact information', async () => {
|
|
43
40
|
// Mock successful response from getArtifactUploadUrl
|
|
@@ -55,7 +52,7 @@ describe('Uploader Class Tests', () => {
|
|
|
55
52
|
},
|
|
56
53
|
});
|
|
57
54
|
});
|
|
58
|
-
it('should handle failure
|
|
55
|
+
it('[edge] should handle failure when getting artifact upload URL', async () => {
|
|
59
56
|
var _a;
|
|
60
57
|
// Mock unsuccessful response for getArtifactUploadUrl
|
|
61
58
|
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(undefined);
|
|
@@ -63,9 +60,9 @@ describe('Uploader Class Tests', () => {
|
|
|
63
60
|
const fetchedObjects = [{ key: 'value' }];
|
|
64
61
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
65
62
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
66
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
63
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
67
64
|
});
|
|
68
|
-
it('should handle failure
|
|
65
|
+
it('[edge] should handle failure when uploading artifact', async () => {
|
|
69
66
|
var _a;
|
|
70
67
|
// Mock successful response for getArtifactUploadUrl
|
|
71
68
|
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(getArtifactUploadUrlMockResponse);
|
|
@@ -75,9 +72,9 @@ describe('Uploader Class Tests', () => {
|
|
|
75
72
|
const fetchedObjects = [{ key: 'value' }];
|
|
76
73
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
77
74
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
78
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
75
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
79
76
|
});
|
|
80
|
-
it('should handle failure
|
|
77
|
+
it('[edge] should handle failure when confirming artifact upload', async () => {
|
|
81
78
|
var _a;
|
|
82
79
|
// Mock successful response for getArtifactUploadUrl
|
|
83
80
|
axios_client_internal_1.axiosClient.get.mockResolvedValueOnce(getArtifactUploadUrlMockResponse);
|
|
@@ -89,6 +86,6 @@ describe('Uploader Class Tests', () => {
|
|
|
89
86
|
const fetchedObjects = [{ key: 'value' }];
|
|
90
87
|
const uploadResponse = await uploader.upload(entity, fetchedObjects);
|
|
91
88
|
expect(uploadResponse.error).toBeInstanceOf(Error);
|
|
92
|
-
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).
|
|
89
|
+
expect((_a = uploadResponse.error) === null || _a === void 0 ? void 0 : _a.message).toBeDefined();
|
|
93
90
|
});
|
|
94
91
|
});
|
|
@@ -4,8 +4,8 @@ const worker_threads_1 = require("worker_threads");
|
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
5
|
const extraction_1 = require("../types/extraction");
|
|
6
6
|
const create_worker_1 = require("./create-worker");
|
|
7
|
-
describe(
|
|
8
|
-
it('should
|
|
7
|
+
describe(create_worker_1.createWorker.name, () => {
|
|
8
|
+
it('should create a Worker instance when valid parameters are provided', async () => {
|
|
9
9
|
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
10
10
|
const worker = worker_threads_1.isMainThread
|
|
11
11
|
? await (0, create_worker_1.createWorker)({
|
|
@@ -22,4 +22,66 @@ describe('createWorker function', () => {
|
|
|
22
22
|
await worker.terminate();
|
|
23
23
|
}
|
|
24
24
|
});
|
|
25
|
+
it('should throw error when not in main thread', async () => {
|
|
26
|
+
const originalIsMainThread = worker_threads_1.isMainThread;
|
|
27
|
+
worker_threads_1.isMainThread = false;
|
|
28
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
29
|
+
await expect((0, create_worker_1.createWorker)({
|
|
30
|
+
event: (0, test_helpers_1.createEvent)({
|
|
31
|
+
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
32
|
+
}),
|
|
33
|
+
initialState: {},
|
|
34
|
+
workerPath,
|
|
35
|
+
})).rejects.toThrow('Worker threads can not start more worker threads.');
|
|
36
|
+
// Restore original value
|
|
37
|
+
worker_threads_1.isMainThread = originalIsMainThread;
|
|
38
|
+
});
|
|
39
|
+
it('[edge] should handle worker creation with minimal valid data', async () => {
|
|
40
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
41
|
+
if (worker_threads_1.isMainThread) {
|
|
42
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
43
|
+
event: (0, test_helpers_1.createEvent)({
|
|
44
|
+
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
45
|
+
}),
|
|
46
|
+
initialState: {},
|
|
47
|
+
workerPath,
|
|
48
|
+
});
|
|
49
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
50
|
+
await worker.terminate();
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
it('[edge] should handle worker creation with complex initial state', async () => {
|
|
54
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
55
|
+
const complexState = {
|
|
56
|
+
nested: {
|
|
57
|
+
data: [1, 2, 3],
|
|
58
|
+
config: { enabled: true }
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
if (worker_threads_1.isMainThread) {
|
|
62
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
63
|
+
event: (0, test_helpers_1.createEvent)({
|
|
64
|
+
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
65
|
+
}),
|
|
66
|
+
initialState: complexState,
|
|
67
|
+
workerPath,
|
|
68
|
+
});
|
|
69
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
70
|
+
await worker.terminate();
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
it('[edge] should handle different event types', async () => {
|
|
74
|
+
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
75
|
+
if (worker_threads_1.isMainThread) {
|
|
76
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
77
|
+
event: (0, test_helpers_1.createEvent)({
|
|
78
|
+
eventType: extraction_1.EventType.ExtractionMetadataStart,
|
|
79
|
+
}),
|
|
80
|
+
initialState: {},
|
|
81
|
+
workerPath,
|
|
82
|
+
});
|
|
83
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
84
|
+
await worker.terminate();
|
|
85
|
+
}
|
|
86
|
+
});
|
|
25
87
|
});
|
|
@@ -29,7 +29,7 @@ jest.mock('../attachments-streaming/attachments-streaming-pool', () => {
|
|
|
29
29
|
}),
|
|
30
30
|
};
|
|
31
31
|
});
|
|
32
|
-
describe(
|
|
32
|
+
describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
33
33
|
let adapter;
|
|
34
34
|
let mockEvent;
|
|
35
35
|
let mockAdapterState;
|
|
@@ -61,9 +61,8 @@ describe('WorkerAdapter', () => {
|
|
|
61
61
|
adapterState: mockAdapterState,
|
|
62
62
|
});
|
|
63
63
|
});
|
|
64
|
-
describe(
|
|
64
|
+
describe(worker_adapter_1.WorkerAdapter.prototype.streamAttachments.name, () => {
|
|
65
65
|
it('should process all artifact batches successfully', async () => {
|
|
66
|
-
// Arrange
|
|
67
66
|
const mockStream = jest.fn();
|
|
68
67
|
// Set up adapter state with artifact IDs
|
|
69
68
|
adapter.state.toDevRev = {
|
|
@@ -107,7 +106,6 @@ describe('WorkerAdapter', () => {
|
|
|
107
106
|
const result = await adapter.streamAttachments({
|
|
108
107
|
stream: mockStream,
|
|
109
108
|
});
|
|
110
|
-
// Assert
|
|
111
109
|
expect(adapter.initializeRepos).toHaveBeenCalledWith([
|
|
112
110
|
{ itemType: 'ssor_attachment' },
|
|
113
111
|
]);
|
|
@@ -118,10 +116,8 @@ describe('WorkerAdapter', () => {
|
|
|
118
116
|
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
119
117
|
expect(result).toBeUndefined();
|
|
120
118
|
});
|
|
121
|
-
it('should handle invalid batch size', async () => {
|
|
122
|
-
// Arrange
|
|
119
|
+
it('[edge] should handle invalid batch size by using 1 instead', async () => {
|
|
123
120
|
const mockStream = jest.fn();
|
|
124
|
-
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
125
121
|
// Set up adapter state with artifact IDs
|
|
126
122
|
adapter.state.toDevRev = {
|
|
127
123
|
attachmentsMetadata: {
|
|
@@ -131,34 +127,20 @@ describe('WorkerAdapter', () => {
|
|
|
131
127
|
},
|
|
132
128
|
};
|
|
133
129
|
// Mock getting attachments
|
|
134
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
135
|
-
.fn()
|
|
136
|
-
.mockResolvedValue({
|
|
130
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
137
131
|
attachments: [
|
|
138
|
-
{
|
|
139
|
-
url: 'http://example.com/file1.pdf',
|
|
140
|
-
id: 'attachment1',
|
|
141
|
-
file_name: 'file1.pdf',
|
|
142
|
-
parent_id: 'parent1',
|
|
143
|
-
},
|
|
132
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
144
133
|
],
|
|
145
134
|
});
|
|
146
135
|
adapter.initializeRepos = jest.fn();
|
|
147
|
-
// Act
|
|
148
136
|
const result = await adapter.streamAttachments({
|
|
149
137
|
stream: mockStream,
|
|
150
138
|
batchSize: 0,
|
|
151
139
|
});
|
|
152
|
-
// Assert
|
|
153
|
-
expect(consoleWarnSpy).toHaveBeenCalledWith('The specified batch size (0) is invalid. Using 1 instead.');
|
|
154
140
|
expect(result).toBeUndefined();
|
|
155
|
-
// Restore console.warn
|
|
156
|
-
consoleWarnSpy.mockRestore();
|
|
157
141
|
});
|
|
158
|
-
it('should cap batch size to 50 when batchSize is greater than 50', async () => {
|
|
159
|
-
// Arrange
|
|
142
|
+
it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
|
|
160
143
|
const mockStream = jest.fn();
|
|
161
|
-
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
162
144
|
// Set up adapter state with artifact IDs
|
|
163
145
|
adapter.state.toDevRev = {
|
|
164
146
|
attachmentsMetadata: {
|
|
@@ -168,33 +150,20 @@ describe('WorkerAdapter', () => {
|
|
|
168
150
|
},
|
|
169
151
|
};
|
|
170
152
|
// Mock getting attachments
|
|
171
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
172
|
-
.fn()
|
|
173
|
-
.mockResolvedValue({
|
|
153
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
174
154
|
attachments: [
|
|
175
|
-
{
|
|
176
|
-
url: 'http://example.com/file1.pdf',
|
|
177
|
-
id: 'attachment1',
|
|
178
|
-
file_name: 'file1.pdf',
|
|
179
|
-
parent_id: 'parent1',
|
|
180
|
-
},
|
|
155
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
181
156
|
],
|
|
182
157
|
});
|
|
183
158
|
// Mock the required methods
|
|
184
159
|
adapter.initializeRepos = jest.fn();
|
|
185
|
-
// Act
|
|
186
160
|
const result = await adapter.streamAttachments({
|
|
187
161
|
stream: mockStream,
|
|
188
162
|
batchSize: 100, // Set batch size greater than 50
|
|
189
163
|
});
|
|
190
|
-
// Assert
|
|
191
|
-
expect(consoleWarnSpy).toHaveBeenCalledWith('The specified batch size (100) is too large. Using 50 instead.');
|
|
192
164
|
expect(result).toBeUndefined();
|
|
193
|
-
// Restore console.warn
|
|
194
|
-
consoleWarnSpy.mockRestore();
|
|
195
165
|
});
|
|
196
|
-
it('should handle empty attachments metadata artifact IDs', async () => {
|
|
197
|
-
// Arrange
|
|
166
|
+
it('[edge] should handle empty attachments metadata artifact IDs', async () => {
|
|
198
167
|
const mockStream = jest.fn();
|
|
199
168
|
// Set up adapter state with no artifact IDs
|
|
200
169
|
adapter.state.toDevRev = {
|
|
@@ -203,19 +172,12 @@ describe('WorkerAdapter', () => {
|
|
|
203
172
|
lastProcessed: 0,
|
|
204
173
|
},
|
|
205
174
|
};
|
|
206
|
-
const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
207
|
-
// Act
|
|
208
175
|
const result = await adapter.streamAttachments({
|
|
209
176
|
stream: mockStream,
|
|
210
177
|
});
|
|
211
|
-
// Assert
|
|
212
|
-
expect(consoleLogSpy).toHaveBeenCalledWith('No attachments metadata artifact IDs found in state.');
|
|
213
178
|
expect(result).toBeUndefined();
|
|
214
|
-
// Restore console.log
|
|
215
|
-
consoleLogSpy.mockRestore();
|
|
216
179
|
});
|
|
217
|
-
it('should handle errors when getting attachments', async () => {
|
|
218
|
-
// Arrange
|
|
180
|
+
it('[edge] should handle errors when getting attachments', async () => {
|
|
219
181
|
const mockStream = jest.fn();
|
|
220
182
|
// Set up adapter state with artifact IDs
|
|
221
183
|
adapter.state.toDevRev = {
|
|
@@ -227,28 +189,19 @@ describe('WorkerAdapter', () => {
|
|
|
227
189
|
};
|
|
228
190
|
// Mock error when getting attachments
|
|
229
191
|
const mockError = new Error('Failed to get attachments');
|
|
230
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
231
|
-
.fn()
|
|
232
|
-
.mockResolvedValue({
|
|
192
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
233
193
|
error: mockError,
|
|
234
194
|
});
|
|
235
195
|
// Mock methods
|
|
236
196
|
adapter.initializeRepos = jest.fn();
|
|
237
|
-
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
238
|
-
// Act
|
|
239
197
|
const result = await adapter.streamAttachments({
|
|
240
198
|
stream: mockStream,
|
|
241
199
|
});
|
|
242
|
-
// Assert
|
|
243
|
-
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
244
200
|
expect(result).toEqual({
|
|
245
201
|
error: mockError,
|
|
246
202
|
});
|
|
247
|
-
// Restore console.error
|
|
248
|
-
consoleErrorSpy.mockRestore();
|
|
249
203
|
});
|
|
250
|
-
it('should handle empty attachments array from artifact', async () => {
|
|
251
|
-
// Arrange
|
|
204
|
+
it('[edge] should handle empty attachments array from artifact', async () => {
|
|
252
205
|
const mockStream = jest.fn();
|
|
253
206
|
// Set up adapter state with artifact IDs
|
|
254
207
|
adapter.state.toDevRev = {
|
|
@@ -259,27 +212,18 @@ describe('WorkerAdapter', () => {
|
|
|
259
212
|
},
|
|
260
213
|
};
|
|
261
214
|
// Mock getting empty attachments
|
|
262
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
263
|
-
.fn()
|
|
264
|
-
.mockResolvedValue({
|
|
215
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
265
216
|
attachments: [],
|
|
266
217
|
});
|
|
267
218
|
// Mock methods
|
|
268
219
|
adapter.initializeRepos = jest.fn();
|
|
269
|
-
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
270
|
-
// Act
|
|
271
220
|
const result = await adapter.streamAttachments({
|
|
272
221
|
stream: mockStream,
|
|
273
222
|
});
|
|
274
|
-
// Assert
|
|
275
|
-
expect(consoleWarnSpy).toHaveBeenCalled();
|
|
276
223
|
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
|
|
277
224
|
expect(result).toBeUndefined();
|
|
278
|
-
// Restore console.warn
|
|
279
|
-
consoleWarnSpy.mockRestore();
|
|
280
225
|
});
|
|
281
226
|
it('should use custom processors when provided', async () => {
|
|
282
|
-
// Arrange
|
|
283
227
|
const mockStream = jest.fn();
|
|
284
228
|
const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
|
|
285
229
|
const mockIterator = jest.fn().mockResolvedValue({});
|
|
@@ -299,8 +243,6 @@ describe('WorkerAdapter', () => {
|
|
|
299
243
|
});
|
|
300
244
|
// Mock methods
|
|
301
245
|
adapter.initializeRepos = jest.fn();
|
|
302
|
-
const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
303
|
-
// Act
|
|
304
246
|
const result = await adapter.streamAttachments({
|
|
305
247
|
stream: mockStream,
|
|
306
248
|
processors: {
|
|
@@ -308,7 +250,6 @@ describe('WorkerAdapter', () => {
|
|
|
308
250
|
iterator: mockIterator,
|
|
309
251
|
},
|
|
310
252
|
});
|
|
311
|
-
// Assert
|
|
312
253
|
expect(mockReducer).toHaveBeenCalledWith({
|
|
313
254
|
attachments: [{ id: 'attachment1' }],
|
|
314
255
|
adapter: adapter,
|
|
@@ -320,11 +261,8 @@ describe('WorkerAdapter', () => {
|
|
|
320
261
|
stream: mockStream,
|
|
321
262
|
});
|
|
322
263
|
expect(result).toBeUndefined();
|
|
323
|
-
// Restore console.log
|
|
324
|
-
consoleLogSpy.mockRestore();
|
|
325
264
|
});
|
|
326
265
|
it('should handle rate limiting from iterator', async () => {
|
|
327
|
-
// Arrange
|
|
328
266
|
const mockStream = jest.fn();
|
|
329
267
|
attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
|
|
330
268
|
return {
|
|
@@ -348,11 +286,9 @@ describe('WorkerAdapter', () => {
|
|
|
348
286
|
});
|
|
349
287
|
// Mock methods
|
|
350
288
|
adapter.initializeRepos = jest.fn();
|
|
351
|
-
// Act
|
|
352
289
|
const result = await adapter.streamAttachments({
|
|
353
290
|
stream: mockStream,
|
|
354
291
|
});
|
|
355
|
-
// Assert
|
|
356
292
|
expect(result).toEqual({
|
|
357
293
|
delay: 30,
|
|
358
294
|
});
|
|
@@ -362,7 +298,6 @@ describe('WorkerAdapter', () => {
|
|
|
362
298
|
]);
|
|
363
299
|
});
|
|
364
300
|
it('should handle error from iterator', async () => {
|
|
365
|
-
// Arrange
|
|
366
301
|
const mockStream = jest.fn();
|
|
367
302
|
attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
|
|
368
303
|
return {
|
|
@@ -388,11 +323,9 @@ describe('WorkerAdapter', () => {
|
|
|
388
323
|
});
|
|
389
324
|
// Mock methods
|
|
390
325
|
adapter.initializeRepos = jest.fn();
|
|
391
|
-
// Act
|
|
392
326
|
const result = await adapter.streamAttachments({
|
|
393
327
|
stream: mockStream,
|
|
394
328
|
});
|
|
395
|
-
// Assert
|
|
396
329
|
expect(result).toEqual({
|
|
397
330
|
error: 'Mock error',
|
|
398
331
|
});
|
|
@@ -442,7 +375,7 @@ describe('WorkerAdapter', () => {
|
|
|
442
375
|
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
443
376
|
});
|
|
444
377
|
});
|
|
445
|
-
describe(
|
|
378
|
+
describe(worker_adapter_1.WorkerAdapter.prototype.emit.name, () => {
|
|
446
379
|
let counter;
|
|
447
380
|
let mockPostMessage;
|
|
448
381
|
beforeEach(() => {
|
|
@@ -450,7 +383,6 @@ describe('WorkerAdapter', () => {
|
|
|
450
383
|
// Import the worker_threads module and spy on parentPort.postMessage
|
|
451
384
|
const workerThreads = require('node:worker_threads');
|
|
452
385
|
mockPostMessage = jest.fn().mockImplementation((a) => {
|
|
453
|
-
console.log('postMessage called with:', a);
|
|
454
386
|
counter.counter += 1;
|
|
455
387
|
});
|
|
456
388
|
// Spy on the parentPort.postMessage method
|
|
@@ -470,10 +402,21 @@ describe('WorkerAdapter', () => {
|
|
|
470
402
|
// Restore all mocks
|
|
471
403
|
jest.restoreAllMocks();
|
|
472
404
|
});
|
|
473
|
-
|
|
474
|
-
adapter['adapterState'].postState = jest
|
|
475
|
-
|
|
476
|
-
|
|
405
|
+
it('should emit only one event when multiple events of same type are sent', async () => {
|
|
406
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
407
|
+
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
408
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
409
|
+
reports: [],
|
|
410
|
+
processed_files: [],
|
|
411
|
+
});
|
|
412
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
413
|
+
reports: [],
|
|
414
|
+
processed_files: [],
|
|
415
|
+
});
|
|
416
|
+
expect(counter.counter).toBe(1);
|
|
417
|
+
});
|
|
418
|
+
it('should emit event when different event type is sent after previous events', async () => {
|
|
419
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
477
420
|
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
478
421
|
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
479
422
|
reports: [],
|
|
@@ -489,10 +432,8 @@ describe('WorkerAdapter', () => {
|
|
|
489
432
|
});
|
|
490
433
|
expect(counter.counter).toBe(1);
|
|
491
434
|
});
|
|
492
|
-
|
|
493
|
-
adapter['adapterState'].postState = jest
|
|
494
|
-
.fn()
|
|
495
|
-
.mockRejectedValue(new Error('postState error'));
|
|
435
|
+
it('should correctly emit one event even if postState errors', async () => {
|
|
436
|
+
adapter['adapterState'].postState = jest.fn().mockRejectedValue(new Error('postState error'));
|
|
496
437
|
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
497
438
|
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
498
439
|
reports: [],
|
|
@@ -500,13 +441,9 @@ describe('WorkerAdapter', () => {
|
|
|
500
441
|
});
|
|
501
442
|
expect(counter.counter).toBe(1);
|
|
502
443
|
});
|
|
503
|
-
|
|
504
|
-
adapter['adapterState'].postState = jest
|
|
505
|
-
|
|
506
|
-
.mockResolvedValue(undefined);
|
|
507
|
-
adapter.uploadAllRepos = jest
|
|
508
|
-
.fn()
|
|
509
|
-
.mockRejectedValue(new Error('uploadAllRepos error'));
|
|
444
|
+
it('should correctly emit one event even if uploadAllRepos errors', async () => {
|
|
445
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
446
|
+
adapter.uploadAllRepos = jest.fn().mockRejectedValue(new Error('uploadAllRepos error'));
|
|
510
447
|
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
511
448
|
reports: [],
|
|
512
449
|
processed_files: [],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/ts-adaas",
|
|
3
|
-
"version": "1.8.1
|
|
3
|
+
"version": "1.8.1",
|
|
4
4
|
"description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"start": "ts-node src/index.ts",
|
|
12
12
|
"lint": "eslint .",
|
|
13
13
|
"lint:fix": "eslint . --fix",
|
|
14
|
-
"test": "jest --forceExit --coverage --runInBand"
|
|
14
|
+
"test": "jest --forceExit --coverage --runInBand --testPathIgnorePatterns='./dist/'"
|
|
15
15
|
},
|
|
16
16
|
"repository": {
|
|
17
17
|
"type": "git",
|