@devrev/ts-adaas 1.2.6 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -272
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/http/axios-client.d.ts +23 -0
- package/dist/http/axios-client.js +27 -3
- package/dist/repo/repo.d.ts +1 -1
- package/dist/state/state.d.ts +1 -1
- package/dist/state/state.interfaces.d.ts +6 -1
- package/dist/state/state.js +25 -2
- package/dist/tests/test-helpers.js +1 -1
- package/dist/types/extraction.d.ts +3 -1
- package/dist/types/workers.d.ts +4 -0
- package/dist/workers/process-task.js +2 -0
- package/dist/workers/spawn.d.ts +1 -1
- package/dist/workers/spawn.js +2 -1
- package/dist/workers/worker-adapter.d.ts +24 -2
- package/dist/workers/worker-adapter.js +156 -39
- package/dist/workers/worker-adapter.test.d.ts +1 -0
- package/dist/workers/worker-adapter.test.js +664 -0
- package/package.json +8 -8
|
@@ -0,0 +1,664 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const worker_adapter_1 = require("./worker-adapter");
|
|
4
|
+
const state_1 = require("../state/state");
|
|
5
|
+
const test_helpers_1 = require("../tests/test-helpers");
|
|
6
|
+
const types_1 = require("../types");
|
|
7
|
+
// Mock dependencies
|
|
8
|
+
jest.mock('../common/control-protocol', () => ({
|
|
9
|
+
emit: jest.fn().mockResolvedValue({}),
|
|
10
|
+
}));
|
|
11
|
+
// const mockPostState = jest.spyOn(State.prototype, 'postState').mockResolvedValue(); // Mock to resolve void
|
|
12
|
+
// const mockFetchState = jest.spyOn(State.prototype, 'fetchState').mockResolvedValue({}); // Mock to resolve a default state
|
|
13
|
+
jest.mock('../mappers/mappers');
|
|
14
|
+
jest.mock('../uploader/uploader');
|
|
15
|
+
// jest.mock('../state/state');
|
|
16
|
+
jest.mock('../repo/repo');
|
|
17
|
+
jest.mock('node:worker_threads', () => ({
|
|
18
|
+
parentPort: {
|
|
19
|
+
postMessage: jest.fn(),
|
|
20
|
+
},
|
|
21
|
+
}));
|
|
22
|
+
describe('WorkerAdapter', () => {
|
|
23
|
+
let adapter;
|
|
24
|
+
let mockEvent;
|
|
25
|
+
let mockAdapterState;
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
// Reset all mocks
|
|
28
|
+
jest.clearAllMocks();
|
|
29
|
+
// Create mock objects
|
|
30
|
+
mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
|
|
31
|
+
const initialState = {
|
|
32
|
+
attachments: { completed: false },
|
|
33
|
+
lastSyncStarted: '',
|
|
34
|
+
lastSuccessfulSyncStarted: '',
|
|
35
|
+
snapInVersionId: '',
|
|
36
|
+
toDevRev: {
|
|
37
|
+
attachmentsMetadata: {
|
|
38
|
+
artifactIds: [],
|
|
39
|
+
lastProcessed: 0,
|
|
40
|
+
lastProcessedAttachmentsIdsList: [],
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
mockAdapterState = new state_1.State({
|
|
45
|
+
event: mockEvent,
|
|
46
|
+
initialState: initialState,
|
|
47
|
+
});
|
|
48
|
+
// Create the adapter instance
|
|
49
|
+
adapter = new worker_adapter_1.WorkerAdapter({
|
|
50
|
+
event: mockEvent,
|
|
51
|
+
adapterState: mockAdapterState,
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
describe('defaultAttachmentsReducer', () => {
|
|
55
|
+
it('should correctly batch attachments based on the batchSize', () => {
|
|
56
|
+
// Arrange
|
|
57
|
+
const attachments = [
|
|
58
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
59
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
60
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
61
|
+
{ url: 'http://example.com/file4.pdf', id: 'attachment4', file_name: 'file4.pdf', parent_id: 'parent4' },
|
|
62
|
+
{ url: 'http://example.com/file5.pdf', id: 'attachment5', file_name: 'file5.pdf', parent_id: 'parent5' },
|
|
63
|
+
];
|
|
64
|
+
// Act - call the private method using function call notation
|
|
65
|
+
const result = adapter['defaultAttachmentsReducer']({
|
|
66
|
+
attachments,
|
|
67
|
+
adapter,
|
|
68
|
+
batchSize: 2,
|
|
69
|
+
});
|
|
70
|
+
// Assert
|
|
71
|
+
expect(result).toHaveLength(3);
|
|
72
|
+
expect(result[0]).toHaveLength(2);
|
|
73
|
+
expect(result[1]).toHaveLength(2);
|
|
74
|
+
expect(result[2]).toHaveLength(1);
|
|
75
|
+
expect(result[0][0].id).toBe('attachment1');
|
|
76
|
+
expect(result[0][1].id).toBe('attachment2');
|
|
77
|
+
expect(result[1][0].id).toBe('attachment3');
|
|
78
|
+
expect(result[1][1].id).toBe('attachment4');
|
|
79
|
+
expect(result[2][0].id).toBe('attachment5');
|
|
80
|
+
});
|
|
81
|
+
it('should return a single batch when batchSize equals the number of attachments', () => {
|
|
82
|
+
// Arrange
|
|
83
|
+
const attachments = [
|
|
84
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
85
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
86
|
+
];
|
|
87
|
+
// Act
|
|
88
|
+
const result = adapter['defaultAttachmentsReducer']({
|
|
89
|
+
attachments,
|
|
90
|
+
adapter,
|
|
91
|
+
batchSize: 2,
|
|
92
|
+
});
|
|
93
|
+
// Assert
|
|
94
|
+
expect(result).toHaveLength(1);
|
|
95
|
+
expect(result[0]).toHaveLength(2);
|
|
96
|
+
expect(result[0][0].id).toBe('attachment1');
|
|
97
|
+
expect(result[0][1].id).toBe('attachment2');
|
|
98
|
+
});
|
|
99
|
+
it('should return a single batch when batchSize is bigger than the number of attachments', () => {
|
|
100
|
+
// Arrange
|
|
101
|
+
const attachments = [
|
|
102
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
103
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
104
|
+
];
|
|
105
|
+
// Act
|
|
106
|
+
const result = adapter['defaultAttachmentsReducer']({
|
|
107
|
+
attachments,
|
|
108
|
+
adapter,
|
|
109
|
+
batchSize: 10,
|
|
110
|
+
});
|
|
111
|
+
// Assert
|
|
112
|
+
expect(result).toHaveLength(1);
|
|
113
|
+
expect(result[0]).toHaveLength(2);
|
|
114
|
+
expect(result[0][0].id).toBe('attachment1');
|
|
115
|
+
expect(result[0][1].id).toBe('attachment2');
|
|
116
|
+
});
|
|
117
|
+
it('should handle empty attachments array', () => {
|
|
118
|
+
// Arrange
|
|
119
|
+
const attachments = [];
|
|
120
|
+
// Act
|
|
121
|
+
const result = adapter['defaultAttachmentsReducer']({
|
|
122
|
+
attachments,
|
|
123
|
+
adapter,
|
|
124
|
+
batchSize: 2,
|
|
125
|
+
});
|
|
126
|
+
// Assert
|
|
127
|
+
expect(result).toHaveLength(0);
|
|
128
|
+
});
|
|
129
|
+
it('should default to batchSize of 1 if not provided', () => {
|
|
130
|
+
// Arrange
|
|
131
|
+
const attachments = [
|
|
132
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
133
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
134
|
+
];
|
|
135
|
+
// Act
|
|
136
|
+
const result = adapter['defaultAttachmentsReducer']({
|
|
137
|
+
attachments,
|
|
138
|
+
adapter
|
|
139
|
+
});
|
|
140
|
+
// Assert
|
|
141
|
+
expect(result).toHaveLength(2);
|
|
142
|
+
expect(result[0]).toHaveLength(1);
|
|
143
|
+
expect(result[1]).toHaveLength(1);
|
|
144
|
+
});
|
|
145
|
+
it('should handle invalid (0) batch size', async () => {
|
|
146
|
+
var _a;
|
|
147
|
+
// Arrange
|
|
148
|
+
const mockStream = jest.fn();
|
|
149
|
+
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
150
|
+
// Act
|
|
151
|
+
const result = await adapter.streamAttachments({
|
|
152
|
+
stream: mockStream,
|
|
153
|
+
batchSize: 0,
|
|
154
|
+
});
|
|
155
|
+
// Assert
|
|
156
|
+
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
157
|
+
expect(result).toEqual({
|
|
158
|
+
error: expect.any(Error),
|
|
159
|
+
});
|
|
160
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Invalid attachments batch size');
|
|
161
|
+
// Restore console.error
|
|
162
|
+
consoleErrorSpy.mockRestore();
|
|
163
|
+
});
|
|
164
|
+
it('should handle invalid (negative) batch size', async () => {
|
|
165
|
+
var _a;
|
|
166
|
+
// Arrange
|
|
167
|
+
const mockStream = jest.fn();
|
|
168
|
+
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
169
|
+
// Act
|
|
170
|
+
const result = await adapter.streamAttachments({
|
|
171
|
+
stream: mockStream,
|
|
172
|
+
batchSize: -1,
|
|
173
|
+
});
|
|
174
|
+
// Assert
|
|
175
|
+
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
176
|
+
expect(result).toEqual({
|
|
177
|
+
error: expect.any(Error),
|
|
178
|
+
});
|
|
179
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Invalid attachments batch size');
|
|
180
|
+
// Restore console.error
|
|
181
|
+
consoleErrorSpy.mockRestore();
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
describe('defaultAttachmentsIterator', () => {
|
|
185
|
+
it('should process all batches of attachments', async () => {
|
|
186
|
+
var _a, _b;
|
|
187
|
+
// Arrange
|
|
188
|
+
const mockAttachments = [
|
|
189
|
+
[
|
|
190
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
191
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
192
|
+
],
|
|
193
|
+
[
|
|
194
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
195
|
+
],
|
|
196
|
+
];
|
|
197
|
+
const mockStream = jest.fn();
|
|
198
|
+
// Mock the processAttachment method
|
|
199
|
+
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
200
|
+
// Act
|
|
201
|
+
const result = await adapter['defaultAttachmentsIterator']({
|
|
202
|
+
reducedAttachments: mockAttachments,
|
|
203
|
+
adapter: adapter,
|
|
204
|
+
stream: mockStream,
|
|
205
|
+
});
|
|
206
|
+
// Assert
|
|
207
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(3);
|
|
208
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
|
|
209
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
|
|
210
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[1][0], mockStream);
|
|
211
|
+
// Verify the state was updated correctly
|
|
212
|
+
expect((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessed).toBe(2);
|
|
213
|
+
expect((_b = adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
|
|
214
|
+
expect(result).toEqual({});
|
|
215
|
+
});
|
|
216
|
+
it('should handle rate limiting during processing', async () => {
|
|
217
|
+
var _a;
|
|
218
|
+
// Arrange
|
|
219
|
+
const mockAttachments = [
|
|
220
|
+
[
|
|
221
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
222
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
223
|
+
],
|
|
224
|
+
[
|
|
225
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
226
|
+
],
|
|
227
|
+
];
|
|
228
|
+
const mockStream = jest.fn();
|
|
229
|
+
// Mock the processAttachment method to simulate rate limiting on the second attachment
|
|
230
|
+
adapter.processAttachment = jest.fn()
|
|
231
|
+
.mockResolvedValueOnce(null) // First attachment processes successfully
|
|
232
|
+
.mockResolvedValueOnce({ delay: 30 }); // Second attachment hits rate limit
|
|
233
|
+
// Set up adapter state
|
|
234
|
+
adapter.state.toDevRev = {
|
|
235
|
+
attachmentsMetadata: {
|
|
236
|
+
lastProcessed: 0,
|
|
237
|
+
artifactIds: [],
|
|
238
|
+
lastProcessedAttachmentsIdsList: [],
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
// Act
|
|
242
|
+
const result = await adapter['defaultAttachmentsIterator']({
|
|
243
|
+
reducedAttachments: mockAttachments,
|
|
244
|
+
adapter: adapter,
|
|
245
|
+
stream: mockStream,
|
|
246
|
+
});
|
|
247
|
+
// Assert
|
|
248
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(2);
|
|
249
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
|
|
250
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
|
|
251
|
+
// Verify the delay was returned
|
|
252
|
+
expect(result).toEqual({ delay: 30 });
|
|
253
|
+
// And lastProcessed wasn't updated yet
|
|
254
|
+
expect((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessed).toBe(0);
|
|
255
|
+
});
|
|
256
|
+
it('should skip already processed attachments', async () => {
|
|
257
|
+
// Arrange
|
|
258
|
+
const mockAttachments = [
|
|
259
|
+
[
|
|
260
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
261
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
262
|
+
],
|
|
263
|
+
];
|
|
264
|
+
const mockStream = jest.fn();
|
|
265
|
+
// Mock the processAttachment method
|
|
266
|
+
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
267
|
+
// Set up adapter state to indicate attachment1 was already processed
|
|
268
|
+
adapter.state.toDevRev = {
|
|
269
|
+
attachmentsMetadata: {
|
|
270
|
+
lastProcessed: 0,
|
|
271
|
+
artifactIds: [],
|
|
272
|
+
lastProcessedAttachmentsIdsList: ['attachment1'],
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
// Act
|
|
276
|
+
await adapter['defaultAttachmentsIterator']({
|
|
277
|
+
reducedAttachments: mockAttachments,
|
|
278
|
+
adapter: adapter,
|
|
279
|
+
stream: mockStream,
|
|
280
|
+
});
|
|
281
|
+
// Assert
|
|
282
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
283
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
|
|
284
|
+
expect(adapter.processAttachment).not.toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
|
|
285
|
+
});
|
|
286
|
+
it('should continue from last processed batch', async () => {
|
|
287
|
+
// Arrange
|
|
288
|
+
const mockAttachments = [
|
|
289
|
+
[
|
|
290
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
291
|
+
],
|
|
292
|
+
[
|
|
293
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
294
|
+
],
|
|
295
|
+
];
|
|
296
|
+
const mockStream = jest.fn();
|
|
297
|
+
// Mock the processAttachment method
|
|
298
|
+
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
299
|
+
// Set up adapter state to indicate we already processed the first batch
|
|
300
|
+
adapter.state.toDevRev = {
|
|
301
|
+
attachmentsMetadata: {
|
|
302
|
+
lastProcessed: 1, // Skip first batch (index 0)
|
|
303
|
+
artifactIds: [],
|
|
304
|
+
lastProcessedAttachmentsIdsList: [],
|
|
305
|
+
},
|
|
306
|
+
};
|
|
307
|
+
// Act
|
|
308
|
+
await adapter['defaultAttachmentsIterator']({
|
|
309
|
+
reducedAttachments: mockAttachments,
|
|
310
|
+
adapter: adapter,
|
|
311
|
+
stream: mockStream,
|
|
312
|
+
});
|
|
313
|
+
// Assert
|
|
314
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
315
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[1][0], mockStream);
|
|
316
|
+
expect(adapter.processAttachment).not.toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
|
|
317
|
+
});
|
|
318
|
+
it('should handle errors during processing and continue', async () => {
|
|
319
|
+
// Arrange
|
|
320
|
+
const mockAttachments = [
|
|
321
|
+
[
|
|
322
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
323
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
324
|
+
],
|
|
325
|
+
];
|
|
326
|
+
const mockStream = jest.fn();
|
|
327
|
+
// Mock processAttachment to throw an error for the first attachment
|
|
328
|
+
adapter.processAttachment = jest.fn()
|
|
329
|
+
.mockRejectedValueOnce(new Error('Processing error'))
|
|
330
|
+
.mockResolvedValueOnce(null);
|
|
331
|
+
// Mock console.warn to avoid test output noise
|
|
332
|
+
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
333
|
+
// Set up adapter state
|
|
334
|
+
adapter.state.toDevRev = {
|
|
335
|
+
attachmentsMetadata: {
|
|
336
|
+
lastProcessed: 0,
|
|
337
|
+
artifactIds: [],
|
|
338
|
+
lastProcessedAttachmentsIdsList: [],
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
// Act
|
|
342
|
+
const result = await adapter['defaultAttachmentsIterator']({
|
|
343
|
+
reducedAttachments: mockAttachments,
|
|
344
|
+
adapter: adapter,
|
|
345
|
+
stream: mockStream,
|
|
346
|
+
});
|
|
347
|
+
// Assert - both attachments should have been processed
|
|
348
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(2);
|
|
349
|
+
expect(consoleWarnSpy).toHaveBeenCalled();
|
|
350
|
+
expect(result).toEqual({});
|
|
351
|
+
// Restore console.warn
|
|
352
|
+
consoleWarnSpy.mockRestore();
|
|
353
|
+
});
|
|
354
|
+
});
|
|
355
|
+
describe('streamAttachments', () => {
|
|
356
|
+
it('should process all artifact batches successfully', async () => {
|
|
357
|
+
// Arrange
|
|
358
|
+
const mockStream = jest.fn();
|
|
359
|
+
// Set up adapter state with artifact IDs
|
|
360
|
+
adapter.state.toDevRev = {
|
|
361
|
+
attachmentsMetadata: {
|
|
362
|
+
artifactIds: ['artifact1', 'artifact2'],
|
|
363
|
+
lastProcessed: 0,
|
|
364
|
+
lastProcessedAttachmentsIdsList: [],
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
// Mock getting attachments from each artifact
|
|
368
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn()
|
|
369
|
+
.mockResolvedValueOnce({
|
|
370
|
+
attachments: [
|
|
371
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
372
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
373
|
+
],
|
|
374
|
+
})
|
|
375
|
+
.mockResolvedValueOnce({
|
|
376
|
+
attachments: [
|
|
377
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
378
|
+
],
|
|
379
|
+
});
|
|
380
|
+
// Mock the initializeRepos method
|
|
381
|
+
adapter.initializeRepos = jest.fn();
|
|
382
|
+
// Mock the defaultAttachmentsReducer and defaultAttachmentsIterator
|
|
383
|
+
const mockReducedAttachments = [['batch1']];
|
|
384
|
+
adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue(mockReducedAttachments);
|
|
385
|
+
adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({});
|
|
386
|
+
// Act
|
|
387
|
+
const result = await adapter.streamAttachments({
|
|
388
|
+
stream: mockStream,
|
|
389
|
+
});
|
|
390
|
+
// Assert
|
|
391
|
+
expect(adapter.initializeRepos).toHaveBeenCalledWith([
|
|
392
|
+
{ itemType: 'ssor_attachment' },
|
|
393
|
+
]);
|
|
394
|
+
expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
|
|
395
|
+
expect(adapter['defaultAttachmentsReducer']).toHaveBeenCalledTimes(2);
|
|
396
|
+
expect(adapter['defaultAttachmentsIterator']).toHaveBeenCalledTimes(2);
|
|
397
|
+
// Verify state was updated correctly
|
|
398
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
|
|
399
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
400
|
+
expect(result).toBeUndefined();
|
|
401
|
+
});
|
|
402
|
+
it('should handle invalid batch size', async () => {
|
|
403
|
+
var _a;
|
|
404
|
+
// Arrange
|
|
405
|
+
const mockStream = jest.fn();
|
|
406
|
+
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
407
|
+
// Act
|
|
408
|
+
const result = await adapter.streamAttachments({
|
|
409
|
+
stream: mockStream,
|
|
410
|
+
batchSize: 0,
|
|
411
|
+
});
|
|
412
|
+
// Assert
|
|
413
|
+
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
414
|
+
expect(result).toEqual({
|
|
415
|
+
error: expect.any(Error),
|
|
416
|
+
});
|
|
417
|
+
expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Invalid attachments batch size');
|
|
418
|
+
// Restore console.error
|
|
419
|
+
consoleErrorSpy.mockRestore();
|
|
420
|
+
});
|
|
421
|
+
it('should handle empty attachments metadata artifact IDs', async () => {
|
|
422
|
+
// Arrange
|
|
423
|
+
const mockStream = jest.fn();
|
|
424
|
+
// Set up adapter state with no artifact IDs
|
|
425
|
+
adapter.state.toDevRev = {
|
|
426
|
+
attachmentsMetadata: {
|
|
427
|
+
artifactIds: [],
|
|
428
|
+
lastProcessed: 0,
|
|
429
|
+
},
|
|
430
|
+
};
|
|
431
|
+
const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
432
|
+
// Act
|
|
433
|
+
const result = await adapter.streamAttachments({
|
|
434
|
+
stream: mockStream,
|
|
435
|
+
});
|
|
436
|
+
// Assert
|
|
437
|
+
expect(consoleLogSpy).toHaveBeenCalledWith('No attachments metadata artifact IDs found in state.');
|
|
438
|
+
expect(result).toBeUndefined();
|
|
439
|
+
// Restore console.log
|
|
440
|
+
consoleLogSpy.mockRestore();
|
|
441
|
+
});
|
|
442
|
+
it('should handle errors when getting attachments', async () => {
|
|
443
|
+
// Arrange
|
|
444
|
+
const mockStream = jest.fn();
|
|
445
|
+
// Set up adapter state with artifact IDs
|
|
446
|
+
adapter.state.toDevRev = {
|
|
447
|
+
attachmentsMetadata: {
|
|
448
|
+
artifactIds: ['artifact1'],
|
|
449
|
+
lastProcessed: 0,
|
|
450
|
+
lastProcessedAttachmentsIdsList: [],
|
|
451
|
+
},
|
|
452
|
+
};
|
|
453
|
+
// Mock error when getting attachments
|
|
454
|
+
const mockError = new Error('Failed to get attachments');
|
|
455
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
456
|
+
error: mockError,
|
|
457
|
+
});
|
|
458
|
+
// Mock methods
|
|
459
|
+
adapter.initializeRepos = jest.fn();
|
|
460
|
+
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
461
|
+
// Act
|
|
462
|
+
const result = await adapter.streamAttachments({
|
|
463
|
+
stream: mockStream,
|
|
464
|
+
});
|
|
465
|
+
// Assert
|
|
466
|
+
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
467
|
+
expect(result).toEqual({
|
|
468
|
+
error: mockError,
|
|
469
|
+
});
|
|
470
|
+
// Restore console.error
|
|
471
|
+
consoleErrorSpy.mockRestore();
|
|
472
|
+
});
|
|
473
|
+
it('should handle empty attachments array from artifact', async () => {
|
|
474
|
+
// Arrange
|
|
475
|
+
const mockStream = jest.fn();
|
|
476
|
+
// Set up adapter state with artifact IDs
|
|
477
|
+
adapter.state.toDevRev = {
|
|
478
|
+
attachmentsMetadata: {
|
|
479
|
+
artifactIds: ['artifact1'],
|
|
480
|
+
lastProcessed: 0,
|
|
481
|
+
lastProcessedAttachmentsIdsList: [],
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
// Mock getting empty attachments
|
|
485
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
486
|
+
attachments: [],
|
|
487
|
+
});
|
|
488
|
+
// Mock methods
|
|
489
|
+
adapter.initializeRepos = jest.fn();
|
|
490
|
+
const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
491
|
+
// Act
|
|
492
|
+
const result = await adapter.streamAttachments({
|
|
493
|
+
stream: mockStream,
|
|
494
|
+
});
|
|
495
|
+
// Assert
|
|
496
|
+
expect(consoleWarnSpy).toHaveBeenCalled();
|
|
497
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
|
|
498
|
+
expect(result).toBeUndefined();
|
|
499
|
+
// Restore console.warn
|
|
500
|
+
consoleWarnSpy.mockRestore();
|
|
501
|
+
});
|
|
502
|
+
it('should use custom processors when provided', async () => {
|
|
503
|
+
// Arrange
|
|
504
|
+
const mockStream = jest.fn();
|
|
505
|
+
const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
|
|
506
|
+
const mockIterator = jest.fn().mockResolvedValue({});
|
|
507
|
+
// Set up adapter state with artifact IDs
|
|
508
|
+
adapter.state.toDevRev = {
|
|
509
|
+
attachmentsMetadata: {
|
|
510
|
+
artifactIds: ['artifact1'],
|
|
511
|
+
lastProcessed: 0,
|
|
512
|
+
lastProcessedAttachmentsIdsList: [],
|
|
513
|
+
},
|
|
514
|
+
};
|
|
515
|
+
// Mock getting attachments
|
|
516
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
517
|
+
attachments: [{ id: 'attachment1' }],
|
|
518
|
+
});
|
|
519
|
+
// Mock methods
|
|
520
|
+
adapter.initializeRepos = jest.fn();
|
|
521
|
+
const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
522
|
+
// Act
|
|
523
|
+
const result = await adapter.streamAttachments({
|
|
524
|
+
stream: mockStream,
|
|
525
|
+
processors: {
|
|
526
|
+
reducer: mockReducer,
|
|
527
|
+
iterator: mockIterator,
|
|
528
|
+
},
|
|
529
|
+
});
|
|
530
|
+
// Assert
|
|
531
|
+
expect(mockReducer).toHaveBeenCalledWith({
|
|
532
|
+
attachments: [{ id: 'attachment1' }],
|
|
533
|
+
adapter: adapter,
|
|
534
|
+
batchSize: 1,
|
|
535
|
+
});
|
|
536
|
+
expect(mockIterator).toHaveBeenCalledWith({
|
|
537
|
+
reducedAttachments: ['custom-reduced'],
|
|
538
|
+
adapter: adapter,
|
|
539
|
+
stream: mockStream,
|
|
540
|
+
});
|
|
541
|
+
expect(result).toBeUndefined();
|
|
542
|
+
// Restore console.log
|
|
543
|
+
consoleLogSpy.mockRestore();
|
|
544
|
+
});
|
|
545
|
+
it('should handle rate limiting from iterator', async () => {
|
|
546
|
+
// Arrange
|
|
547
|
+
const mockStream = jest.fn();
|
|
548
|
+
// Set up adapter state with artifact IDs
|
|
549
|
+
adapter.state.toDevRev = {
|
|
550
|
+
attachmentsMetadata: {
|
|
551
|
+
artifactIds: ['artifact1'],
|
|
552
|
+
lastProcessed: 0,
|
|
553
|
+
lastProcessedAttachmentsIdsList: [],
|
|
554
|
+
},
|
|
555
|
+
};
|
|
556
|
+
// Mock getting attachments
|
|
557
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
558
|
+
attachments: [{ id: 'attachment1' }],
|
|
559
|
+
});
|
|
560
|
+
// Mock methods
|
|
561
|
+
adapter.initializeRepos = jest.fn();
|
|
562
|
+
adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue([]);
|
|
563
|
+
adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({
|
|
564
|
+
delay: 30,
|
|
565
|
+
});
|
|
566
|
+
// Act
|
|
567
|
+
const result = await adapter.streamAttachments({
|
|
568
|
+
stream: mockStream,
|
|
569
|
+
});
|
|
570
|
+
// Assert
|
|
571
|
+
expect(result).toEqual({
|
|
572
|
+
delay: 30,
|
|
573
|
+
});
|
|
574
|
+
// The artifactIds array should remain unchanged
|
|
575
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
|
|
576
|
+
});
|
|
577
|
+
it('should handle error from iterator', async () => {
|
|
578
|
+
// Arrange
|
|
579
|
+
const mockStream = jest.fn();
|
|
580
|
+
// Set up adapter state with artifact IDs
|
|
581
|
+
adapter.state.toDevRev = {
|
|
582
|
+
attachmentsMetadata: {
|
|
583
|
+
artifactIds: ['artifact1'],
|
|
584
|
+
lastProcessed: 0,
|
|
585
|
+
lastProcessedAttachmentsIdsList: [],
|
|
586
|
+
},
|
|
587
|
+
};
|
|
588
|
+
// Mock getting attachments
|
|
589
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
590
|
+
attachments: [{ id: 'attachment1' }],
|
|
591
|
+
});
|
|
592
|
+
// Mock methods
|
|
593
|
+
adapter.initializeRepos = jest.fn();
|
|
594
|
+
adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue([]);
|
|
595
|
+
const mockError = new Error('Iterator error');
|
|
596
|
+
adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({
|
|
597
|
+
error: mockError,
|
|
598
|
+
});
|
|
599
|
+
// Act
|
|
600
|
+
const result = await adapter.streamAttachments({
|
|
601
|
+
stream: mockStream,
|
|
602
|
+
});
|
|
603
|
+
// Assert
|
|
604
|
+
expect(result).toEqual({
|
|
605
|
+
error: mockError,
|
|
606
|
+
});
|
|
607
|
+
// The artifactIds array should remain unchanged
|
|
608
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
|
|
609
|
+
});
|
|
610
|
+
it('should continue processing from last processed attachment for the current artifact', async () => {
|
|
611
|
+
const mockStream = jest.fn();
|
|
612
|
+
adapter.state.toDevRev = {
|
|
613
|
+
attachmentsMetadata: {
|
|
614
|
+
artifactIds: ['artifact1'],
|
|
615
|
+
lastProcessed: 0,
|
|
616
|
+
lastProcessedAttachmentsIdsList: ['attachment1', 'attachment2'],
|
|
617
|
+
},
|
|
618
|
+
};
|
|
619
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
|
|
620
|
+
attachments: [
|
|
621
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
622
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
623
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
624
|
+
],
|
|
625
|
+
});
|
|
626
|
+
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
627
|
+
await adapter.streamAttachments({
|
|
628
|
+
stream: mockStream,
|
|
629
|
+
batchSize: 3,
|
|
630
|
+
});
|
|
631
|
+
expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
|
|
632
|
+
expect(adapter.processAttachment).toHaveBeenCalledWith({
|
|
633
|
+
url: 'http://example.com/file3.pdf',
|
|
634
|
+
id: 'attachment3',
|
|
635
|
+
file_name: 'file3.pdf',
|
|
636
|
+
parent_id: 'parent3'
|
|
637
|
+
}, mockStream);
|
|
638
|
+
});
|
|
639
|
+
it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
|
|
640
|
+
const mockStream = jest.fn();
|
|
641
|
+
adapter.state.toDevRev = {
|
|
642
|
+
attachmentsMetadata: {
|
|
643
|
+
artifactIds: ['artifact1'],
|
|
644
|
+
lastProcessed: 0,
|
|
645
|
+
lastProcessedAttachmentsIdsList: [],
|
|
646
|
+
},
|
|
647
|
+
};
|
|
648
|
+
adapter['uploader'].getAttachmentsFromArtifactId = jest.fn()
|
|
649
|
+
.mockResolvedValueOnce({
|
|
650
|
+
attachments: [
|
|
651
|
+
{ url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
|
|
652
|
+
{ url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
|
|
653
|
+
{ url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
|
|
654
|
+
],
|
|
655
|
+
});
|
|
656
|
+
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
657
|
+
await adapter.streamAttachments({
|
|
658
|
+
stream: mockStream,
|
|
659
|
+
});
|
|
660
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
|
|
661
|
+
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
662
|
+
});
|
|
663
|
+
});
|
|
664
|
+
});
|