@devrev/ts-adaas 1.2.0-beta.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/common/control-protocol.d.ts +2 -1
- package/dist/common/control-protocol.js +11 -23
- package/dist/common/helpers.d.ts +1 -0
- package/dist/common/helpers.js +47 -0
- package/dist/{http → deprecated/http}/client.d.ts +1 -1
- package/dist/{http → deprecated/http}/client.js +1 -1
- package/dist/http/axios-client.js +13 -21
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/repo/repo.interfaces.d.ts +1 -1
- package/dist/repo/repo.js +4 -1
- package/dist/state/state.js +13 -7
- package/dist/types/extraction.d.ts +37 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/loading.d.ts +2 -0
- package/dist/types/workers.d.ts +1 -6
- package/dist/uploader/uploader.js +13 -13
- package/dist/workers/default-workers/attachments-extraction.js +22 -13
- package/dist/workers/default-workers/load-attachments.js +2 -2
- package/dist/workers/process-task.js +0 -1
- package/dist/workers/spawn.d.ts +3 -4
- package/dist/workers/spawn.js +71 -45
- package/dist/workers/worker-adapter.d.ts +10 -23
- package/dist/workers/worker-adapter.js +155 -125
- package/package.json +1 -1
- package/dist/workers/default-workers/data-loading.d.ts +0 -1
- package/dist/workers/default-workers/data-loading.js +0 -19
- package/dist/workers/default-workers/loader-state-deletion.d.ts +0 -1
- package/dist/workers/default-workers/loader-state-deletion.js +0 -15
|
@@ -6,6 +6,7 @@ const axios_devrev_client_1 = require("../http/axios-devrev-client");
|
|
|
6
6
|
const extraction_1 = require("../types/extraction");
|
|
7
7
|
const loading_1 = require("../types/loading");
|
|
8
8
|
const constants_1 = require("../common/constants");
|
|
9
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
9
10
|
const control_protocol_1 = require("../common/control-protocol");
|
|
10
11
|
const workers_1 = require("../types/workers");
|
|
11
12
|
const repo_1 = require("../repo/repo");
|
|
@@ -14,11 +15,10 @@ const mappers_1 = require("../mappers/mappers");
|
|
|
14
15
|
const uploader_1 = require("../uploader/uploader");
|
|
15
16
|
const logger_1 = require("../logger/logger");
|
|
16
17
|
const mappers_interface_1 = require("../mappers/mappers.interface");
|
|
17
|
-
function createWorkerAdapter({ event, adapterState,
|
|
18
|
+
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
18
19
|
return new WorkerAdapter({
|
|
19
20
|
event,
|
|
20
21
|
adapterState,
|
|
21
|
-
parentPort,
|
|
22
22
|
options,
|
|
23
23
|
});
|
|
24
24
|
}
|
|
@@ -37,13 +37,12 @@ function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
|
|
|
37
37
|
*
|
|
38
38
|
*/
|
|
39
39
|
class WorkerAdapter {
|
|
40
|
-
constructor({ event, adapterState,
|
|
40
|
+
constructor({ event, adapterState, options, }) {
|
|
41
41
|
this.repos = [];
|
|
42
42
|
this.event = event;
|
|
43
43
|
this.options = options;
|
|
44
44
|
this.adapterState = adapterState;
|
|
45
45
|
this._artifacts = [];
|
|
46
|
-
this.parentPort = parentPort;
|
|
47
46
|
this.hasWorkerEmitted = false;
|
|
48
47
|
this.isTimeout = false;
|
|
49
48
|
// Loader
|
|
@@ -113,9 +112,7 @@ class WorkerAdapter {
|
|
|
113
112
|
*/
|
|
114
113
|
async emit(newEventType, data) {
|
|
115
114
|
if (this.hasWorkerEmitted) {
|
|
116
|
-
console.warn(
|
|
117
|
-
newEventType +
|
|
118
|
-
'. Lambda is going to be stopped.');
|
|
115
|
+
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
119
116
|
return;
|
|
120
117
|
}
|
|
121
118
|
// We want to upload all the repos before emitting the event, except for the external sync units done event
|
|
@@ -124,14 +121,20 @@ class WorkerAdapter {
|
|
|
124
121
|
}
|
|
125
122
|
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
126
123
|
if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
|
|
124
|
+
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
|
|
127
125
|
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
126
|
+
this.state.lastSyncStarted = '';
|
|
128
127
|
}
|
|
129
128
|
// We want to save the state every time we emit an event, except for the start and delete events
|
|
130
129
|
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
131
|
-
console.log(`Saving state before emitting event with event type:
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
130
|
+
console.log(`Saving state before emitting event with event type: ${newEventType}.`);
|
|
131
|
+
try {
|
|
132
|
+
await this.adapterState.postState(this.state);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
console.error('Error while posting state', error);
|
|
136
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
137
|
+
}
|
|
135
138
|
}
|
|
136
139
|
try {
|
|
137
140
|
await (0, control_protocol_1.emit)({
|
|
@@ -146,11 +149,17 @@ class WorkerAdapter {
|
|
|
146
149
|
payload: { eventType: newEventType },
|
|
147
150
|
};
|
|
148
151
|
this.artifacts = [];
|
|
149
|
-
this.
|
|
152
|
+
this.hasWorkerEmitted = true;
|
|
153
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
|
|
150
154
|
}
|
|
151
155
|
catch (error) {
|
|
152
|
-
|
|
153
|
-
|
|
156
|
+
if (axios_devrev_client_1.axios.isAxiosError(error)) {
|
|
157
|
+
console.error(`Error while emitting event with event type: ${newEventType}`, (0, logger_1.serializeAxiosError)(error));
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
console.error(`Unknown error while emitting event with event type: ${newEventType}`, error);
|
|
161
|
+
}
|
|
162
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
154
163
|
}
|
|
155
164
|
}
|
|
156
165
|
async uploadAllRepos() {
|
|
@@ -162,33 +171,40 @@ class WorkerAdapter {
|
|
|
162
171
|
this.isTimeout = true;
|
|
163
172
|
}
|
|
164
173
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
165
|
-
var _a
|
|
174
|
+
var _a;
|
|
166
175
|
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
|
|
176
|
+
const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
|
|
177
|
+
if (!itemTypes.length) {
|
|
178
|
+
console.warn('No item types to load, returning.');
|
|
179
|
+
return {
|
|
180
|
+
reports: this.reports,
|
|
181
|
+
processed_files: this.processedFiles,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
const filesToLoad = await this.getLoaderBatches({
|
|
185
|
+
supportedItemTypes: itemTypes,
|
|
186
|
+
});
|
|
167
187
|
this.adapterState.state.fromDevRev = {
|
|
168
|
-
filesToLoad
|
|
169
|
-
supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
|
|
170
|
-
}),
|
|
188
|
+
filesToLoad,
|
|
171
189
|
};
|
|
172
190
|
}
|
|
173
|
-
if (
|
|
191
|
+
if (!this.adapterState.state.fromDevRev ||
|
|
192
|
+
!this.adapterState.state.fromDevRev.filesToLoad.length) {
|
|
174
193
|
console.warn('No files to load, returning.');
|
|
175
194
|
return {
|
|
176
195
|
reports: this.reports,
|
|
177
196
|
processed_files: this.processedFiles,
|
|
178
197
|
};
|
|
179
198
|
}
|
|
180
|
-
console.log('Files to load in state', (
|
|
181
|
-
outerloop: for (const fileToLoad of
|
|
199
|
+
console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
200
|
+
outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
|
|
201
|
+
.filesToLoad) {
|
|
182
202
|
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
183
203
|
if (!itemTypeToLoad) {
|
|
184
204
|
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
185
|
-
await (
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
data: {
|
|
189
|
-
error: {
|
|
190
|
-
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
191
|
-
},
|
|
205
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
206
|
+
error: {
|
|
207
|
+
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
192
208
|
},
|
|
193
209
|
});
|
|
194
210
|
break;
|
|
@@ -199,8 +215,12 @@ class WorkerAdapter {
|
|
|
199
215
|
isGzipped: true,
|
|
200
216
|
}));
|
|
201
217
|
if (!transformerFile) {
|
|
202
|
-
console.error(
|
|
203
|
-
|
|
218
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
219
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
220
|
+
error: {
|
|
221
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
222
|
+
},
|
|
223
|
+
});
|
|
204
224
|
}
|
|
205
225
|
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
206
226
|
const { report, rateLimit } = await this.loadItem({
|
|
@@ -208,14 +228,10 @@ class WorkerAdapter {
|
|
|
208
228
|
itemTypeToLoad,
|
|
209
229
|
});
|
|
210
230
|
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
211
|
-
await (
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
delay: rateLimit.delay,
|
|
216
|
-
reports: this.reports,
|
|
217
|
-
processed_files: this.processedFiles,
|
|
218
|
-
},
|
|
231
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
|
|
232
|
+
delay: rateLimit.delay,
|
|
233
|
+
reports: this.reports,
|
|
234
|
+
processed_files: this.processedFiles,
|
|
219
235
|
});
|
|
220
236
|
break outerloop;
|
|
221
237
|
}
|
|
@@ -265,6 +281,7 @@ class WorkerAdapter {
|
|
|
265
281
|
}
|
|
266
282
|
if (!this.adapterState.state.fromDevRev ||
|
|
267
283
|
((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
|
|
284
|
+
console.log('No files to load, returning.');
|
|
268
285
|
return {
|
|
269
286
|
reports: this.reports,
|
|
270
287
|
processed_files: this.processedFiles,
|
|
@@ -277,7 +294,7 @@ class WorkerAdapter {
|
|
|
277
294
|
isGzipped: true,
|
|
278
295
|
}));
|
|
279
296
|
if (!transformerFile) {
|
|
280
|
-
console.error(
|
|
297
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
281
298
|
break outerloop;
|
|
282
299
|
}
|
|
283
300
|
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
@@ -286,14 +303,10 @@ class WorkerAdapter {
|
|
|
286
303
|
create,
|
|
287
304
|
});
|
|
288
305
|
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
289
|
-
await (
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
delay: rateLimit.delay,
|
|
294
|
-
reports: this.reports,
|
|
295
|
-
processed_files: this.processedFiles,
|
|
296
|
-
},
|
|
306
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
|
|
307
|
+
delay: rateLimit.delay,
|
|
308
|
+
reports: this.reports,
|
|
309
|
+
processed_files: this.processedFiles,
|
|
297
310
|
});
|
|
298
311
|
break outerloop;
|
|
299
312
|
}
|
|
@@ -324,14 +337,14 @@ class WorkerAdapter {
|
|
|
324
337
|
});
|
|
325
338
|
const syncMapperRecord = syncMapperRecordResponse.data;
|
|
326
339
|
if (!syncMapperRecord) {
|
|
327
|
-
console.
|
|
340
|
+
console.warn('Failed to get sync mapper record from response.');
|
|
328
341
|
return {
|
|
329
342
|
error: {
|
|
330
343
|
message: 'Failed to get sync mapper record from response.',
|
|
331
344
|
},
|
|
332
345
|
};
|
|
333
346
|
}
|
|
334
|
-
// Update item
|
|
347
|
+
// Update item in external system
|
|
335
348
|
const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
|
|
336
349
|
item,
|
|
337
350
|
mappers: this.mappers,
|
|
@@ -340,7 +353,7 @@ class WorkerAdapter {
|
|
|
340
353
|
if (id) {
|
|
341
354
|
if (modifiedDate) {
|
|
342
355
|
try {
|
|
343
|
-
|
|
356
|
+
await this.mappers.update({
|
|
344
357
|
id: syncMapperRecord.sync_mapper_record.id,
|
|
345
358
|
sync_unit: this.event.payload.event_context.sync_unit,
|
|
346
359
|
status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
|
|
@@ -359,11 +372,10 @@ class WorkerAdapter {
|
|
|
359
372
|
add: [devrevId],
|
|
360
373
|
},
|
|
361
374
|
});
|
|
362
|
-
console.log('Updated sync mapper record', JSON.stringify(updateSyncMapperRecordResponse.data));
|
|
363
375
|
}
|
|
364
376
|
catch (error) {
|
|
365
377
|
if (axios_devrev_client_1.axios.isAxiosError(error)) {
|
|
366
|
-
console.
|
|
378
|
+
console.warn('Failed to update sync mapper record', (0, logger_1.serializeAxiosError)(error));
|
|
367
379
|
return {
|
|
368
380
|
error: {
|
|
369
381
|
message: error.message,
|
|
@@ -371,7 +383,7 @@ class WorkerAdapter {
|
|
|
371
383
|
};
|
|
372
384
|
}
|
|
373
385
|
else {
|
|
374
|
-
console.
|
|
386
|
+
console.warn('Failed to update sync mapper record', error);
|
|
375
387
|
return {
|
|
376
388
|
error: {
|
|
377
389
|
message: 'Failed to update sync mapper record' + error,
|
|
@@ -388,7 +400,7 @@ class WorkerAdapter {
|
|
|
388
400
|
};
|
|
389
401
|
}
|
|
390
402
|
else if (delay) {
|
|
391
|
-
console.log(
|
|
403
|
+
console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
|
|
392
404
|
return {
|
|
393
405
|
rateLimit: {
|
|
394
406
|
delay,
|
|
@@ -396,7 +408,7 @@ class WorkerAdapter {
|
|
|
396
408
|
};
|
|
397
409
|
}
|
|
398
410
|
else {
|
|
399
|
-
console.
|
|
411
|
+
console.warn('Failed to update item in external system', error);
|
|
400
412
|
return {
|
|
401
413
|
report: {
|
|
402
414
|
item_type: itemTypeToLoad.itemType,
|
|
@@ -404,12 +416,12 @@ class WorkerAdapter {
|
|
|
404
416
|
},
|
|
405
417
|
};
|
|
406
418
|
}
|
|
407
|
-
// Update mapper (optional)
|
|
419
|
+
// TODO: Update mapper (optional)
|
|
408
420
|
}
|
|
409
421
|
catch (error) {
|
|
410
422
|
if (axios_devrev_client_1.axios.isAxiosError(error)) {
|
|
411
423
|
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
412
|
-
// Create item
|
|
424
|
+
// Create item in external system if mapper record not found
|
|
413
425
|
const { id, delay, error } = await itemTypeToLoad.create({
|
|
414
426
|
item,
|
|
415
427
|
mappers: this.mappers,
|
|
@@ -418,13 +430,12 @@ class WorkerAdapter {
|
|
|
418
430
|
if (id) {
|
|
419
431
|
// Create mapper
|
|
420
432
|
try {
|
|
421
|
-
|
|
433
|
+
await this.mappers.create({
|
|
422
434
|
sync_unit: this.event.payload.event_context.sync_unit,
|
|
423
435
|
status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
|
|
424
436
|
external_ids: [id],
|
|
425
437
|
targets: [devrevId],
|
|
426
438
|
});
|
|
427
|
-
console.log('Created sync mapper record', createdSyncMapperRecordResponse.data.sync_mapper_record.id);
|
|
428
439
|
return {
|
|
429
440
|
report: {
|
|
430
441
|
item_type: itemTypeToLoad.itemType,
|
|
@@ -434,14 +445,14 @@ class WorkerAdapter {
|
|
|
434
445
|
}
|
|
435
446
|
catch (error) {
|
|
436
447
|
if (axios_devrev_client_1.axios.isAxiosError(error)) {
|
|
437
|
-
console.
|
|
448
|
+
console.warn('Failed to create sync mapper record', (0, logger_1.serializeAxiosError)(error));
|
|
438
449
|
return {
|
|
439
450
|
error: {
|
|
440
451
|
message: error.message,
|
|
441
452
|
},
|
|
442
453
|
};
|
|
443
454
|
}
|
|
444
|
-
console.
|
|
455
|
+
console.warn('Failed to create sync mapper record', error);
|
|
445
456
|
return {
|
|
446
457
|
error: {
|
|
447
458
|
message: 'Failed to create sync mapper record' + error,
|
|
@@ -457,7 +468,7 @@ class WorkerAdapter {
|
|
|
457
468
|
};
|
|
458
469
|
}
|
|
459
470
|
else {
|
|
460
|
-
console.
|
|
471
|
+
console.warn('Failed to create item in external system', error);
|
|
461
472
|
return {
|
|
462
473
|
report: {
|
|
463
474
|
item_type: itemTypeToLoad.itemType,
|
|
@@ -467,7 +478,7 @@ class WorkerAdapter {
|
|
|
467
478
|
}
|
|
468
479
|
}
|
|
469
480
|
else {
|
|
470
|
-
console.
|
|
481
|
+
console.warn('Failed to get sync mapper record', (0, logger_1.serializeAxiosError)(error));
|
|
471
482
|
return {
|
|
472
483
|
error: {
|
|
473
484
|
message: error.message,
|
|
@@ -475,7 +486,7 @@ class WorkerAdapter {
|
|
|
475
486
|
};
|
|
476
487
|
}
|
|
477
488
|
}
|
|
478
|
-
console.
|
|
489
|
+
console.warn('Failed to get sync mapper record', error);
|
|
479
490
|
return {
|
|
480
491
|
error: {
|
|
481
492
|
message: 'Failed to get sync mapper record' + error,
|
|
@@ -483,6 +494,49 @@ class WorkerAdapter {
|
|
|
483
494
|
};
|
|
484
495
|
}
|
|
485
496
|
}
|
|
497
|
+
async processAttachment(attachment, stream) {
|
|
498
|
+
var _a, _b;
|
|
499
|
+
const { httpStream, delay, error } = await stream({
|
|
500
|
+
item: attachment,
|
|
501
|
+
event: this.event,
|
|
502
|
+
});
|
|
503
|
+
if (error) {
|
|
504
|
+
console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
|
|
505
|
+
return { error };
|
|
506
|
+
}
|
|
507
|
+
else if (delay) {
|
|
508
|
+
return { delay };
|
|
509
|
+
}
|
|
510
|
+
if (httpStream) {
|
|
511
|
+
const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
|
|
512
|
+
const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
|
|
513
|
+
if (!preparedArtifact) {
|
|
514
|
+
console.warn('Error while preparing artifact for attachment ID ' +
|
|
515
|
+
attachment.id +
|
|
516
|
+
'. Skipping attachment');
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
|
|
520
|
+
if (!uploadedArtifact) {
|
|
521
|
+
console.warn('Error while preparing artifact for attachment ID ' + attachment.id);
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
const ssorAttachment = {
|
|
525
|
+
id: {
|
|
526
|
+
devrev: preparedArtifact.id,
|
|
527
|
+
external: attachment.id,
|
|
528
|
+
},
|
|
529
|
+
parent_id: {
|
|
530
|
+
external: attachment.parent_id,
|
|
531
|
+
},
|
|
532
|
+
actor_id: {
|
|
533
|
+
external: attachment.author_id,
|
|
534
|
+
},
|
|
535
|
+
};
|
|
536
|
+
await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
|
|
537
|
+
}
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
486
540
|
async loadAttachment({ item, create, }) {
|
|
487
541
|
// Create item
|
|
488
542
|
const { id, delay, error } = await create({
|
|
@@ -506,7 +560,7 @@ class WorkerAdapter {
|
|
|
506
560
|
};
|
|
507
561
|
}
|
|
508
562
|
else {
|
|
509
|
-
console.
|
|
563
|
+
console.warn('Failed to create attachment in external system', error);
|
|
510
564
|
return {
|
|
511
565
|
report: {
|
|
512
566
|
item_type: 'attachment',
|
|
@@ -518,90 +572,66 @@ class WorkerAdapter {
|
|
|
518
572
|
/**
|
|
519
573
|
* Streams the attachments to the DevRev platform.
|
|
520
574
|
* The attachments are streamed to the platform and the artifact information is returned.
|
|
521
|
-
* @param {
|
|
522
|
-
* @returns {Promise<
|
|
575
|
+
* @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors }} Params - The parameters to stream the attachments
|
|
576
|
+
* @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
|
|
523
577
|
* or error information if there was an error
|
|
524
578
|
*/
|
|
525
|
-
async streamAttachments({ stream, }) {
|
|
526
|
-
var _a, _b, _c
|
|
579
|
+
async streamAttachments({ stream, processors, }) {
|
|
580
|
+
var _a, _b, _c;
|
|
527
581
|
const repos = [
|
|
528
582
|
{
|
|
529
583
|
itemType: 'ssor_attachment',
|
|
530
584
|
},
|
|
531
585
|
];
|
|
532
586
|
this.initializeRepos(repos);
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
}
|
|
537
|
-
console.log('Started streaming attachments to the platform.');
|
|
587
|
+
const attachmentsState = (((_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) || []).slice();
|
|
588
|
+
console.log('Attachments metadata artifact IDs', attachmentsState);
|
|
589
|
+
for (const attachmentsMetadataArtifactId of attachmentsState) {
|
|
590
|
+
console.log(`Started processing attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
538
591
|
const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
|
|
539
592
|
artifact: attachmentsMetadataArtifactId,
|
|
540
593
|
});
|
|
541
594
|
if (error) {
|
|
595
|
+
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
542
596
|
return { error };
|
|
543
597
|
}
|
|
544
|
-
if (attachments) {
|
|
545
|
-
|
|
598
|
+
if (!attachments || attachments.length === 0) {
|
|
599
|
+
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
if (processors) {
|
|
603
|
+
console.log(`Using custom processors for attachments.`);
|
|
604
|
+
const { reducer, iterator } = processors;
|
|
605
|
+
const reducedAttachments = reducer({ attachments, adapter: this });
|
|
606
|
+
const response = await iterator({
|
|
607
|
+
reducedAttachments,
|
|
608
|
+
adapter: this,
|
|
609
|
+
stream,
|
|
610
|
+
});
|
|
611
|
+
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
612
|
+
return response;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
else {
|
|
616
|
+
console.log(`Using default processors for attachments.`);
|
|
617
|
+
const attachmentsToProcess = attachments.slice((_c = (_b = this.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessed, attachments.length);
|
|
546
618
|
for (const attachment of attachmentsToProcess) {
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
});
|
|
551
|
-
if (error) {
|
|
552
|
-
console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
|
|
553
|
-
continue;
|
|
619
|
+
const response = await this.processAttachment(attachment, stream);
|
|
620
|
+
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
621
|
+
return response;
|
|
554
622
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
}
|
|
558
|
-
if (httpStream) {
|
|
559
|
-
const fileType = ((_e = httpStream.headers) === null || _e === void 0 ? void 0 : _e['content-type']) ||
|
|
560
|
-
'application/octet-stream';
|
|
561
|
-
const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
|
|
562
|
-
if (!preparedArtifact) {
|
|
563
|
-
console.warn('Error while preparing artifact for attachment ID ' +
|
|
564
|
-
attachment.id +
|
|
565
|
-
'. Skipping attachment');
|
|
566
|
-
if (this.state.toDevRev) {
|
|
567
|
-
this.state.toDevRev.attachmentsMetadata.lastProcessed++;
|
|
568
|
-
}
|
|
569
|
-
continue;
|
|
570
|
-
}
|
|
571
|
-
const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
|
|
572
|
-
if (!uploadedArtifact) {
|
|
573
|
-
console.warn('Error while preparing artifact for attachment ID ' +
|
|
574
|
-
attachment.id);
|
|
575
|
-
if (this.state.toDevRev) {
|
|
576
|
-
this.state.toDevRev.attachmentsMetadata.lastProcessed++;
|
|
577
|
-
}
|
|
578
|
-
continue;
|
|
579
|
-
}
|
|
580
|
-
const ssorAttachment = {
|
|
581
|
-
id: {
|
|
582
|
-
devrev: preparedArtifact.id,
|
|
583
|
-
external: attachment.id,
|
|
584
|
-
},
|
|
585
|
-
parent_id: {
|
|
586
|
-
external: attachment.parent_id,
|
|
587
|
-
},
|
|
588
|
-
actor_id: {
|
|
589
|
-
external: attachment.author_id,
|
|
590
|
-
},
|
|
591
|
-
};
|
|
592
|
-
await ((_f = this.getRepo('ssor_attachment')) === null || _f === void 0 ? void 0 : _f.push([ssorAttachment]));
|
|
593
|
-
if (this.state.toDevRev) {
|
|
594
|
-
this.state.toDevRev.attachmentsMetadata.lastProcessed++;
|
|
595
|
-
}
|
|
623
|
+
if (this.state.toDevRev) {
|
|
624
|
+
this.state.toDevRev.attachmentsMetadata.lastProcessed += 1;
|
|
596
625
|
}
|
|
597
626
|
}
|
|
598
627
|
}
|
|
599
628
|
if (this.state.toDevRev) {
|
|
629
|
+
console.log(`Finished processing attachments for artifact ID. Setting last processed to 0 and removing artifact ID from state.`);
|
|
600
630
|
this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
|
|
601
631
|
this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
|
|
602
632
|
}
|
|
603
633
|
}
|
|
604
|
-
return
|
|
634
|
+
return;
|
|
605
635
|
}
|
|
606
636
|
}
|
|
607
637
|
exports.WorkerAdapter = WorkerAdapter;
|
package/package.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const process_task_1 = require("workers/process-task");
|
|
4
|
-
const loading_1 = require("../../types/loading");
|
|
5
|
-
(0, process_task_1.processTask)({
|
|
6
|
-
task: async ({ adapter }) => {
|
|
7
|
-
await adapter.emit(loading_1.LoaderEventType.DataLoadingDone, {
|
|
8
|
-
reports: adapter.reports,
|
|
9
|
-
processed_files: adapter.processedFiles,
|
|
10
|
-
});
|
|
11
|
-
},
|
|
12
|
-
onTimeout: async ({ adapter }) => {
|
|
13
|
-
await adapter.postState();
|
|
14
|
-
await adapter.emit(loading_1.LoaderEventType.DataLoadingProgress, {
|
|
15
|
-
reports: adapter.reports,
|
|
16
|
-
processed_files: adapter.processedFiles,
|
|
17
|
-
});
|
|
18
|
-
},
|
|
19
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const index_1 = require("../../index");
|
|
4
|
-
(0, index_1.processTask)({
|
|
5
|
-
task: async ({ adapter }) => {
|
|
6
|
-
await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionDone);
|
|
7
|
-
},
|
|
8
|
-
onTimeout: async ({ adapter }) => {
|
|
9
|
-
await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionError, {
|
|
10
|
-
error: {
|
|
11
|
-
message: 'Failed to delete data. Lambda timeout.',
|
|
12
|
-
},
|
|
13
|
-
});
|
|
14
|
-
},
|
|
15
|
-
});
|