@devrev/ts-adaas 1.18.0 → 1.18.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/install-initial-domain-mapping.test.js +5 -2
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +109 -0
- package/dist/index.d.ts +9 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -10
- package/dist/logger/logger.test.js +4 -19
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.js +1 -1
- package/dist/multithreading/create-worker.test.js +12 -11
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +7 -3
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +102 -64
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +274 -7
- package/dist/repo/repo.test.js +17 -5
- package/dist/state/state.test.js +187 -156
- package/dist/types/external-domain-metadata.d.ts +20 -1
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.test.js +17 -10
- package/dist/uploader/uploader.test.js +3 -1
- package/package.json +1 -1
|
@@ -286,57 +286,74 @@ class WorkerAdapter {
|
|
|
286
286
|
};
|
|
287
287
|
}
|
|
288
288
|
console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
289
|
-
|
|
290
|
-
.
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
error: {
|
|
296
|
-
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
297
|
-
},
|
|
298
|
-
});
|
|
299
|
-
break;
|
|
300
|
-
}
|
|
301
|
-
if (!fileToLoad.completed) {
|
|
302
|
-
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
303
|
-
artifactId: fileToLoad.id,
|
|
304
|
-
isGzipped: true,
|
|
305
|
-
});
|
|
306
|
-
if (transformerFileError) {
|
|
307
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
289
|
+
try {
|
|
290
|
+
outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
|
|
291
|
+
.filesToLoad) {
|
|
292
|
+
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
293
|
+
if (!itemTypeToLoad) {
|
|
294
|
+
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
308
295
|
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
309
296
|
error: {
|
|
310
|
-
message: `
|
|
297
|
+
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
311
298
|
},
|
|
312
299
|
});
|
|
300
|
+
break;
|
|
313
301
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
itemTypeToLoad,
|
|
302
|
+
if (!fileToLoad.completed) {
|
|
303
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
304
|
+
artifactId: fileToLoad.id,
|
|
305
|
+
isGzipped: true,
|
|
319
306
|
});
|
|
320
|
-
if (
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
307
|
+
if (transformerFileError) {
|
|
308
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
309
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
310
|
+
error: {
|
|
311
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
312
|
+
},
|
|
325
313
|
});
|
|
326
314
|
break outerloop;
|
|
327
315
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
316
|
+
const transformerFile = response;
|
|
317
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
318
|
+
if (this.isTimeout) {
|
|
319
|
+
console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
|
|
320
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
|
|
321
|
+
process.exit(0);
|
|
322
|
+
}
|
|
323
|
+
const { report, rateLimit } = await this.loadItem({
|
|
324
|
+
item: transformerFile[i],
|
|
325
|
+
itemTypeToLoad,
|
|
332
326
|
});
|
|
333
|
-
|
|
327
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
328
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
329
|
+
delay: rateLimit.delay,
|
|
330
|
+
reports: this.reports,
|
|
331
|
+
processed_files: this.processedFiles,
|
|
332
|
+
});
|
|
333
|
+
break outerloop;
|
|
334
|
+
}
|
|
335
|
+
if (report) {
|
|
336
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
337
|
+
loaderReports: this.loaderReports,
|
|
338
|
+
report,
|
|
339
|
+
});
|
|
340
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
341
|
+
}
|
|
334
342
|
}
|
|
343
|
+
fileToLoad.completed = true;
|
|
344
|
+
this._processedFiles.push(fileToLoad.id);
|
|
335
345
|
}
|
|
336
|
-
fileToLoad.completed = true;
|
|
337
|
-
this._processedFiles.push(fileToLoad.id);
|
|
338
346
|
}
|
|
339
347
|
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
console.error('Error during data loading.', (0, logger_1.serializeError)(error));
|
|
350
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
351
|
+
error: {
|
|
352
|
+
message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
|
|
353
|
+
},
|
|
354
|
+
});
|
|
355
|
+
process.exit(1);
|
|
356
|
+
}
|
|
340
357
|
return {
|
|
341
358
|
reports: this.reports,
|
|
342
359
|
processed_files: this.processedFiles,
|
|
@@ -383,42 +400,63 @@ class WorkerAdapter {
|
|
|
383
400
|
};
|
|
384
401
|
}
|
|
385
402
|
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
const transformerFile = response;
|
|
393
|
-
if (transformerFileError) {
|
|
394
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
395
|
-
break outerloop;
|
|
396
|
-
}
|
|
397
|
-
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
398
|
-
const { report, rateLimit } = await this.loadAttachment({
|
|
399
|
-
item: transformerFile[i],
|
|
400
|
-
create,
|
|
403
|
+
try {
|
|
404
|
+
outerloop: for (const fileToLoad of filesToLoad) {
|
|
405
|
+
if (!fileToLoad.completed) {
|
|
406
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
407
|
+
artifactId: fileToLoad.id,
|
|
408
|
+
isGzipped: true,
|
|
401
409
|
});
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
410
|
+
const transformerFile = response;
|
|
411
|
+
if (transformerFileError) {
|
|
412
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
413
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
414
|
+
error: {
|
|
415
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
416
|
+
},
|
|
407
417
|
});
|
|
408
418
|
break outerloop;
|
|
409
419
|
}
|
|
410
|
-
|
|
411
|
-
(
|
|
412
|
-
|
|
413
|
-
|
|
420
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
421
|
+
if (this.isTimeout) {
|
|
422
|
+
console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
|
|
423
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
|
|
424
|
+
process.exit(0);
|
|
425
|
+
}
|
|
426
|
+
const { report, rateLimit } = await this.loadAttachment({
|
|
427
|
+
item: transformerFile[i],
|
|
428
|
+
create,
|
|
414
429
|
});
|
|
415
|
-
|
|
430
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
431
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
432
|
+
delay: rateLimit.delay,
|
|
433
|
+
reports: this.reports,
|
|
434
|
+
processed_files: this.processedFiles,
|
|
435
|
+
});
|
|
436
|
+
break outerloop;
|
|
437
|
+
}
|
|
438
|
+
if (report) {
|
|
439
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
440
|
+
loaderReports: this.loaderReports,
|
|
441
|
+
report,
|
|
442
|
+
});
|
|
443
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
444
|
+
}
|
|
416
445
|
}
|
|
446
|
+
fileToLoad.completed = true;
|
|
447
|
+
this._processedFiles.push(fileToLoad.id);
|
|
417
448
|
}
|
|
418
|
-
fileToLoad.completed = true;
|
|
419
|
-
this._processedFiles.push(fileToLoad.id);
|
|
420
449
|
}
|
|
421
450
|
}
|
|
451
|
+
catch (error) {
|
|
452
|
+
console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
|
|
453
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
454
|
+
error: {
|
|
455
|
+
message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
|
|
456
|
+
},
|
|
457
|
+
});
|
|
458
|
+
process.exit(1);
|
|
459
|
+
}
|
|
422
460
|
return {
|
|
423
461
|
reports: this.reports,
|
|
424
462
|
processed_files: this.processedFiles,
|
|
@@ -3,7 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
|
|
4
4
|
const constants_1 = require("../../common/constants");
|
|
5
5
|
const state_1 = require("../../state/state");
|
|
6
|
-
const
|
|
6
|
+
const jest_setup_1 = require("../../tests/jest.setup");
|
|
7
|
+
const test_utils_1 = require("../../common/test-utils");
|
|
7
8
|
const types_1 = require("../../types");
|
|
8
9
|
const loading_1 = require("../../types/loading");
|
|
9
10
|
const worker_adapter_1 = require("./worker-adapter");
|
|
@@ -40,7 +41,9 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
40
41
|
// Reset all mocks
|
|
41
42
|
jest.clearAllMocks();
|
|
42
43
|
// Create mock objects
|
|
43
|
-
mockEvent = (0,
|
|
44
|
+
mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
45
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
46
|
+
});
|
|
44
47
|
const initialState = {
|
|
45
48
|
attachments: { completed: false },
|
|
46
49
|
lastSyncStarted: '',
|
|
@@ -385,7 +388,7 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
385
388
|
// Mock the pool to simulate timeout happening during the first artifact
|
|
386
389
|
attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
|
|
387
390
|
return {
|
|
388
|
-
streamAll: jest.fn().mockImplementation(
|
|
391
|
+
streamAll: jest.fn().mockImplementation(() => {
|
|
389
392
|
adapter.isTimeout = true;
|
|
390
393
|
return {};
|
|
391
394
|
}),
|
|
@@ -661,7 +664,9 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
661
664
|
.fn()
|
|
662
665
|
.mockResolvedValue(undefined);
|
|
663
666
|
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
664
|
-
adapter['loaderReports'] = [
|
|
667
|
+
adapter['loaderReports'] = [
|
|
668
|
+
{ item_type: 'tasks', [loading_1.ActionType.CREATED]: 5 },
|
|
669
|
+
];
|
|
665
670
|
adapter['_processedFiles'] = ['file-1', 'file-2'];
|
|
666
671
|
await adapter.emit(types_1.LoaderEventType.DataLoadingDone);
|
|
667
672
|
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
@@ -685,7 +690,9 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
685
690
|
adapter['_artifacts'] = [
|
|
686
691
|
{ id: 'art-1', item_count: 10, item_type: 'issues' },
|
|
687
692
|
];
|
|
688
|
-
adapter['loaderReports'] = [
|
|
693
|
+
adapter['loaderReports'] = [
|
|
694
|
+
{ item_type: 'tasks', [loading_1.ActionType.CREATED]: 5 },
|
|
695
|
+
];
|
|
689
696
|
adapter['_processedFiles'] = ['file-1'];
|
|
690
697
|
await adapter.emit('SOME_UNKNOWN_EVENT');
|
|
691
698
|
const callData = mockEmit.mock.calls[0][0].data;
|
|
@@ -770,8 +777,7 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
770
777
|
async function emitDone(adapterInstance, extractionStart, extractionEnd) {
|
|
771
778
|
adapterInstance.event.payload.event_context.extract_from =
|
|
772
779
|
extractionStart;
|
|
773
|
-
adapterInstance.event.payload.event_context.extract_to =
|
|
774
|
-
extractionEnd;
|
|
780
|
+
adapterInstance.event.payload.event_context.extract_to = extractionEnd;
|
|
775
781
|
// Reset the emit guard so we can emit multiple times in a single test
|
|
776
782
|
adapterInstance['hasWorkerEmitted'] = false;
|
|
777
783
|
await adapterInstance.emit(types_1.ExtractorEventType.AttachmentExtractionDone, {
|
|
@@ -973,4 +979,265 @@ describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
|
973
979
|
expect(adapter.shouldExtract('users')).toBe(true);
|
|
974
980
|
});
|
|
975
981
|
});
|
|
982
|
+
describe(worker_adapter_1.WorkerAdapter.prototype.loadItemTypes.name, () => {
|
|
983
|
+
let exitSpy;
|
|
984
|
+
let emitSpy;
|
|
985
|
+
beforeEach(() => {
|
|
986
|
+
exitSpy = jest
|
|
987
|
+
.spyOn(process, 'exit')
|
|
988
|
+
.mockImplementation(() => undefined);
|
|
989
|
+
emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
|
|
990
|
+
// Set event type to loading continuation (not StartLoadingData) so we can
|
|
991
|
+
// set fromDevRev state directly without mocking getLoaderBatches
|
|
992
|
+
mockEvent.payload.event_type =
|
|
993
|
+
types_1.EventType.ContinueLoadingData;
|
|
994
|
+
});
|
|
995
|
+
afterEach(() => {
|
|
996
|
+
exitSpy.mockRestore();
|
|
997
|
+
});
|
|
998
|
+
function setupFilesToLoad(items) {
|
|
999
|
+
adapter['adapterState'].state.fromDevRev = {
|
|
1000
|
+
filesToLoad: [
|
|
1001
|
+
{
|
|
1002
|
+
id: 'artifact-1',
|
|
1003
|
+
file_name: 'file1.json',
|
|
1004
|
+
itemType: 'tasks',
|
|
1005
|
+
count: items.length,
|
|
1006
|
+
lineToProcess: 0,
|
|
1007
|
+
completed: false,
|
|
1008
|
+
},
|
|
1009
|
+
],
|
|
1010
|
+
};
|
|
1011
|
+
adapter['uploader'].getJsonObjectByArtifactId = jest
|
|
1012
|
+
.fn()
|
|
1013
|
+
.mockResolvedValue({ response: items });
|
|
1014
|
+
}
|
|
1015
|
+
it('should emit DataLoadingProgress and exit on timeout', async () => {
|
|
1016
|
+
const items = [
|
|
1017
|
+
{
|
|
1018
|
+
id: { devrev: 'dev-1', external: 'ext-1' },
|
|
1019
|
+
created_date: '',
|
|
1020
|
+
modified_date: '',
|
|
1021
|
+
data: {},
|
|
1022
|
+
},
|
|
1023
|
+
{
|
|
1024
|
+
id: { devrev: 'dev-2', external: 'ext-2' },
|
|
1025
|
+
created_date: '',
|
|
1026
|
+
modified_date: '',
|
|
1027
|
+
data: {},
|
|
1028
|
+
},
|
|
1029
|
+
];
|
|
1030
|
+
setupFilesToLoad(items);
|
|
1031
|
+
// Set timeout before calling loadItemTypes
|
|
1032
|
+
adapter.isTimeout = true;
|
|
1033
|
+
const itemTypesToLoad = [
|
|
1034
|
+
{
|
|
1035
|
+
itemType: 'tasks',
|
|
1036
|
+
create: jest.fn(),
|
|
1037
|
+
update: jest.fn(),
|
|
1038
|
+
},
|
|
1039
|
+
];
|
|
1040
|
+
await adapter.loadItemTypes({ itemTypesToLoad });
|
|
1041
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.DataLoadingProgress);
|
|
1042
|
+
expect(exitSpy).toHaveBeenCalledWith(0);
|
|
1043
|
+
});
|
|
1044
|
+
it('should emit DataLoadingProgress mid-loop when timeout arrives between items', async () => {
|
|
1045
|
+
const items = [
|
|
1046
|
+
{
|
|
1047
|
+
id: { devrev: 'dev-1', external: 'ext-1' },
|
|
1048
|
+
created_date: '',
|
|
1049
|
+
modified_date: '',
|
|
1050
|
+
data: {},
|
|
1051
|
+
},
|
|
1052
|
+
{
|
|
1053
|
+
id: { devrev: 'dev-2', external: 'ext-2' },
|
|
1054
|
+
created_date: '',
|
|
1055
|
+
modified_date: '',
|
|
1056
|
+
data: {},
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
id: { devrev: 'dev-3', external: 'ext-3' },
|
|
1060
|
+
created_date: '',
|
|
1061
|
+
modified_date: '',
|
|
1062
|
+
data: {},
|
|
1063
|
+
},
|
|
1064
|
+
];
|
|
1065
|
+
setupFilesToLoad(items);
|
|
1066
|
+
// Mock process.exit to throw so it stops execution like a real exit would
|
|
1067
|
+
exitSpy.mockRestore();
|
|
1068
|
+
exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => {
|
|
1069
|
+
throw new Error(`process.exit`);
|
|
1070
|
+
}));
|
|
1071
|
+
let loadItemCallCount = 0;
|
|
1072
|
+
// Mock loadItem to set timeout after the first call
|
|
1073
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/require-await
|
|
1074
|
+
jest.spyOn(adapter, 'loadItem').mockImplementation(async () => {
|
|
1075
|
+
loadItemCallCount++;
|
|
1076
|
+
if (loadItemCallCount === 1) {
|
|
1077
|
+
adapter.isTimeout = true;
|
|
1078
|
+
}
|
|
1079
|
+
return { report: { item_type: 'tasks', updated: 1 } };
|
|
1080
|
+
});
|
|
1081
|
+
const itemTypesToLoad = [
|
|
1082
|
+
{
|
|
1083
|
+
itemType: 'tasks',
|
|
1084
|
+
create: jest.fn(),
|
|
1085
|
+
update: jest.fn(),
|
|
1086
|
+
},
|
|
1087
|
+
];
|
|
1088
|
+
// process.exit throws, so this will throw
|
|
1089
|
+
await expect(adapter.loadItemTypes({ itemTypesToLoad })).rejects.toThrow('process.exit');
|
|
1090
|
+
// First item processed, then timeout detected on second iteration
|
|
1091
|
+
expect(loadItemCallCount).toBe(1);
|
|
1092
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.DataLoadingProgress);
|
|
1093
|
+
});
|
|
1094
|
+
it('should emit DataLoadingError and exit(1) on unexpected error', async () => {
|
|
1095
|
+
adapter['adapterState'].state.fromDevRev = {
|
|
1096
|
+
filesToLoad: [
|
|
1097
|
+
{
|
|
1098
|
+
id: 'artifact-1',
|
|
1099
|
+
file_name: 'file1.json',
|
|
1100
|
+
itemType: 'tasks',
|
|
1101
|
+
count: 1,
|
|
1102
|
+
lineToProcess: 0,
|
|
1103
|
+
completed: false,
|
|
1104
|
+
},
|
|
1105
|
+
],
|
|
1106
|
+
};
|
|
1107
|
+
// Make getJsonObjectByArtifactId throw (not return error — throw)
|
|
1108
|
+
adapter['uploader'].getJsonObjectByArtifactId = jest
|
|
1109
|
+
.fn()
|
|
1110
|
+
.mockRejectedValue(new Error('Unexpected network failure'));
|
|
1111
|
+
const itemTypesToLoad = [
|
|
1112
|
+
{
|
|
1113
|
+
itemType: 'tasks',
|
|
1114
|
+
create: jest.fn(),
|
|
1115
|
+
update: jest.fn(),
|
|
1116
|
+
},
|
|
1117
|
+
];
|
|
1118
|
+
await adapter.loadItemTypes({ itemTypesToLoad });
|
|
1119
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.DataLoadingError, expect.objectContaining({
|
|
1120
|
+
error: expect.objectContaining({
|
|
1121
|
+
message: expect.stringContaining('Error during data loading'),
|
|
1122
|
+
}),
|
|
1123
|
+
}));
|
|
1124
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
1125
|
+
});
|
|
1126
|
+
});
|
|
1127
|
+
describe(worker_adapter_1.WorkerAdapter.prototype.loadAttachments.name, () => {
|
|
1128
|
+
let exitSpy;
|
|
1129
|
+
let emitSpy;
|
|
1130
|
+
beforeEach(() => {
|
|
1131
|
+
exitSpy = jest
|
|
1132
|
+
.spyOn(process, 'exit')
|
|
1133
|
+
.mockImplementation(() => undefined);
|
|
1134
|
+
emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
|
|
1135
|
+
// Set event type to continuation so we can set fromDevRev state directly
|
|
1136
|
+
mockEvent.payload.event_type =
|
|
1137
|
+
types_1.EventType.ContinueLoadingAttachments;
|
|
1138
|
+
});
|
|
1139
|
+
afterEach(() => {
|
|
1140
|
+
exitSpy.mockRestore();
|
|
1141
|
+
});
|
|
1142
|
+
function setupFilesToLoad(items) {
|
|
1143
|
+
adapter['adapterState'].state.fromDevRev = {
|
|
1144
|
+
filesToLoad: [
|
|
1145
|
+
{
|
|
1146
|
+
id: 'artifact-1',
|
|
1147
|
+
file_name: 'attachments.json',
|
|
1148
|
+
itemType: 'attachment',
|
|
1149
|
+
count: items.length,
|
|
1150
|
+
lineToProcess: 0,
|
|
1151
|
+
completed: false,
|
|
1152
|
+
},
|
|
1153
|
+
],
|
|
1154
|
+
};
|
|
1155
|
+
adapter['uploader'].getJsonObjectByArtifactId = jest
|
|
1156
|
+
.fn()
|
|
1157
|
+
.mockResolvedValue({ response: items });
|
|
1158
|
+
}
|
|
1159
|
+
it('should emit AttachmentLoadingProgress and exit on timeout', async () => {
|
|
1160
|
+
const items = [
|
|
1161
|
+
{
|
|
1162
|
+
reference_id: 'ref-1',
|
|
1163
|
+
parent_type: 'task',
|
|
1164
|
+
parent_reference_id: 'parent-1',
|
|
1165
|
+
file_name: 'file.pdf',
|
|
1166
|
+
file_type: 'application/pdf',
|
|
1167
|
+
file_size: 100,
|
|
1168
|
+
url: 'https://example.com/file.pdf',
|
|
1169
|
+
valid_until: '',
|
|
1170
|
+
created_by_id: 'user-1',
|
|
1171
|
+
created_date: '',
|
|
1172
|
+
modified_by_id: 'user-1',
|
|
1173
|
+
modified_date: '',
|
|
1174
|
+
},
|
|
1175
|
+
];
|
|
1176
|
+
setupFilesToLoad(items);
|
|
1177
|
+
adapter.isTimeout = true;
|
|
1178
|
+
await adapter.loadAttachments({
|
|
1179
|
+
create: jest.fn(),
|
|
1180
|
+
});
|
|
1181
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.AttachmentLoadingProgress);
|
|
1182
|
+
expect(exitSpy).toHaveBeenCalledWith(0);
|
|
1183
|
+
});
|
|
1184
|
+
it('should emit AttachmentLoadingError on transformer file error', async () => {
|
|
1185
|
+
adapter['adapterState'].state.fromDevRev = {
|
|
1186
|
+
filesToLoad: [
|
|
1187
|
+
{
|
|
1188
|
+
id: 'bad-artifact',
|
|
1189
|
+
file_name: 'attachments.json',
|
|
1190
|
+
itemType: 'attachment',
|
|
1191
|
+
count: 1,
|
|
1192
|
+
lineToProcess: 0,
|
|
1193
|
+
completed: false,
|
|
1194
|
+
},
|
|
1195
|
+
],
|
|
1196
|
+
};
|
|
1197
|
+
adapter['uploader'].getJsonObjectByArtifactId = jest
|
|
1198
|
+
.fn()
|
|
1199
|
+
.mockResolvedValue({
|
|
1200
|
+
response: null,
|
|
1201
|
+
error: new Error('Artifact not found'),
|
|
1202
|
+
});
|
|
1203
|
+
await adapter.loadAttachments({
|
|
1204
|
+
create: jest.fn(),
|
|
1205
|
+
});
|
|
1206
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.AttachmentLoadingError, expect.objectContaining({
|
|
1207
|
+
error: expect.objectContaining({
|
|
1208
|
+
message: expect.stringContaining('Transformer file not found'),
|
|
1209
|
+
}),
|
|
1210
|
+
}));
|
|
1211
|
+
});
|
|
1212
|
+
it('should emit AttachmentLoadingError and exit(1) on unexpected error', async () => {
|
|
1213
|
+
const items = [
|
|
1214
|
+
{
|
|
1215
|
+
reference_id: 'ref-1',
|
|
1216
|
+
parent_type: 'task',
|
|
1217
|
+
parent_reference_id: 'parent-1',
|
|
1218
|
+
file_name: 'file.pdf',
|
|
1219
|
+
file_type: 'application/pdf',
|
|
1220
|
+
file_size: 100,
|
|
1221
|
+
url: 'https://example.com/file.pdf',
|
|
1222
|
+
valid_until: '',
|
|
1223
|
+
created_by_id: 'user-1',
|
|
1224
|
+
created_date: '',
|
|
1225
|
+
modified_by_id: 'user-1',
|
|
1226
|
+
modified_date: '',
|
|
1227
|
+
},
|
|
1228
|
+
];
|
|
1229
|
+
setupFilesToLoad(items);
|
|
1230
|
+
// Make the create function throw
|
|
1231
|
+
const mockCreate = jest
|
|
1232
|
+
.fn()
|
|
1233
|
+
.mockRejectedValue(new Error('Unexpected API failure'));
|
|
1234
|
+
await adapter.loadAttachments({ create: mockCreate });
|
|
1235
|
+
expect(emitSpy).toHaveBeenCalledWith(types_1.LoaderEventType.AttachmentLoadingError, expect.objectContaining({
|
|
1236
|
+
error: expect.objectContaining({
|
|
1237
|
+
message: expect.stringContaining('Error during attachment loading'),
|
|
1238
|
+
}),
|
|
1239
|
+
}));
|
|
1240
|
+
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
1241
|
+
});
|
|
1242
|
+
});
|
|
976
1243
|
});
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const constants_1 = require("../common/constants");
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
+
const jest_setup_1 = require("../tests/jest.setup");
|
|
6
|
+
const test_utils_1 = require("../common/test-utils");
|
|
5
7
|
const types_1 = require("../types");
|
|
6
8
|
const repo_1 = require("./repo");
|
|
7
9
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
@@ -11,7 +13,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
11
13
|
beforeEach(() => {
|
|
12
14
|
normalize = jest.fn();
|
|
13
15
|
repo = new repo_1.Repo({
|
|
14
|
-
event: (0,
|
|
16
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
17
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
18
|
+
}),
|
|
15
19
|
itemType: 'test_item_type',
|
|
16
20
|
normalize,
|
|
17
21
|
onUpload: jest.fn(),
|
|
@@ -29,7 +33,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
29
33
|
});
|
|
30
34
|
it('should not normalize items when normalize function is not provided', async () => {
|
|
31
35
|
repo = new repo_1.Repo({
|
|
32
|
-
event: (0,
|
|
36
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
37
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
38
|
+
}),
|
|
33
39
|
itemType: 'test_item_type',
|
|
34
40
|
onUpload: jest.fn(),
|
|
35
41
|
options: {},
|
|
@@ -44,7 +50,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
44
50
|
});
|
|
45
51
|
it('should not normalize items when item type is external_domain_metadata', async () => {
|
|
46
52
|
repo = new repo_1.Repo({
|
|
47
|
-
event: (0,
|
|
53
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
54
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
55
|
+
}),
|
|
48
56
|
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA,
|
|
49
57
|
normalize,
|
|
50
58
|
onUpload: jest.fn(),
|
|
@@ -56,7 +64,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
56
64
|
});
|
|
57
65
|
it('should not normalize items when item type is ssor_attachment', async () => {
|
|
58
66
|
repo = new repo_1.Repo({
|
|
59
|
-
event: (0,
|
|
67
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
68
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
69
|
+
}),
|
|
60
70
|
itemType: constants_1.SSOR_ATTACHMENT,
|
|
61
71
|
normalize,
|
|
62
72
|
onUpload: jest.fn(),
|
|
@@ -91,7 +101,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
91
101
|
describe('should take batch size into account', () => {
|
|
92
102
|
beforeEach(() => {
|
|
93
103
|
repo = new repo_1.Repo({
|
|
94
|
-
event: (0,
|
|
104
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
105
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
106
|
+
}),
|
|
95
107
|
itemType: 'test_item_type',
|
|
96
108
|
normalize,
|
|
97
109
|
onUpload: jest.fn(),
|