@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
- package/dist/common/constants.d.ts +4 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +6 -7
- package/dist/common/helpers.d.ts +3 -35
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +5 -50
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/{axios-client-internal.js → client.js} +4 -8
- package/dist/index.d.ts +24 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +43 -32
- package/dist/logger/logger.d.ts +12 -53
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +36 -68
- package/dist/mappers/mappers.d.ts +7 -46
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +175 -0
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
- package/dist/mappers/mappers.js +11 -45
- package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +97 -62
- package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +203 -206
- package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +294 -346
- package/dist/multithreading/emit.d.ts +11 -0
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -43
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +128 -20
- package/dist/multithreading/spawn/spawn.js +24 -39
- package/dist/multithreading/worker.js +1 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +16 -38
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +19 -41
- package/dist/state/extraction-state.d.ts +6 -21
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +8 -25
- package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
- package/dist/state/loading-state.d.ts +3 -14
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +3 -14
- package/dist/state/state.d.ts +4 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +33 -37
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +16 -11
- package/dist/state/state.js +11 -0
- package/dist/state/time-value-resolver.d.ts +17 -0
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -41
- package/dist/testing/mock-event.d.ts +10 -0
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
- package/dist/testing/mock-server.d.ts +29 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts +48 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +21 -33
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +64 -94
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +6 -34
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +39 -94
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +8 -59
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +23 -89
- package/package.json +8 -6
- package/dist/airsync-sdk.d.ts +0 -2059
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
- package/dist/common/control-protocol.d.ts +0 -10
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts +0 -19
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts +0 -47
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts +0 -294
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts +0 -47
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
- package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
- package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/serialization.test.js +0 -71
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -78
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -271
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -10
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
|
@@ -1,403 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const constants_1 = require("../../common/constants");
|
|
4
|
-
const extraction_state_1 = require("../../state/extraction-state");
|
|
5
|
-
const loading_state_1 = require("../../state/loading-state");
|
|
6
|
-
const jest_setup_1 = require("../../tests/jest.setup");
|
|
7
|
-
const test_utils_1 = require("../../common/test-utils");
|
|
8
|
-
const types_1 = require("../../types");
|
|
9
|
-
const loading_1 = require("../../types/loading");
|
|
10
|
-
const extraction_adapter_1 = require("./extraction-adapter");
|
|
11
|
-
const loading_adapter_1 = require("./loading-adapter");
|
|
12
|
-
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
13
|
-
jest.mock('../../common/control-protocol', () => ({
|
|
14
|
-
emit: jest.fn().mockResolvedValue({}),
|
|
15
|
-
}));
|
|
16
|
-
jest.mock('../../mappers/mappers');
|
|
17
|
-
jest.mock('../../uploader/uploader');
|
|
18
|
-
jest.mock('../../repo/repo');
|
|
19
|
-
jest.mock('node:worker_threads', () => ({
|
|
20
|
-
parentPort: { postMessage: jest.fn() },
|
|
21
|
-
}));
|
|
22
|
-
jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
|
|
23
|
-
AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
|
|
24
|
-
streamAll: jest.fn().mockResolvedValue(undefined),
|
|
25
|
-
})),
|
|
26
|
-
}));
|
|
27
|
-
function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
|
|
28
|
-
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
29
|
-
payload: { event_type: eventType },
|
|
30
|
-
});
|
|
31
|
-
const initialState = {
|
|
32
|
-
attachments: { completed: false },
|
|
33
|
-
};
|
|
34
|
-
const adapterState = new extraction_state_1.ExtractionState({ event, initialState });
|
|
35
|
-
adapterState.sdkState.snapInVersionId = '';
|
|
36
|
-
adapterState.sdkState.toDevRev = {
|
|
37
|
-
attachmentsMetadata: {
|
|
38
|
-
artifactIds: [],
|
|
39
|
-
lastProcessed: 0,
|
|
40
|
-
lastProcessedAttachmentsIdsList: [],
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
const adapter = new extraction_adapter_1.ExtractionAdapter({ event, adapterState });
|
|
44
|
-
return { adapter, event, adapterState };
|
|
45
|
-
}
|
|
46
|
-
function makeLoadingAdapter(eventType = types_1.EventType.StartLoadingData) {
|
|
47
|
-
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
48
|
-
payload: { event_type: eventType },
|
|
49
|
-
});
|
|
50
|
-
const initialState = {
|
|
51
|
-
attachments: { completed: false },
|
|
52
|
-
};
|
|
53
|
-
const adapterState = new loading_state_1.LoadingState({ event, initialState });
|
|
54
|
-
adapterState.sdkState.snapInVersionId = '';
|
|
55
|
-
const adapter = new loading_adapter_1.LoadingAdapter({ event, adapterState });
|
|
56
|
-
return { adapter, event, adapterState };
|
|
57
|
-
}
|
|
58
|
-
describe(`${extraction_adapter_1.ExtractionAdapter.name}.emit`, () => {
|
|
59
|
-
let adapter;
|
|
60
|
-
let mockPostMessage;
|
|
61
|
-
beforeEach(() => {
|
|
62
|
-
jest.clearAllMocks();
|
|
63
|
-
({ adapter } = makeAdapter());
|
|
64
|
-
const workerThreads = require('node:worker_threads');
|
|
65
|
-
mockPostMessage = jest.fn();
|
|
66
|
-
if (workerThreads.parentPort) {
|
|
67
|
-
jest
|
|
68
|
-
.spyOn(workerThreads.parentPort, 'postMessage')
|
|
69
|
-
.mockImplementation(mockPostMessage);
|
|
70
|
-
}
|
|
71
|
-
else {
|
|
72
|
-
workerThreads.parentPort = { postMessage: mockPostMessage };
|
|
73
|
-
}
|
|
74
|
-
});
|
|
75
|
-
afterEach(() => {
|
|
76
|
-
jest.restoreAllMocks();
|
|
77
|
-
});
|
|
78
|
-
it('should emit only one event when multiple events of same type are sent', async () => {
|
|
79
|
-
// Arrange
|
|
80
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
81
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
82
|
-
// Act
|
|
83
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
84
|
-
reports: [],
|
|
85
|
-
processed_files: [],
|
|
86
|
-
});
|
|
87
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
88
|
-
reports: [],
|
|
89
|
-
processed_files: [],
|
|
90
|
-
});
|
|
91
|
-
// Assert
|
|
92
|
-
expect(mockPostMessage).toHaveBeenCalledTimes(1);
|
|
93
|
-
});
|
|
94
|
-
it('should emit only once even when a different event type follows', async () => {
|
|
95
|
-
// Arrange
|
|
96
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
97
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
98
|
-
// Act
|
|
99
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
100
|
-
reports: [],
|
|
101
|
-
processed_files: [],
|
|
102
|
-
});
|
|
103
|
-
await adapter.emit(types_1.ExtractorEventType.DataExtractionError, {
|
|
104
|
-
reports: [],
|
|
105
|
-
processed_files: [],
|
|
106
|
-
});
|
|
107
|
-
await adapter.emit(types_1.ExtractorEventType.AttachmentExtractionError, {
|
|
108
|
-
reports: [],
|
|
109
|
-
processed_files: [],
|
|
110
|
-
});
|
|
111
|
-
// Assert
|
|
112
|
-
expect(mockPostMessage).toHaveBeenCalledTimes(1);
|
|
113
|
-
});
|
|
114
|
-
it('should correctly emit one event even if postState errors', async () => {
|
|
115
|
-
// Arrange
|
|
116
|
-
adapter['adapterState'].postState = jest
|
|
117
|
-
.fn()
|
|
118
|
-
.mockRejectedValue(new Error('postState error'));
|
|
119
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
120
|
-
// Act
|
|
121
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
122
|
-
reports: [],
|
|
123
|
-
processed_files: [],
|
|
124
|
-
});
|
|
125
|
-
// Assert
|
|
126
|
-
expect(mockPostMessage).toHaveBeenCalledTimes(1);
|
|
127
|
-
});
|
|
128
|
-
it('should correctly emit one event even if uploadAllRepos errors', async () => {
|
|
129
|
-
// Arrange
|
|
130
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
131
|
-
adapter.uploadAllRepos = jest
|
|
132
|
-
.fn()
|
|
133
|
-
.mockRejectedValue(new Error('uploadAllRepos error'));
|
|
134
|
-
// Act
|
|
135
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
136
|
-
reports: [],
|
|
137
|
-
processed_files: [],
|
|
138
|
-
});
|
|
139
|
-
// Assert
|
|
140
|
-
expect(mockPostMessage).toHaveBeenCalledTimes(1);
|
|
141
|
-
});
|
|
142
|
-
it('should include artifacts in data for extraction events', async () => {
|
|
143
|
-
// Arrange
|
|
144
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
145
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
146
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
147
|
-
adapter['_artifacts'] = [
|
|
148
|
-
{ id: 'art-1', item_count: 10, item_type: 'issues' },
|
|
149
|
-
];
|
|
150
|
-
// Act
|
|
151
|
-
await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
|
|
152
|
-
// Assert
|
|
153
|
-
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
154
|
-
data: expect.objectContaining({
|
|
155
|
-
artifacts: expect.arrayContaining([
|
|
156
|
-
expect.objectContaining({ id: 'art-1' }),
|
|
157
|
-
]),
|
|
158
|
-
}),
|
|
159
|
-
}));
|
|
160
|
-
const callData = mockEmit.mock.calls[0][0].data;
|
|
161
|
-
expect(callData).not.toHaveProperty('reports');
|
|
162
|
-
expect(callData).not.toHaveProperty('processed_files');
|
|
163
|
-
});
|
|
164
|
-
it('should include reports and processed_files in data for loader events', async () => {
|
|
165
|
-
// Arrange
|
|
166
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
167
|
-
const { adapter: loadingAdapter } = makeLoadingAdapter();
|
|
168
|
-
loadingAdapter['adapterState'].postState = jest
|
|
169
|
-
.fn()
|
|
170
|
-
.mockResolvedValue(undefined);
|
|
171
|
-
loadingAdapter['loaderReports'] = [
|
|
172
|
-
{ item_type: 'tasks', [loading_1.ActionType.CREATED]: 5 },
|
|
173
|
-
];
|
|
174
|
-
loadingAdapter['_processedFiles'] = ['file-1', 'file-2'];
|
|
175
|
-
// Act
|
|
176
|
-
await loadingAdapter.emit(types_1.LoaderEventType.DataLoadingDone);
|
|
177
|
-
// Assert
|
|
178
|
-
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
179
|
-
data: expect.objectContaining({
|
|
180
|
-
reports: expect.arrayContaining([
|
|
181
|
-
expect.objectContaining({ item_type: 'tasks' }),
|
|
182
|
-
]),
|
|
183
|
-
processed_files: ['file-1', 'file-2'],
|
|
184
|
-
}),
|
|
185
|
-
}));
|
|
186
|
-
const callData = mockEmit.mock.calls[0][0].data;
|
|
187
|
-
expect(callData).not.toHaveProperty('artifacts');
|
|
188
|
-
});
|
|
189
|
-
it('should not include artifacts, reports, or processed_files for unknown event types', async () => {
|
|
190
|
-
// Arrange
|
|
191
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
192
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
193
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
194
|
-
adapter['_artifacts'] = [
|
|
195
|
-
{ id: 'art-1', item_count: 10, item_type: 'issues' },
|
|
196
|
-
];
|
|
197
|
-
// Act
|
|
198
|
-
await adapter.emit('SOME_UNKNOWN_EVENT');
|
|
199
|
-
// Assert
|
|
200
|
-
const callData = mockEmit.mock.calls[0][0].data;
|
|
201
|
-
expect(callData).not.toHaveProperty('artifacts');
|
|
202
|
-
expect(callData).not.toHaveProperty('reports');
|
|
203
|
-
expect(callData).not.toHaveProperty('processed_files');
|
|
204
|
-
});
|
|
205
|
-
it('should include artifacts for all ExtractorEventType values', async () => {
|
|
206
|
-
var _a, _b;
|
|
207
|
-
// Arrange
|
|
208
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
209
|
-
const extractorEvents = [
|
|
210
|
-
types_1.ExtractorEventType.DataExtractionDone,
|
|
211
|
-
types_1.ExtractorEventType.DataExtractionProgress,
|
|
212
|
-
types_1.ExtractorEventType.DataExtractionError,
|
|
213
|
-
types_1.ExtractorEventType.AttachmentExtractionDone,
|
|
214
|
-
types_1.ExtractorEventType.AttachmentExtractionProgress,
|
|
215
|
-
];
|
|
216
|
-
for (const eventType of extractorEvents) {
|
|
217
|
-
jest.clearAllMocks();
|
|
218
|
-
adapter.hasWorkerEmitted = false;
|
|
219
|
-
adapter['adapterState'].postState = jest
|
|
220
|
-
.fn()
|
|
221
|
-
.mockResolvedValue(undefined);
|
|
222
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
223
|
-
// Act
|
|
224
|
-
await adapter.emit(eventType);
|
|
225
|
-
// Assert
|
|
226
|
-
const callData = (_b = (_a = mockEmit.mock.calls[0]) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.data;
|
|
227
|
-
expect(callData).toHaveProperty('artifacts');
|
|
228
|
-
expect(callData).not.toHaveProperty('reports');
|
|
229
|
-
}
|
|
230
|
-
});
|
|
231
|
-
it('should include reports and processed_files for all LoaderEventType values', async () => {
|
|
232
|
-
var _a, _b;
|
|
233
|
-
// Arrange
|
|
234
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
235
|
-
const loaderEvents = [
|
|
236
|
-
types_1.LoaderEventType.DataLoadingDone,
|
|
237
|
-
types_1.LoaderEventType.DataLoadingProgress,
|
|
238
|
-
types_1.LoaderEventType.DataLoadingError,
|
|
239
|
-
types_1.LoaderEventType.AttachmentLoadingDone,
|
|
240
|
-
types_1.LoaderEventType.AttachmentLoadingProgress,
|
|
241
|
-
];
|
|
242
|
-
const { adapter: loadingAdapter } = makeLoadingAdapter();
|
|
243
|
-
loadingAdapter['adapterState'].postState = jest
|
|
244
|
-
.fn()
|
|
245
|
-
.mockResolvedValue(undefined);
|
|
246
|
-
for (const eventType of loaderEvents) {
|
|
247
|
-
jest.clearAllMocks();
|
|
248
|
-
loadingAdapter.hasWorkerEmitted = false;
|
|
249
|
-
// Act
|
|
250
|
-
await loadingAdapter.emit(eventType);
|
|
251
|
-
// Assert
|
|
252
|
-
const callData = (_b = (_a = mockEmit.mock.calls[0]) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.data;
|
|
253
|
-
expect(callData).toHaveProperty('reports');
|
|
254
|
-
expect(callData).toHaveProperty('processed_files');
|
|
255
|
-
expect(callData).not.toHaveProperty('artifacts');
|
|
256
|
-
}
|
|
257
|
-
});
|
|
258
|
-
it('should truncate a long error message, preserving the original prefix', async () => {
|
|
259
|
-
var _a, _b;
|
|
260
|
-
// Arrange
|
|
261
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
262
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
263
|
-
const longMessage = 'E'.repeat(20000);
|
|
264
|
-
// Act
|
|
265
|
-
await adapter.emit(types_1.ExtractorEventType.DataExtractionError, {
|
|
266
|
-
error: { message: longMessage },
|
|
267
|
-
});
|
|
268
|
-
// Assert
|
|
269
|
-
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
270
|
-
const emittedMessage = (_b = (_a = mockEmit.mock.calls[0][0].data) === null || _a === void 0 ? void 0 : _a.error) === null || _b === void 0 ? void 0 : _b.message;
|
|
271
|
-
expect(emittedMessage.length).toBeLessThan(longMessage.length);
|
|
272
|
-
expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true);
|
|
273
|
-
});
|
|
274
|
-
});
|
|
275
|
-
describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => {
|
|
276
|
-
let adapter;
|
|
277
|
-
let adapterState;
|
|
278
|
-
let mockPostMessage;
|
|
279
|
-
beforeEach(() => {
|
|
280
|
-
jest.clearAllMocks();
|
|
281
|
-
({ adapter, adapterState } = makeAdapter());
|
|
282
|
-
const workerThreads = require('node:worker_threads');
|
|
283
|
-
mockPostMessage = jest.fn();
|
|
284
|
-
if (workerThreads.parentPort) {
|
|
285
|
-
jest
|
|
286
|
-
.spyOn(workerThreads.parentPort, 'postMessage')
|
|
287
|
-
.mockImplementation(mockPostMessage);
|
|
288
|
-
}
|
|
289
|
-
else {
|
|
290
|
-
workerThreads.parentPort = { postMessage: mockPostMessage };
|
|
291
|
-
}
|
|
292
|
-
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
293
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
294
|
-
});
|
|
295
|
-
afterEach(() => {
|
|
296
|
-
jest.restoreAllMocks();
|
|
297
|
-
});
|
|
298
|
-
async function emitDone(adapterInstance, extractionStart, extractionEnd) {
|
|
299
|
-
adapterInstance.event.payload.event_context.extract_from = extractionStart;
|
|
300
|
-
adapterInstance.event.payload.event_context.extract_to = extractionEnd;
|
|
301
|
-
// Reset the emit guard so we can emit multiple times within one test.
|
|
302
|
-
adapterInstance['hasWorkerEmitted'] = false;
|
|
303
|
-
await adapterInstance.emit(types_1.ExtractorEventType.AttachmentExtractionDone, {
|
|
304
|
-
reports: [],
|
|
305
|
-
processed_files: [],
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
describe('initial import with UNBOUNDED start', () => {
|
|
309
|
-
it('should set workersOldest to UNBOUNDED_DATE_TIME_VALUE and workersNewest to extraction end', async () => {
|
|
310
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
311
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
312
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
|
|
313
|
-
});
|
|
314
|
-
});
|
|
315
|
-
describe('reconciliation after UNBOUNDED initial import', () => {
|
|
316
|
-
it('should NOT overwrite workersOldest when reconciliation start is later than sentinel', async () => {
|
|
317
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
318
|
-
await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
|
|
319
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
320
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
|
|
321
|
-
});
|
|
322
|
-
it('should NOT overwrite workersOldest even when reconciliation start is very early', async () => {
|
|
323
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
324
|
-
await emitDone(adapter, '1980-01-01T00:00:00.000Z', '1990-01-01T00:00:00.000Z');
|
|
325
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
326
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
|
|
327
|
-
});
|
|
328
|
-
});
|
|
329
|
-
describe('forward sync after UNBOUNDED initial import', () => {
|
|
330
|
-
it('should expand workersNewest forward while preserving workersOldest', async () => {
|
|
331
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
332
|
-
await emitDone(adapter, '2025-06-01T00:00:00.000Z', '2025-07-01T00:00:00.000Z');
|
|
333
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
334
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-07-01T00:00:00.000Z');
|
|
335
|
-
});
|
|
336
|
-
});
|
|
337
|
-
describe('reconciliation with end beyond current newest', () => {
|
|
338
|
-
it('should expand workersNewest when reconciliation end is later', async () => {
|
|
339
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
340
|
-
await emitDone(adapter, '2024-01-01T00:00:00.000Z', '2025-08-01T00:00:00.000Z');
|
|
341
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
342
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-08-01T00:00:00.000Z');
|
|
343
|
-
});
|
|
344
|
-
});
|
|
345
|
-
describe('first sync with absolute dates (no UNBOUNDED)', () => {
|
|
346
|
-
it('should set both boundaries from the extraction range', async () => {
|
|
347
|
-
await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
|
|
348
|
-
expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
|
|
349
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
|
|
350
|
-
});
|
|
351
|
-
});
|
|
352
|
-
describe('reconciliation after absolute initial sync', () => {
|
|
353
|
-
it('should expand workersOldest backward when reconciliation start is earlier', async () => {
|
|
354
|
-
await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
|
|
355
|
-
await emitDone(adapter, '2024-06-01T00:00:00.000Z', '2025-02-01T00:00:00.000Z');
|
|
356
|
-
expect(adapterState.sdkState.workersOldest).toBe('2024-06-01T00:00:00.000Z');
|
|
357
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
|
|
358
|
-
});
|
|
359
|
-
it('should NOT change boundaries when reconciliation is within existing range', async () => {
|
|
360
|
-
await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
|
|
361
|
-
await emitDone(adapter, '2025-01-15T00:00:00.000Z', '2025-02-15T00:00:00.000Z');
|
|
362
|
-
expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
|
|
363
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
|
|
364
|
-
});
|
|
365
|
-
it('should expand both boundaries when reconciliation exceeds both', async () => {
|
|
366
|
-
await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
|
|
367
|
-
await emitDone(adapter, '2024-06-01T00:00:00.000Z', '2025-09-01T00:00:00.000Z');
|
|
368
|
-
expect(adapterState.sdkState.workersOldest).toBe('2024-06-01T00:00:00.000Z');
|
|
369
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-09-01T00:00:00.000Z');
|
|
370
|
-
});
|
|
371
|
-
});
|
|
372
|
-
describe('multiple forward syncs', () => {
|
|
373
|
-
it('should progressively expand workersNewest while preserving workersOldest', async () => {
|
|
374
|
-
await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
|
|
375
|
-
await emitDone(adapter, '2025-06-01T00:00:00.000Z', '2025-07-01T00:00:00.000Z');
|
|
376
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-07-01T00:00:00.000Z');
|
|
377
|
-
await emitDone(adapter, '2025-07-01T00:00:00.000Z', '2025-08-01T00:00:00.000Z');
|
|
378
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-08-01T00:00:00.000Z');
|
|
379
|
-
expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
|
|
380
|
-
});
|
|
381
|
-
});
|
|
382
|
-
describe('non-AttachmentExtractionDone events should NOT update boundaries', () => {
|
|
383
|
-
it.each([
|
|
384
|
-
types_1.ExtractorEventType.DataExtractionDone,
|
|
385
|
-
types_1.ExtractorEventType.DataExtractionProgress,
|
|
386
|
-
types_1.ExtractorEventType.MetadataExtractionError,
|
|
387
|
-
types_1.ExtractorEventType.AttachmentExtractionError,
|
|
388
|
-
])('should not update boundaries on %s', async (eventType) => {
|
|
389
|
-
adapterState.sdkState.workersOldest = '2025-01-01T00:00:00.000Z';
|
|
390
|
-
adapterState.sdkState.workersNewest = '2025-03-01T00:00:00.000Z';
|
|
391
|
-
adapter.event.payload.event_context.extract_from =
|
|
392
|
-
'2024-01-01T00:00:00.000Z';
|
|
393
|
-
adapter.event.payload.event_context.extract_to =
|
|
394
|
-
'2025-12-01T00:00:00.000Z';
|
|
395
|
-
await adapter.emit(eventType, {
|
|
396
|
-
reports: [],
|
|
397
|
-
processed_files: [],
|
|
398
|
-
});
|
|
399
|
-
expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
|
|
400
|
-
expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
|
|
401
|
-
});
|
|
402
|
-
});
|
|
403
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.artifacts.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const constants_1 = require("../../common/constants");
|
|
4
|
-
const extraction_state_1 = require("../../state/extraction-state");
|
|
5
|
-
const jest_setup_1 = require("../../tests/jest.setup");
|
|
6
|
-
const test_helpers_1 = require("../../tests/test-helpers");
|
|
7
|
-
const test_utils_1 = require("../../common/test-utils");
|
|
8
|
-
const types_1 = require("../../types");
|
|
9
|
-
const extraction_adapter_1 = require("./extraction-adapter");
|
|
10
|
-
// 1. Create a mock function for the method you want to override.
|
|
11
|
-
const mockUpload = (itemType, objects) => {
|
|
12
|
-
return {
|
|
13
|
-
error: null,
|
|
14
|
-
artifact: {
|
|
15
|
-
id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`,
|
|
16
|
-
item_type: itemType,
|
|
17
|
-
item_count: objects.length,
|
|
18
|
-
},
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
// 2. Mock the entire 'uploader' module.
|
|
22
|
-
// The factory function () => { ... } returns the mock implementation.
|
|
23
|
-
jest.mock('../../uploader/uploader', () => {
|
|
24
|
-
return {
|
|
25
|
-
// The mocked Uploader class
|
|
26
|
-
Uploader: jest.fn().mockImplementation(() => {
|
|
27
|
-
// The constructor of the mocked Uploader returns an object
|
|
28
|
-
// with the methods you want to control.
|
|
29
|
-
return {
|
|
30
|
-
upload: mockUpload,
|
|
31
|
-
};
|
|
32
|
-
}),
|
|
33
|
-
};
|
|
34
|
-
});
|
|
35
|
-
function checkArtifactOrder(artifacts, expectedOrder) {
|
|
36
|
-
let outerIndex = 0;
|
|
37
|
-
for (const artifact of artifacts) {
|
|
38
|
-
try {
|
|
39
|
-
// Always increase outer index. If items are out of order, the array will overflow and exception will be thrown
|
|
40
|
-
while (artifact.item_type != expectedOrder[outerIndex].itemType) {
|
|
41
|
-
outerIndex++;
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
catch (e) {
|
|
45
|
-
console.error('Error finding artifact type in repos:', e);
|
|
46
|
-
return false;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
return true;
|
|
50
|
-
}
|
|
51
|
-
describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => {
|
|
52
|
-
let testAdapter;
|
|
53
|
-
beforeEach(() => {
|
|
54
|
-
// Create a fresh adapter instance for this test to avoid mocking conflicts
|
|
55
|
-
const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
56
|
-
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
57
|
-
});
|
|
58
|
-
const mockAdapterState = new extraction_state_1.ExtractionState({
|
|
59
|
-
event: mockEvent,
|
|
60
|
-
initialState: { attachments: { completed: false } },
|
|
61
|
-
});
|
|
62
|
-
testAdapter = new extraction_adapter_1.ExtractionAdapter({
|
|
63
|
-
event: mockEvent,
|
|
64
|
-
adapterState: mockAdapterState,
|
|
65
|
-
options: {
|
|
66
|
-
batchSize: 50,
|
|
67
|
-
},
|
|
68
|
-
});
|
|
69
|
-
});
|
|
70
|
-
it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => {
|
|
71
|
-
var _a, _b;
|
|
72
|
-
const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
|
|
73
|
-
// Initialize repos
|
|
74
|
-
testAdapter.initializeRepos(repos);
|
|
75
|
-
await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
|
|
76
|
-
await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(105)));
|
|
77
|
-
await testAdapter.uploadAllRepos();
|
|
78
|
-
const artifacts = testAdapter.artifacts;
|
|
79
|
-
expect(artifacts.length).toBe(4);
|
|
80
|
-
expect(checkArtifactOrder(artifacts, repos)).toBe(true);
|
|
81
|
-
});
|
|
82
|
-
it('should work with more than 2 repos', async () => {
|
|
83
|
-
var _a, _b, _c, _d;
|
|
84
|
-
const repos = [
|
|
85
|
-
{ itemType: 'ItemTypeA' },
|
|
86
|
-
{ itemType: 'ItemTypeB' },
|
|
87
|
-
{ itemType: 'ItemTypeC' },
|
|
88
|
-
{ itemType: 'ItemTypeD' },
|
|
89
|
-
];
|
|
90
|
-
// Initialize repos
|
|
91
|
-
testAdapter.initializeRepos(repos);
|
|
92
|
-
await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
|
|
93
|
-
await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
|
|
94
|
-
await ((_c = testAdapter.getRepo('ItemTypeC')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(103)));
|
|
95
|
-
await ((_d = testAdapter.getRepo('ItemTypeD')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(104)));
|
|
96
|
-
await testAdapter.uploadAllRepos();
|
|
97
|
-
const artifacts = testAdapter.artifacts;
|
|
98
|
-
expect(artifacts.length).toBe(12);
|
|
99
|
-
expect(checkArtifactOrder(artifacts, repos)).toBe(true);
|
|
100
|
-
});
|
|
101
|
-
it('should maintain order with multiple pushes and uploads', async () => {
|
|
102
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
|
|
103
|
-
const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
|
|
104
|
-
// Initialize repos
|
|
105
|
-
testAdapter.initializeRepos(repos);
|
|
106
|
-
await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
|
|
107
|
-
await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
|
|
108
|
-
await ((_c = testAdapter.getRepo('ItemTypeA')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(101)));
|
|
109
|
-
await ((_d = testAdapter.getRepo('ItemTypeB')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(102)));
|
|
110
|
-
await ((_e = testAdapter.getRepo('ItemTypeA')) === null || _e === void 0 ? void 0 : _e.upload());
|
|
111
|
-
await ((_f = testAdapter.getRepo('ItemTypeB')) === null || _f === void 0 ? void 0 : _f.upload());
|
|
112
|
-
await ((_g = testAdapter.getRepo('ItemTypeA')) === null || _g === void 0 ? void 0 : _g.push((0, test_helpers_1.createItems)(101)));
|
|
113
|
-
await ((_h = testAdapter.getRepo('ItemTypeB')) === null || _h === void 0 ? void 0 : _h.push((0, test_helpers_1.createItems)(102)));
|
|
114
|
-
await ((_j = testAdapter.getRepo('ItemTypeA')) === null || _j === void 0 ? void 0 : _j.push((0, test_helpers_1.createItems)(101)));
|
|
115
|
-
await ((_k = testAdapter.getRepo('ItemTypeB')) === null || _k === void 0 ? void 0 : _k.push((0, test_helpers_1.createItems)(102)));
|
|
116
|
-
await testAdapter.uploadAllRepos();
|
|
117
|
-
const artifacts = testAdapter.artifacts;
|
|
118
|
-
expect(artifacts.length).toBe(20);
|
|
119
|
-
expect(checkArtifactOrder(artifacts, repos)).toBe(true);
|
|
120
|
-
});
|
|
121
|
-
it('should not count artifacts if 0 items are pushed to the repo', async () => {
|
|
122
|
-
var _a;
|
|
123
|
-
const repos = [{ itemType: 'ItemTypeA' }];
|
|
124
|
-
// Initialize repos
|
|
125
|
-
testAdapter.initializeRepos(repos);
|
|
126
|
-
await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push([]));
|
|
127
|
-
await testAdapter.uploadAllRepos();
|
|
128
|
-
const artifacts = testAdapter.artifacts;
|
|
129
|
-
expect(artifacts.length).toBe(0);
|
|
130
|
-
expect(checkArtifactOrder(artifacts, repos)).toBe(true);
|
|
131
|
-
});
|
|
132
|
-
});
|
|
133
|
-
describe('External sync units splitting into artifacts', () => {
|
|
134
|
-
let testAdapter;
|
|
135
|
-
beforeEach(() => {
|
|
136
|
-
const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
137
|
-
payload: { event_type: types_1.EventType.StartExtractingExternalSyncUnits },
|
|
138
|
-
});
|
|
139
|
-
const mockAdapterState = new extraction_state_1.ExtractionState({
|
|
140
|
-
event: mockEvent,
|
|
141
|
-
initialState: {},
|
|
142
|
-
});
|
|
143
|
-
testAdapter = new extraction_adapter_1.ExtractionAdapter({
|
|
144
|
-
event: mockEvent,
|
|
145
|
-
adapterState: mockAdapterState,
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
it('should split 125k external sync units into 5 artifacts', async () => {
|
|
149
|
-
const BATCH_SIZE = 25000;
|
|
150
|
-
const TOTAL_UNITS = 125000;
|
|
151
|
-
const externalSyncUnits = Array.from({ length: TOTAL_UNITS }, (_, i) => ({
|
|
152
|
-
id: String(i),
|
|
153
|
-
name: `Unit ${i}`,
|
|
154
|
-
description: `Description ${i}`,
|
|
155
|
-
}));
|
|
156
|
-
testAdapter.initializeRepos([
|
|
157
|
-
{
|
|
158
|
-
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
|
|
159
|
-
overridenOptions: {
|
|
160
|
-
batchSize: BATCH_SIZE,
|
|
161
|
-
skipConfirmation: true,
|
|
162
|
-
},
|
|
163
|
-
},
|
|
164
|
-
]);
|
|
165
|
-
const repo = testAdapter.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS);
|
|
166
|
-
const chunkSize = 10000;
|
|
167
|
-
for (let i = 0; i < TOTAL_UNITS; i += chunkSize) {
|
|
168
|
-
await (repo === null || repo === void 0 ? void 0 : repo.push(externalSyncUnits.slice(i, i + chunkSize)));
|
|
169
|
-
}
|
|
170
|
-
await testAdapter.uploadAllRepos();
|
|
171
|
-
expect(testAdapter.artifacts.length).toBe(TOTAL_UNITS / BATCH_SIZE // 5
|
|
172
|
-
);
|
|
173
|
-
expect(testAdapter.artifacts.every((a) => a.item_type === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)).toBe(true);
|
|
174
|
-
});
|
|
175
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.test.ts"],"names":[],"mappings":""}
|