@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/README.md +5 -11
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
  8. package/dist/common/constants.d.ts +4 -5
  9. package/dist/common/constants.d.ts.map +1 -1
  10. package/dist/common/constants.js +6 -7
  11. package/dist/common/helpers.d.ts +3 -35
  12. package/dist/common/helpers.d.ts.map +1 -1
  13. package/dist/common/helpers.js +5 -50
  14. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  15. package/dist/http/client.d.ts.map +1 -0
  16. package/dist/http/{axios-client-internal.js → client.js} +4 -8
  17. package/dist/index.d.ts +24 -13
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +43 -32
  20. package/dist/logger/logger.d.ts +12 -53
  21. package/dist/logger/logger.d.ts.map +1 -1
  22. package/dist/logger/logger.interfaces.d.ts +0 -1
  23. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  24. package/dist/logger/logger.js +36 -68
  25. package/dist/mappers/mappers.d.ts +7 -46
  26. package/dist/mappers/mappers.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.d.ts +175 -0
  28. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  29. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
  30. package/dist/mappers/mappers.js +11 -45
  31. package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
  32. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  33. package/dist/multithreading/adapters/base-adapter.js +97 -62
  34. package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
  35. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  36. package/dist/multithreading/adapters/extraction-adapter.js +203 -206
  37. package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
  38. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  41. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  42. package/dist/multithreading/adapters/loading-adapter.js +294 -346
  43. package/dist/multithreading/emit.d.ts +11 -0
  44. package/dist/multithreading/emit.d.ts.map +1 -0
  45. package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
  46. package/dist/multithreading/process-task.d.ts +2 -12
  47. package/dist/multithreading/process-task.d.ts.map +1 -1
  48. package/dist/multithreading/process-task.js +46 -43
  49. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  50. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
  52. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  53. package/dist/multithreading/spawn/spawn.helpers.js +128 -20
  54. package/dist/multithreading/spawn/spawn.js +24 -39
  55. package/dist/multithreading/worker.js +1 -4
  56. package/dist/repo/repo.d.ts +10 -0
  57. package/dist/repo/repo.d.ts.map +1 -1
  58. package/dist/repo/repo.helpers.d.ts +6 -0
  59. package/dist/repo/repo.helpers.d.ts.map +1 -0
  60. package/dist/repo/repo.helpers.js +16 -0
  61. package/dist/repo/repo.interfaces.d.ts +4 -16
  62. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  63. package/dist/repo/repo.js +23 -8
  64. package/dist/state/base-state.d.ts +16 -38
  65. package/dist/state/base-state.d.ts.map +1 -1
  66. package/dist/state/base-state.js +19 -41
  67. package/dist/state/extraction-state.d.ts +6 -21
  68. package/dist/state/extraction-state.d.ts.map +1 -1
  69. package/dist/state/extraction-state.js +8 -25
  70. package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
  71. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  72. package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
  73. package/dist/state/loading-state.d.ts +3 -14
  74. package/dist/state/loading-state.d.ts.map +1 -1
  75. package/dist/state/loading-state.js +3 -14
  76. package/dist/state/state.d.ts +4 -0
  77. package/dist/state/state.d.ts.map +1 -0
  78. package/dist/state/state.interfaces.d.ts +33 -37
  79. package/dist/state/state.interfaces.d.ts.map +1 -1
  80. package/dist/state/state.interfaces.js +16 -11
  81. package/dist/state/state.js +11 -0
  82. package/dist/state/time-value-resolver.d.ts +17 -0
  83. package/dist/state/time-value-resolver.d.ts.map +1 -0
  84. package/dist/{common → state}/time-value-resolver.js +9 -41
  85. package/dist/testing/mock-event.d.ts +10 -0
  86. package/dist/testing/mock-event.d.ts.map +1 -0
  87. package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
  88. package/dist/testing/mock-server.d.ts +29 -0
  89. package/dist/testing/mock-server.d.ts.map +1 -0
  90. package/dist/testing/mock-server.interfaces.d.ts +48 -0
  91. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  92. package/dist/{mock-server → testing}/mock-server.js +21 -33
  93. package/dist/types/common.d.ts +1 -12
  94. package/dist/types/common.d.ts.map +1 -1
  95. package/dist/types/common.js +1 -6
  96. package/dist/types/errors.d.ts +9 -0
  97. package/dist/types/errors.d.ts.map +1 -0
  98. package/dist/types/external-domain-metadata.d.ts +30 -171
  99. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  100. package/dist/types/extraction.d.ts +64 -94
  101. package/dist/types/extraction.d.ts.map +1 -1
  102. package/dist/types/extraction.js +6 -34
  103. package/dist/types/loading.d.ts +1 -2
  104. package/dist/types/loading.d.ts.map +1 -1
  105. package/dist/types/loading.js +0 -1
  106. package/dist/types/workers.d.ts +39 -94
  107. package/dist/types/workers.d.ts.map +1 -1
  108. package/dist/types/workers.js +0 -6
  109. package/dist/uploader/uploader.d.ts +8 -59
  110. package/dist/uploader/uploader.d.ts.map +1 -1
  111. package/dist/uploader/uploader.helpers.d.ts +4 -27
  112. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  113. package/dist/uploader/uploader.helpers.js +64 -30
  114. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  115. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  116. package/dist/uploader/uploader.interfaces.js +8 -0
  117. package/dist/uploader/uploader.js +23 -89
  118. package/package.json +8 -6
  119. package/dist/airsync-sdk.d.ts +0 -2059
  120. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  121. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  122. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
  123. package/dist/common/control-protocol.d.ts +0 -10
  124. package/dist/common/control-protocol.d.ts.map +0 -1
  125. package/dist/common/errors.d.ts +0 -6
  126. package/dist/common/errors.d.ts.map +0 -1
  127. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  128. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  129. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  130. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  131. package/dist/common/test-utils.d.ts +0 -19
  132. package/dist/common/test-utils.d.ts.map +0 -1
  133. package/dist/common/time-value-resolver.d.ts +0 -47
  134. package/dist/common/time-value-resolver.d.ts.map +0 -1
  135. package/dist/common/time-value-resolver.test.d.ts +0 -2
  136. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  137. package/dist/common/time-value-resolver.test.js +0 -268
  138. package/dist/http/axios-client-internal.d.ts.map +0 -1
  139. package/dist/http/axios-client-internal.test.d.ts +0 -2
  140. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  141. package/dist/http/axios-client-internal.test.js +0 -175
  142. package/dist/http/axios-client.d.ts +0 -27
  143. package/dist/http/axios-client.d.ts.map +0 -1
  144. package/dist/http/axios-client.js +0 -60
  145. package/dist/http/constants.d.ts +0 -4
  146. package/dist/http/constants.d.ts.map +0 -1
  147. package/dist/http/constants.js +0 -6
  148. package/dist/http/index.d.ts +0 -2
  149. package/dist/http/index.d.ts.map +0 -1
  150. package/dist/http/index.js +0 -17
  151. package/dist/logger/logger.constants.d.ts +0 -6
  152. package/dist/logger/logger.constants.d.ts.map +0 -1
  153. package/dist/logger/logger.constants.js +0 -13
  154. package/dist/logger/logger.context.d.ts +0 -58
  155. package/dist/logger/logger.context.d.ts.map +0 -1
  156. package/dist/logger/logger.context.js +0 -86
  157. package/dist/logger/logger.test.d.ts +0 -2
  158. package/dist/logger/logger.test.d.ts.map +0 -1
  159. package/dist/logger/logger.test.js +0 -537
  160. package/dist/mappers/mappers.interface.d.ts +0 -294
  161. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  162. package/dist/mappers/mappers.test.d.ts +0 -2
  163. package/dist/mappers/mappers.test.d.ts.map +0 -1
  164. package/dist/mappers/mappers.test.js +0 -105
  165. package/dist/mock-server/mock-server.d.ts +0 -47
  166. package/dist/mock-server/mock-server.d.ts.map +0 -1
  167. package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
  168. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  169. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
  170. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
  171. package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
  172. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
  173. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
  174. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
  175. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
  176. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
  177. package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
  178. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
  179. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
  180. package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
  181. package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
  182. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
  183. package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
  184. package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
  185. package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
  186. package/dist/multithreading/adapters/serialization.test.js +0 -71
  187. package/dist/multithreading/create-worker.test.d.ts +0 -2
  188. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  189. package/dist/multithreading/create-worker.test.js +0 -108
  190. package/dist/multithreading/process-task.test.d.ts +0 -2
  191. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  192. package/dist/multithreading/process-task.test.js +0 -146
  193. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  194. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  195. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  196. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  197. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  198. package/dist/multithreading/spawn/spawn.test.js +0 -232
  199. package/dist/repo/repo.test.d.ts +0 -2
  200. package/dist/repo/repo.test.d.ts.map +0 -1
  201. package/dist/repo/repo.test.js +0 -184
  202. package/dist/state/state.extract-window.test.d.ts +0 -2
  203. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  204. package/dist/state/state.extract-window.test.js +0 -163
  205. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  206. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  207. package/dist/state/state.pending-boundaries.test.js +0 -189
  208. package/dist/state/state.post-state.test.d.ts +0 -2
  209. package/dist/state/state.post-state.test.d.ts.map +0 -1
  210. package/dist/state/state.post-state.test.js +0 -78
  211. package/dist/state/state.test.d.ts +0 -2
  212. package/dist/state/state.test.d.ts.map +0 -1
  213. package/dist/state/state.test.js +0 -271
  214. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  215. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  216. package/dist/state/state.time-value-resolution.test.js +0 -175
  217. package/dist/tsdoc-metadata.json +0 -11
  218. package/dist/types/extraction.test.d.ts +0 -2
  219. package/dist/types/extraction.test.d.ts.map +0 -1
  220. package/dist/types/extraction.test.js +0 -96
  221. package/dist/types/index.d.ts +0 -10
  222. package/dist/types/index.d.ts.map +0 -1
  223. package/dist/types/index.js +0 -21
  224. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  225. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  226. package/dist/uploader/uploader.helpers.test.js +0 -256
  227. package/dist/uploader/uploader.test.d.ts +0 -2
  228. package/dist/uploader/uploader.test.d.ts.map +0 -1
  229. package/dist/uploader/uploader.test.js +0 -601
  230. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  231. /package/dist/{common → types}/errors.js +0 -0
@@ -1,403 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const constants_1 = require("../../common/constants");
4
- const extraction_state_1 = require("../../state/extraction-state");
5
- const loading_state_1 = require("../../state/loading-state");
6
- const jest_setup_1 = require("../../tests/jest.setup");
7
- const test_utils_1 = require("../../common/test-utils");
8
- const types_1 = require("../../types");
9
- const loading_1 = require("../../types/loading");
10
- const extraction_adapter_1 = require("./extraction-adapter");
11
- const loading_adapter_1 = require("./loading-adapter");
12
- /* eslint-disable @typescript-eslint/no-require-imports */
13
- jest.mock('../../common/control-protocol', () => ({
14
- emit: jest.fn().mockResolvedValue({}),
15
- }));
16
- jest.mock('../../mappers/mappers');
17
- jest.mock('../../uploader/uploader');
18
- jest.mock('../../repo/repo');
19
- jest.mock('node:worker_threads', () => ({
20
- parentPort: { postMessage: jest.fn() },
21
- }));
22
- jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
23
- AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
24
- streamAll: jest.fn().mockResolvedValue(undefined),
25
- })),
26
- }));
27
- function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
28
- const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
29
- payload: { event_type: eventType },
30
- });
31
- const initialState = {
32
- attachments: { completed: false },
33
- };
34
- const adapterState = new extraction_state_1.ExtractionState({ event, initialState });
35
- adapterState.sdkState.snapInVersionId = '';
36
- adapterState.sdkState.toDevRev = {
37
- attachmentsMetadata: {
38
- artifactIds: [],
39
- lastProcessed: 0,
40
- lastProcessedAttachmentsIdsList: [],
41
- },
42
- };
43
- const adapter = new extraction_adapter_1.ExtractionAdapter({ event, adapterState });
44
- return { adapter, event, adapterState };
45
- }
46
- function makeLoadingAdapter(eventType = types_1.EventType.StartLoadingData) {
47
- const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
48
- payload: { event_type: eventType },
49
- });
50
- const initialState = {
51
- attachments: { completed: false },
52
- };
53
- const adapterState = new loading_state_1.LoadingState({ event, initialState });
54
- adapterState.sdkState.snapInVersionId = '';
55
- const adapter = new loading_adapter_1.LoadingAdapter({ event, adapterState });
56
- return { adapter, event, adapterState };
57
- }
58
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.emit`, () => {
59
- let adapter;
60
- let mockPostMessage;
61
- beforeEach(() => {
62
- jest.clearAllMocks();
63
- ({ adapter } = makeAdapter());
64
- const workerThreads = require('node:worker_threads');
65
- mockPostMessage = jest.fn();
66
- if (workerThreads.parentPort) {
67
- jest
68
- .spyOn(workerThreads.parentPort, 'postMessage')
69
- .mockImplementation(mockPostMessage);
70
- }
71
- else {
72
- workerThreads.parentPort = { postMessage: mockPostMessage };
73
- }
74
- });
75
- afterEach(() => {
76
- jest.restoreAllMocks();
77
- });
78
- it('should emit only one event when multiple events of same type are sent', async () => {
79
- // Arrange
80
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
81
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
82
- // Act
83
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
84
- reports: [],
85
- processed_files: [],
86
- });
87
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
88
- reports: [],
89
- processed_files: [],
90
- });
91
- // Assert
92
- expect(mockPostMessage).toHaveBeenCalledTimes(1);
93
- });
94
- it('should emit only once even when a different event type follows', async () => {
95
- // Arrange
96
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
97
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
98
- // Act
99
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
100
- reports: [],
101
- processed_files: [],
102
- });
103
- await adapter.emit(types_1.ExtractorEventType.DataExtractionError, {
104
- reports: [],
105
- processed_files: [],
106
- });
107
- await adapter.emit(types_1.ExtractorEventType.AttachmentExtractionError, {
108
- reports: [],
109
- processed_files: [],
110
- });
111
- // Assert
112
- expect(mockPostMessage).toHaveBeenCalledTimes(1);
113
- });
114
- it('should correctly emit one event even if postState errors', async () => {
115
- // Arrange
116
- adapter['adapterState'].postState = jest
117
- .fn()
118
- .mockRejectedValue(new Error('postState error'));
119
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
120
- // Act
121
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
122
- reports: [],
123
- processed_files: [],
124
- });
125
- // Assert
126
- expect(mockPostMessage).toHaveBeenCalledTimes(1);
127
- });
128
- it('should correctly emit one event even if uploadAllRepos errors', async () => {
129
- // Arrange
130
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
131
- adapter.uploadAllRepos = jest
132
- .fn()
133
- .mockRejectedValue(new Error('uploadAllRepos error'));
134
- // Act
135
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
136
- reports: [],
137
- processed_files: [],
138
- });
139
- // Assert
140
- expect(mockPostMessage).toHaveBeenCalledTimes(1);
141
- });
142
- it('should include artifacts in data for extraction events', async () => {
143
- // Arrange
144
- const { emit: mockEmit } = require('../../common/control-protocol');
145
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
146
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
147
- adapter['_artifacts'] = [
148
- { id: 'art-1', item_count: 10, item_type: 'issues' },
149
- ];
150
- // Act
151
- await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
152
- // Assert
153
- expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
154
- data: expect.objectContaining({
155
- artifacts: expect.arrayContaining([
156
- expect.objectContaining({ id: 'art-1' }),
157
- ]),
158
- }),
159
- }));
160
- const callData = mockEmit.mock.calls[0][0].data;
161
- expect(callData).not.toHaveProperty('reports');
162
- expect(callData).not.toHaveProperty('processed_files');
163
- });
164
- it('should include reports and processed_files in data for loader events', async () => {
165
- // Arrange
166
- const { emit: mockEmit } = require('../../common/control-protocol');
167
- const { adapter: loadingAdapter } = makeLoadingAdapter();
168
- loadingAdapter['adapterState'].postState = jest
169
- .fn()
170
- .mockResolvedValue(undefined);
171
- loadingAdapter['loaderReports'] = [
172
- { item_type: 'tasks', [loading_1.ActionType.CREATED]: 5 },
173
- ];
174
- loadingAdapter['_processedFiles'] = ['file-1', 'file-2'];
175
- // Act
176
- await loadingAdapter.emit(types_1.LoaderEventType.DataLoadingDone);
177
- // Assert
178
- expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
179
- data: expect.objectContaining({
180
- reports: expect.arrayContaining([
181
- expect.objectContaining({ item_type: 'tasks' }),
182
- ]),
183
- processed_files: ['file-1', 'file-2'],
184
- }),
185
- }));
186
- const callData = mockEmit.mock.calls[0][0].data;
187
- expect(callData).not.toHaveProperty('artifacts');
188
- });
189
- it('should not include artifacts, reports, or processed_files for unknown event types', async () => {
190
- // Arrange
191
- const { emit: mockEmit } = require('../../common/control-protocol');
192
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
193
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
194
- adapter['_artifacts'] = [
195
- { id: 'art-1', item_count: 10, item_type: 'issues' },
196
- ];
197
- // Act
198
- await adapter.emit('SOME_UNKNOWN_EVENT');
199
- // Assert
200
- const callData = mockEmit.mock.calls[0][0].data;
201
- expect(callData).not.toHaveProperty('artifacts');
202
- expect(callData).not.toHaveProperty('reports');
203
- expect(callData).not.toHaveProperty('processed_files');
204
- });
205
- it('should include artifacts for all ExtractorEventType values', async () => {
206
- var _a, _b;
207
- // Arrange
208
- const { emit: mockEmit } = require('../../common/control-protocol');
209
- const extractorEvents = [
210
- types_1.ExtractorEventType.DataExtractionDone,
211
- types_1.ExtractorEventType.DataExtractionProgress,
212
- types_1.ExtractorEventType.DataExtractionError,
213
- types_1.ExtractorEventType.AttachmentExtractionDone,
214
- types_1.ExtractorEventType.AttachmentExtractionProgress,
215
- ];
216
- for (const eventType of extractorEvents) {
217
- jest.clearAllMocks();
218
- adapter.hasWorkerEmitted = false;
219
- adapter['adapterState'].postState = jest
220
- .fn()
221
- .mockResolvedValue(undefined);
222
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
223
- // Act
224
- await adapter.emit(eventType);
225
- // Assert
226
- const callData = (_b = (_a = mockEmit.mock.calls[0]) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.data;
227
- expect(callData).toHaveProperty('artifacts');
228
- expect(callData).not.toHaveProperty('reports');
229
- }
230
- });
231
- it('should include reports and processed_files for all LoaderEventType values', async () => {
232
- var _a, _b;
233
- // Arrange
234
- const { emit: mockEmit } = require('../../common/control-protocol');
235
- const loaderEvents = [
236
- types_1.LoaderEventType.DataLoadingDone,
237
- types_1.LoaderEventType.DataLoadingProgress,
238
- types_1.LoaderEventType.DataLoadingError,
239
- types_1.LoaderEventType.AttachmentLoadingDone,
240
- types_1.LoaderEventType.AttachmentLoadingProgress,
241
- ];
242
- const { adapter: loadingAdapter } = makeLoadingAdapter();
243
- loadingAdapter['adapterState'].postState = jest
244
- .fn()
245
- .mockResolvedValue(undefined);
246
- for (const eventType of loaderEvents) {
247
- jest.clearAllMocks();
248
- loadingAdapter.hasWorkerEmitted = false;
249
- // Act
250
- await loadingAdapter.emit(eventType);
251
- // Assert
252
- const callData = (_b = (_a = mockEmit.mock.calls[0]) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.data;
253
- expect(callData).toHaveProperty('reports');
254
- expect(callData).toHaveProperty('processed_files');
255
- expect(callData).not.toHaveProperty('artifacts');
256
- }
257
- });
258
- it('should truncate a long error message, preserving the original prefix', async () => {
259
- var _a, _b;
260
- // Arrange
261
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
262
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
263
- const longMessage = 'E'.repeat(20000);
264
- // Act
265
- await adapter.emit(types_1.ExtractorEventType.DataExtractionError, {
266
- error: { message: longMessage },
267
- });
268
- // Assert
269
- const { emit: mockEmit } = require('../../common/control-protocol');
270
- const emittedMessage = (_b = (_a = mockEmit.mock.calls[0][0].data) === null || _a === void 0 ? void 0 : _a.error) === null || _b === void 0 ? void 0 : _b.message;
271
- expect(emittedMessage.length).toBeLessThan(longMessage.length);
272
- expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true);
273
- });
274
- });
275
- describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => {
276
- let adapter;
277
- let adapterState;
278
- let mockPostMessage;
279
- beforeEach(() => {
280
- jest.clearAllMocks();
281
- ({ adapter, adapterState } = makeAdapter());
282
- const workerThreads = require('node:worker_threads');
283
- mockPostMessage = jest.fn();
284
- if (workerThreads.parentPort) {
285
- jest
286
- .spyOn(workerThreads.parentPort, 'postMessage')
287
- .mockImplementation(mockPostMessage);
288
- }
289
- else {
290
- workerThreads.parentPort = { postMessage: mockPostMessage };
291
- }
292
- adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
293
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
294
- });
295
- afterEach(() => {
296
- jest.restoreAllMocks();
297
- });
298
- async function emitDone(adapterInstance, extractionStart, extractionEnd) {
299
- adapterInstance.event.payload.event_context.extract_from = extractionStart;
300
- adapterInstance.event.payload.event_context.extract_to = extractionEnd;
301
- // Reset the emit guard so we can emit multiple times within one test.
302
- adapterInstance['hasWorkerEmitted'] = false;
303
- await adapterInstance.emit(types_1.ExtractorEventType.AttachmentExtractionDone, {
304
- reports: [],
305
- processed_files: [],
306
- });
307
- }
308
- describe('initial import with UNBOUNDED start', () => {
309
- it('should set workersOldest to UNBOUNDED_DATE_TIME_VALUE and workersNewest to extraction end', async () => {
310
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
311
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
312
- expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
313
- });
314
- });
315
- describe('reconciliation after UNBOUNDED initial import', () => {
316
- it('should NOT overwrite workersOldest when reconciliation start is later than sentinel', async () => {
317
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
318
- await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
319
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
320
- expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
321
- });
322
- it('should NOT overwrite workersOldest even when reconciliation start is very early', async () => {
323
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
324
- await emitDone(adapter, '1980-01-01T00:00:00.000Z', '1990-01-01T00:00:00.000Z');
325
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
326
- expect(adapterState.sdkState.workersNewest).toBe('2025-06-01T00:00:00.000Z');
327
- });
328
- });
329
- describe('forward sync after UNBOUNDED initial import', () => {
330
- it('should expand workersNewest forward while preserving workersOldest', async () => {
331
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
332
- await emitDone(adapter, '2025-06-01T00:00:00.000Z', '2025-07-01T00:00:00.000Z');
333
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
334
- expect(adapterState.sdkState.workersNewest).toBe('2025-07-01T00:00:00.000Z');
335
- });
336
- });
337
- describe('reconciliation with end beyond current newest', () => {
338
- it('should expand workersNewest when reconciliation end is later', async () => {
339
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
340
- await emitDone(adapter, '2024-01-01T00:00:00.000Z', '2025-08-01T00:00:00.000Z');
341
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
342
- expect(adapterState.sdkState.workersNewest).toBe('2025-08-01T00:00:00.000Z');
343
- });
344
- });
345
- describe('first sync with absolute dates (no UNBOUNDED)', () => {
346
- it('should set both boundaries from the extraction range', async () => {
347
- await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
348
- expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
349
- expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
350
- });
351
- });
352
- describe('reconciliation after absolute initial sync', () => {
353
- it('should expand workersOldest backward when reconciliation start is earlier', async () => {
354
- await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
355
- await emitDone(adapter, '2024-06-01T00:00:00.000Z', '2025-02-01T00:00:00.000Z');
356
- expect(adapterState.sdkState.workersOldest).toBe('2024-06-01T00:00:00.000Z');
357
- expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
358
- });
359
- it('should NOT change boundaries when reconciliation is within existing range', async () => {
360
- await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
361
- await emitDone(adapter, '2025-01-15T00:00:00.000Z', '2025-02-15T00:00:00.000Z');
362
- expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
363
- expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
364
- });
365
- it('should expand both boundaries when reconciliation exceeds both', async () => {
366
- await emitDone(adapter, '2025-01-01T00:00:00.000Z', '2025-03-01T00:00:00.000Z');
367
- await emitDone(adapter, '2024-06-01T00:00:00.000Z', '2025-09-01T00:00:00.000Z');
368
- expect(adapterState.sdkState.workersOldest).toBe('2024-06-01T00:00:00.000Z');
369
- expect(adapterState.sdkState.workersNewest).toBe('2025-09-01T00:00:00.000Z');
370
- });
371
- });
372
- describe('multiple forward syncs', () => {
373
- it('should progressively expand workersNewest while preserving workersOldest', async () => {
374
- await emitDone(adapter, constants_1.UNBOUNDED_DATE_TIME_VALUE, '2025-06-01T00:00:00.000Z');
375
- await emitDone(adapter, '2025-06-01T00:00:00.000Z', '2025-07-01T00:00:00.000Z');
376
- expect(adapterState.sdkState.workersNewest).toBe('2025-07-01T00:00:00.000Z');
377
- await emitDone(adapter, '2025-07-01T00:00:00.000Z', '2025-08-01T00:00:00.000Z');
378
- expect(adapterState.sdkState.workersNewest).toBe('2025-08-01T00:00:00.000Z');
379
- expect(adapterState.sdkState.workersOldest).toBe(constants_1.UNBOUNDED_DATE_TIME_VALUE);
380
- });
381
- });
382
- describe('non-AttachmentExtractionDone events should NOT update boundaries', () => {
383
- it.each([
384
- types_1.ExtractorEventType.DataExtractionDone,
385
- types_1.ExtractorEventType.DataExtractionProgress,
386
- types_1.ExtractorEventType.MetadataExtractionError,
387
- types_1.ExtractorEventType.AttachmentExtractionError,
388
- ])('should not update boundaries on %s', async (eventType) => {
389
- adapterState.sdkState.workersOldest = '2025-01-01T00:00:00.000Z';
390
- adapterState.sdkState.workersNewest = '2025-03-01T00:00:00.000Z';
391
- adapter.event.payload.event_context.extract_from =
392
- '2024-01-01T00:00:00.000Z';
393
- adapter.event.payload.event_context.extract_to =
394
- '2025-12-01T00:00:00.000Z';
395
- await adapter.emit(eventType, {
396
- reports: [],
397
- processed_files: [],
398
- });
399
- expect(adapterState.sdkState.workersOldest).toBe('2025-01-01T00:00:00.000Z');
400
- expect(adapterState.sdkState.workersNewest).toBe('2025-03-01T00:00:00.000Z');
401
- });
402
- });
403
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=extraction-adapter.artifacts.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"extraction-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.artifacts.test.ts"],"names":[],"mappings":""}
@@ -1,175 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const constants_1 = require("../../common/constants");
4
- const extraction_state_1 = require("../../state/extraction-state");
5
- const jest_setup_1 = require("../../tests/jest.setup");
6
- const test_helpers_1 = require("../../tests/test-helpers");
7
- const test_utils_1 = require("../../common/test-utils");
8
- const types_1 = require("../../types");
9
- const extraction_adapter_1 = require("./extraction-adapter");
10
- // 1. Create a mock function for the method you want to override.
11
- const mockUpload = (itemType, objects) => {
12
- return {
13
- error: null,
14
- artifact: {
15
- id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`,
16
- item_type: itemType,
17
- item_count: objects.length,
18
- },
19
- };
20
- };
21
- // 2. Mock the entire 'uploader' module.
22
- // The factory function () => { ... } returns the mock implementation.
23
- jest.mock('../../uploader/uploader', () => {
24
- return {
25
- // The mocked Uploader class
26
- Uploader: jest.fn().mockImplementation(() => {
27
- // The constructor of the mocked Uploader returns an object
28
- // with the methods you want to control.
29
- return {
30
- upload: mockUpload,
31
- };
32
- }),
33
- };
34
- });
35
- function checkArtifactOrder(artifacts, expectedOrder) {
36
- let outerIndex = 0;
37
- for (const artifact of artifacts) {
38
- try {
39
- // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown
40
- while (artifact.item_type != expectedOrder[outerIndex].itemType) {
41
- outerIndex++;
42
- }
43
- }
44
- catch (e) {
45
- console.error('Error finding artifact type in repos:', e);
46
- return false;
47
- }
48
- }
49
- return true;
50
- }
51
- describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => {
52
- let testAdapter;
53
- beforeEach(() => {
54
- // Create a fresh adapter instance for this test to avoid mocking conflicts
55
- const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
56
- payload: { event_type: types_1.EventType.StartExtractingData },
57
- });
58
- const mockAdapterState = new extraction_state_1.ExtractionState({
59
- event: mockEvent,
60
- initialState: { attachments: { completed: false } },
61
- });
62
- testAdapter = new extraction_adapter_1.ExtractionAdapter({
63
- event: mockEvent,
64
- adapterState: mockAdapterState,
65
- options: {
66
- batchSize: 50,
67
- },
68
- });
69
- });
70
- it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => {
71
- var _a, _b;
72
- const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
73
- // Initialize repos
74
- testAdapter.initializeRepos(repos);
75
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
76
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(105)));
77
- await testAdapter.uploadAllRepos();
78
- const artifacts = testAdapter.artifacts;
79
- expect(artifacts.length).toBe(4);
80
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
81
- });
82
- it('should work with more than 2 repos', async () => {
83
- var _a, _b, _c, _d;
84
- const repos = [
85
- { itemType: 'ItemTypeA' },
86
- { itemType: 'ItemTypeB' },
87
- { itemType: 'ItemTypeC' },
88
- { itemType: 'ItemTypeD' },
89
- ];
90
- // Initialize repos
91
- testAdapter.initializeRepos(repos);
92
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
93
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
94
- await ((_c = testAdapter.getRepo('ItemTypeC')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(103)));
95
- await ((_d = testAdapter.getRepo('ItemTypeD')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(104)));
96
- await testAdapter.uploadAllRepos();
97
- const artifacts = testAdapter.artifacts;
98
- expect(artifacts.length).toBe(12);
99
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
100
- });
101
- it('should maintain order with multiple pushes and uploads', async () => {
102
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
103
- const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
104
- // Initialize repos
105
- testAdapter.initializeRepos(repos);
106
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
107
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
108
- await ((_c = testAdapter.getRepo('ItemTypeA')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(101)));
109
- await ((_d = testAdapter.getRepo('ItemTypeB')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(102)));
110
- await ((_e = testAdapter.getRepo('ItemTypeA')) === null || _e === void 0 ? void 0 : _e.upload());
111
- await ((_f = testAdapter.getRepo('ItemTypeB')) === null || _f === void 0 ? void 0 : _f.upload());
112
- await ((_g = testAdapter.getRepo('ItemTypeA')) === null || _g === void 0 ? void 0 : _g.push((0, test_helpers_1.createItems)(101)));
113
- await ((_h = testAdapter.getRepo('ItemTypeB')) === null || _h === void 0 ? void 0 : _h.push((0, test_helpers_1.createItems)(102)));
114
- await ((_j = testAdapter.getRepo('ItemTypeA')) === null || _j === void 0 ? void 0 : _j.push((0, test_helpers_1.createItems)(101)));
115
- await ((_k = testAdapter.getRepo('ItemTypeB')) === null || _k === void 0 ? void 0 : _k.push((0, test_helpers_1.createItems)(102)));
116
- await testAdapter.uploadAllRepos();
117
- const artifacts = testAdapter.artifacts;
118
- expect(artifacts.length).toBe(20);
119
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
120
- });
121
- it('should not count artifacts if 0 items are pushed to the repo', async () => {
122
- var _a;
123
- const repos = [{ itemType: 'ItemTypeA' }];
124
- // Initialize repos
125
- testAdapter.initializeRepos(repos);
126
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push([]));
127
- await testAdapter.uploadAllRepos();
128
- const artifacts = testAdapter.artifacts;
129
- expect(artifacts.length).toBe(0);
130
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
131
- });
132
- });
133
- describe('External sync units splitting into artifacts', () => {
134
- let testAdapter;
135
- beforeEach(() => {
136
- const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
137
- payload: { event_type: types_1.EventType.StartExtractingExternalSyncUnits },
138
- });
139
- const mockAdapterState = new extraction_state_1.ExtractionState({
140
- event: mockEvent,
141
- initialState: {},
142
- });
143
- testAdapter = new extraction_adapter_1.ExtractionAdapter({
144
- event: mockEvent,
145
- adapterState: mockAdapterState,
146
- });
147
- });
148
- it('should split 125k external sync units into 5 artifacts', async () => {
149
- const BATCH_SIZE = 25000;
150
- const TOTAL_UNITS = 125000;
151
- const externalSyncUnits = Array.from({ length: TOTAL_UNITS }, (_, i) => ({
152
- id: String(i),
153
- name: `Unit ${i}`,
154
- description: `Description ${i}`,
155
- }));
156
- testAdapter.initializeRepos([
157
- {
158
- itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
159
- overridenOptions: {
160
- batchSize: BATCH_SIZE,
161
- skipConfirmation: true,
162
- },
163
- },
164
- ]);
165
- const repo = testAdapter.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS);
166
- const chunkSize = 10000;
167
- for (let i = 0; i < TOTAL_UNITS; i += chunkSize) {
168
- await (repo === null || repo === void 0 ? void 0 : repo.push(externalSyncUnits.slice(i, i + chunkSize)));
169
- }
170
- await testAdapter.uploadAllRepos();
171
- expect(testAdapter.artifacts.length).toBe(TOTAL_UNITS / BATCH_SIZE // 5
172
- );
173
- expect(testAdapter.artifacts.every((a) => a.item_type === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)).toBe(true);
174
- });
175
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=extraction-adapter.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"extraction-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.test.ts"],"names":[],"mappings":""}