@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/README.md +5 -11
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
  8. package/dist/common/constants.d.ts +4 -5
  9. package/dist/common/constants.d.ts.map +1 -1
  10. package/dist/common/constants.js +6 -7
  11. package/dist/common/helpers.d.ts +3 -35
  12. package/dist/common/helpers.d.ts.map +1 -1
  13. package/dist/common/helpers.js +5 -50
  14. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  15. package/dist/http/client.d.ts.map +1 -0
  16. package/dist/http/{axios-client-internal.js → client.js} +4 -8
  17. package/dist/index.d.ts +24 -13
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +43 -32
  20. package/dist/logger/logger.d.ts +12 -53
  21. package/dist/logger/logger.d.ts.map +1 -1
  22. package/dist/logger/logger.interfaces.d.ts +0 -1
  23. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  24. package/dist/logger/logger.js +36 -68
  25. package/dist/mappers/mappers.d.ts +7 -46
  26. package/dist/mappers/mappers.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.d.ts +175 -0
  28. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  29. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
  30. package/dist/mappers/mappers.js +11 -45
  31. package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
  32. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  33. package/dist/multithreading/adapters/base-adapter.js +97 -62
  34. package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
  35. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  36. package/dist/multithreading/adapters/extraction-adapter.js +203 -206
  37. package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
  38. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  41. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  42. package/dist/multithreading/adapters/loading-adapter.js +294 -346
  43. package/dist/multithreading/emit.d.ts +11 -0
  44. package/dist/multithreading/emit.d.ts.map +1 -0
  45. package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
  46. package/dist/multithreading/process-task.d.ts +2 -12
  47. package/dist/multithreading/process-task.d.ts.map +1 -1
  48. package/dist/multithreading/process-task.js +46 -43
  49. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  50. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
  52. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  53. package/dist/multithreading/spawn/spawn.helpers.js +128 -20
  54. package/dist/multithreading/spawn/spawn.js +24 -39
  55. package/dist/multithreading/worker.js +1 -4
  56. package/dist/repo/repo.d.ts +10 -0
  57. package/dist/repo/repo.d.ts.map +1 -1
  58. package/dist/repo/repo.helpers.d.ts +6 -0
  59. package/dist/repo/repo.helpers.d.ts.map +1 -0
  60. package/dist/repo/repo.helpers.js +16 -0
  61. package/dist/repo/repo.interfaces.d.ts +4 -16
  62. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  63. package/dist/repo/repo.js +23 -8
  64. package/dist/state/base-state.d.ts +16 -38
  65. package/dist/state/base-state.d.ts.map +1 -1
  66. package/dist/state/base-state.js +19 -41
  67. package/dist/state/extraction-state.d.ts +6 -21
  68. package/dist/state/extraction-state.d.ts.map +1 -1
  69. package/dist/state/extraction-state.js +8 -25
  70. package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
  71. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  72. package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
  73. package/dist/state/loading-state.d.ts +3 -14
  74. package/dist/state/loading-state.d.ts.map +1 -1
  75. package/dist/state/loading-state.js +3 -14
  76. package/dist/state/state.d.ts +4 -0
  77. package/dist/state/state.d.ts.map +1 -0
  78. package/dist/state/state.interfaces.d.ts +33 -37
  79. package/dist/state/state.interfaces.d.ts.map +1 -1
  80. package/dist/state/state.interfaces.js +16 -11
  81. package/dist/state/state.js +11 -0
  82. package/dist/state/time-value-resolver.d.ts +17 -0
  83. package/dist/state/time-value-resolver.d.ts.map +1 -0
  84. package/dist/{common → state}/time-value-resolver.js +9 -41
  85. package/dist/testing/mock-event.d.ts +10 -0
  86. package/dist/testing/mock-event.d.ts.map +1 -0
  87. package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
  88. package/dist/testing/mock-server.d.ts +29 -0
  89. package/dist/testing/mock-server.d.ts.map +1 -0
  90. package/dist/testing/mock-server.interfaces.d.ts +48 -0
  91. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  92. package/dist/{mock-server → testing}/mock-server.js +21 -33
  93. package/dist/types/common.d.ts +1 -12
  94. package/dist/types/common.d.ts.map +1 -1
  95. package/dist/types/common.js +1 -6
  96. package/dist/types/errors.d.ts +9 -0
  97. package/dist/types/errors.d.ts.map +1 -0
  98. package/dist/types/external-domain-metadata.d.ts +30 -171
  99. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  100. package/dist/types/extraction.d.ts +64 -94
  101. package/dist/types/extraction.d.ts.map +1 -1
  102. package/dist/types/extraction.js +6 -34
  103. package/dist/types/loading.d.ts +1 -2
  104. package/dist/types/loading.d.ts.map +1 -1
  105. package/dist/types/loading.js +0 -1
  106. package/dist/types/workers.d.ts +39 -94
  107. package/dist/types/workers.d.ts.map +1 -1
  108. package/dist/types/workers.js +0 -6
  109. package/dist/uploader/uploader.d.ts +8 -59
  110. package/dist/uploader/uploader.d.ts.map +1 -1
  111. package/dist/uploader/uploader.helpers.d.ts +4 -27
  112. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  113. package/dist/uploader/uploader.helpers.js +64 -30
  114. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  115. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  116. package/dist/uploader/uploader.interfaces.js +8 -0
  117. package/dist/uploader/uploader.js +23 -89
  118. package/package.json +8 -6
  119. package/dist/airsync-sdk.d.ts +0 -2059
  120. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  121. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  122. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
  123. package/dist/common/control-protocol.d.ts +0 -10
  124. package/dist/common/control-protocol.d.ts.map +0 -1
  125. package/dist/common/errors.d.ts +0 -6
  126. package/dist/common/errors.d.ts.map +0 -1
  127. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  128. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  129. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  130. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  131. package/dist/common/test-utils.d.ts +0 -19
  132. package/dist/common/test-utils.d.ts.map +0 -1
  133. package/dist/common/time-value-resolver.d.ts +0 -47
  134. package/dist/common/time-value-resolver.d.ts.map +0 -1
  135. package/dist/common/time-value-resolver.test.d.ts +0 -2
  136. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  137. package/dist/common/time-value-resolver.test.js +0 -268
  138. package/dist/http/axios-client-internal.d.ts.map +0 -1
  139. package/dist/http/axios-client-internal.test.d.ts +0 -2
  140. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  141. package/dist/http/axios-client-internal.test.js +0 -175
  142. package/dist/http/axios-client.d.ts +0 -27
  143. package/dist/http/axios-client.d.ts.map +0 -1
  144. package/dist/http/axios-client.js +0 -60
  145. package/dist/http/constants.d.ts +0 -4
  146. package/dist/http/constants.d.ts.map +0 -1
  147. package/dist/http/constants.js +0 -6
  148. package/dist/http/index.d.ts +0 -2
  149. package/dist/http/index.d.ts.map +0 -1
  150. package/dist/http/index.js +0 -17
  151. package/dist/logger/logger.constants.d.ts +0 -6
  152. package/dist/logger/logger.constants.d.ts.map +0 -1
  153. package/dist/logger/logger.constants.js +0 -13
  154. package/dist/logger/logger.context.d.ts +0 -58
  155. package/dist/logger/logger.context.d.ts.map +0 -1
  156. package/dist/logger/logger.context.js +0 -86
  157. package/dist/logger/logger.test.d.ts +0 -2
  158. package/dist/logger/logger.test.d.ts.map +0 -1
  159. package/dist/logger/logger.test.js +0 -537
  160. package/dist/mappers/mappers.interface.d.ts +0 -294
  161. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  162. package/dist/mappers/mappers.test.d.ts +0 -2
  163. package/dist/mappers/mappers.test.d.ts.map +0 -1
  164. package/dist/mappers/mappers.test.js +0 -105
  165. package/dist/mock-server/mock-server.d.ts +0 -47
  166. package/dist/mock-server/mock-server.d.ts.map +0 -1
  167. package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
  168. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  169. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
  170. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
  171. package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
  172. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
  173. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
  174. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
  175. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
  176. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
  177. package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
  178. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
  179. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
  180. package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
  181. package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
  182. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
  183. package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
  184. package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
  185. package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
  186. package/dist/multithreading/adapters/serialization.test.js +0 -71
  187. package/dist/multithreading/create-worker.test.d.ts +0 -2
  188. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  189. package/dist/multithreading/create-worker.test.js +0 -108
  190. package/dist/multithreading/process-task.test.d.ts +0 -2
  191. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  192. package/dist/multithreading/process-task.test.js +0 -146
  193. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  194. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  195. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  196. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  197. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  198. package/dist/multithreading/spawn/spawn.test.js +0 -232
  199. package/dist/repo/repo.test.d.ts +0 -2
  200. package/dist/repo/repo.test.d.ts.map +0 -1
  201. package/dist/repo/repo.test.js +0 -184
  202. package/dist/state/state.extract-window.test.d.ts +0 -2
  203. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  204. package/dist/state/state.extract-window.test.js +0 -163
  205. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  206. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  207. package/dist/state/state.pending-boundaries.test.js +0 -189
  208. package/dist/state/state.post-state.test.d.ts +0 -2
  209. package/dist/state/state.post-state.test.d.ts.map +0 -1
  210. package/dist/state/state.post-state.test.js +0 -78
  211. package/dist/state/state.test.d.ts +0 -2
  212. package/dist/state/state.test.d.ts.map +0 -1
  213. package/dist/state/state.test.js +0 -271
  214. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  215. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  216. package/dist/state/state.time-value-resolution.test.js +0 -175
  217. package/dist/tsdoc-metadata.json +0 -11
  218. package/dist/types/extraction.test.d.ts +0 -2
  219. package/dist/types/extraction.test.d.ts.map +0 -1
  220. package/dist/types/extraction.test.js +0 -96
  221. package/dist/types/index.d.ts +0 -10
  222. package/dist/types/index.d.ts.map +0 -1
  223. package/dist/types/index.js +0 -21
  224. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  225. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  226. package/dist/uploader/uploader.helpers.test.js +0 -256
  227. package/dist/uploader/uploader.test.d.ts +0 -2
  228. package/dist/uploader/uploader.test.d.ts.map +0 -1
  229. package/dist/uploader/uploader.test.js +0 -601
  230. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  231. /package/dist/{common → types}/errors.js +0 -0
@@ -1,789 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
4
- const extraction_state_1 = require("../../state/extraction-state");
5
- const jest_setup_1 = require("../../tests/jest.setup");
6
- const test_utils_1 = require("../../common/test-utils");
7
- const types_1 = require("../../types");
8
- const extraction_adapter_1 = require("./extraction-adapter");
9
- /* eslint-disable @typescript-eslint/no-require-imports */
10
- jest.mock('../../common/control-protocol', () => ({
11
- emit: jest.fn().mockResolvedValue({}),
12
- }));
13
- jest.mock('../../mappers/mappers');
14
- jest.mock('../../uploader/uploader');
15
- jest.mock('../../repo/repo');
16
- jest.mock('node:worker_threads', () => ({
17
- parentPort: { postMessage: jest.fn() },
18
- }));
19
- jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
20
- AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
21
- streamAll: jest.fn().mockResolvedValue(undefined),
22
- })),
23
- }));
24
- function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
25
- const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
26
- payload: { event_type: eventType },
27
- });
28
- const initialState = {
29
- attachments: { completed: false },
30
- };
31
- const adapterState = new extraction_state_1.ExtractionState({ event, initialState });
32
- adapterState.sdkState.snapInVersionId = '';
33
- adapterState.sdkState.toDevRev = {
34
- attachmentsMetadata: {
35
- artifactIds: [],
36
- lastProcessed: 0,
37
- lastProcessedAttachmentsIdsList: [],
38
- },
39
- };
40
- const adapter = new extraction_adapter_1.ExtractionAdapter({ event, adapterState });
41
- return { adapter, event, adapterState };
42
- }
43
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.streamAttachments`, () => {
44
- let adapter;
45
- let adapterState;
46
- beforeEach(() => {
47
- jest.clearAllMocks();
48
- ({ adapter, adapterState } = makeAdapter());
49
- });
50
- it('should process all artifact batches successfully', async () => {
51
- // Arrange
52
- const mockStream = jest.fn();
53
- adapterState.sdkState.toDevRev = {
54
- attachmentsMetadata: {
55
- artifactIds: ['artifact1', 'artifact2'],
56
- lastProcessed: 0,
57
- lastProcessedAttachmentsIdsList: [],
58
- },
59
- };
60
- adapter['uploader'].getAttachmentsFromArtifactId = jest
61
- .fn()
62
- .mockResolvedValueOnce({
63
- attachments: [
64
- {
65
- url: 'http://example.com/file1.pdf',
66
- id: 'attachment1',
67
- file_name: 'file1.pdf',
68
- parent_id: 'parent1',
69
- },
70
- {
71
- url: 'http://example.com/file2.pdf',
72
- id: 'attachment2',
73
- file_name: 'file2.pdf',
74
- parent_id: 'parent2',
75
- },
76
- ],
77
- })
78
- .mockResolvedValueOnce({
79
- attachments: [
80
- {
81
- url: 'http://example.com/file3.pdf',
82
- id: 'attachment3',
83
- file_name: 'file3.pdf',
84
- parent_id: 'parent3',
85
- },
86
- ],
87
- });
88
- adapter.initializeRepos = jest.fn();
89
- // Act
90
- const result = await adapter.streamAttachments({
91
- stream: mockStream,
92
- });
93
- // Assert
94
- expect(adapter.initializeRepos).toHaveBeenCalledWith([
95
- { itemType: 'ssor_attachment' },
96
- ]);
97
- expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
98
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
99
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
100
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
101
- expect(result).toBeUndefined();
102
- });
103
- it('[edge] should handle invalid batch size by using 1 instead', async () => {
104
- // Arrange
105
- const mockStream = jest.fn();
106
- adapterState.sdkState.toDevRev = {
107
- attachmentsMetadata: {
108
- artifactIds: ['artifact1'],
109
- lastProcessed: 0,
110
- lastProcessedAttachmentsIdsList: [],
111
- },
112
- };
113
- adapter['uploader'].getAttachmentsFromArtifactId = jest
114
- .fn()
115
- .mockResolvedValue({
116
- attachments: [
117
- {
118
- url: 'http://example.com/file1.pdf',
119
- id: 'attachment1',
120
- file_name: 'file1.pdf',
121
- parent_id: 'parent1',
122
- },
123
- ],
124
- });
125
- adapter.initializeRepos = jest.fn();
126
- // Act
127
- const result = await adapter.streamAttachments({
128
- stream: mockStream,
129
- batchSize: 0,
130
- });
131
- // Assert
132
- expect(result).toBeUndefined();
133
- });
134
- it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
135
- // Arrange
136
- const mockStream = jest.fn();
137
- adapterState.sdkState.toDevRev = {
138
- attachmentsMetadata: {
139
- artifactIds: ['artifact1'],
140
- lastProcessed: 0,
141
- lastProcessedAttachmentsIdsList: [],
142
- },
143
- };
144
- adapter['uploader'].getAttachmentsFromArtifactId = jest
145
- .fn()
146
- .mockResolvedValue({
147
- attachments: [
148
- {
149
- url: 'http://example.com/file1.pdf',
150
- id: 'attachment1',
151
- file_name: 'file1.pdf',
152
- parent_id: 'parent1',
153
- },
154
- ],
155
- });
156
- adapter.initializeRepos = jest.fn();
157
- // Act
158
- const result = await adapter.streamAttachments({
159
- stream: mockStream,
160
- batchSize: 100,
161
- });
162
- // Assert
163
- expect(result).toBeUndefined();
164
- });
165
- it('[edge] should handle empty attachments metadata artifact IDs', async () => {
166
- // Arrange
167
- const mockStream = jest.fn();
168
- adapterState.sdkState.toDevRev = {
169
- attachmentsMetadata: {
170
- artifactIds: [],
171
- lastProcessed: 0,
172
- },
173
- };
174
- // Act
175
- const result = await adapter.streamAttachments({
176
- stream: mockStream,
177
- });
178
- // Assert
179
- expect(result).toBeUndefined();
180
- });
181
- it('[edge] should handle errors when getting attachments', async () => {
182
- // Arrange
183
- const mockStream = jest.fn();
184
- adapterState.sdkState.toDevRev = {
185
- attachmentsMetadata: {
186
- artifactIds: ['artifact1'],
187
- lastProcessed: 0,
188
- lastProcessedAttachmentsIdsList: [],
189
- },
190
- };
191
- const mockError = new Error('Failed to get attachments');
192
- adapter['uploader'].getAttachmentsFromArtifactId = jest
193
- .fn()
194
- .mockResolvedValue({
195
- error: mockError,
196
- });
197
- adapter.initializeRepos = jest.fn();
198
- // Act
199
- const result = await adapter.streamAttachments({
200
- stream: mockStream,
201
- });
202
- // Assert
203
- expect(result).toEqual({
204
- error: mockError,
205
- });
206
- });
207
- it('[edge] should handle empty attachments array from artifact', async () => {
208
- // Arrange
209
- const mockStream = jest.fn();
210
- adapterState.sdkState.toDevRev = {
211
- attachmentsMetadata: {
212
- artifactIds: ['artifact1'],
213
- lastProcessed: 0,
214
- lastProcessedAttachmentsIdsList: [],
215
- },
216
- };
217
- adapter['uploader'].getAttachmentsFromArtifactId = jest
218
- .fn()
219
- .mockResolvedValue({
220
- attachments: [],
221
- });
222
- adapter.initializeRepos = jest.fn();
223
- // Act
224
- const result = await adapter.streamAttachments({
225
- stream: mockStream,
226
- });
227
- // Assert
228
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
229
- expect(result).toBeUndefined();
230
- });
231
- it('should use custom processors when provided', async () => {
232
- // Arrange
233
- const mockStream = jest.fn();
234
- const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
235
- const mockIterator = jest.fn().mockResolvedValue({});
236
- adapterState.sdkState.toDevRev = {
237
- attachmentsMetadata: {
238
- artifactIds: ['artifact1'],
239
- lastProcessed: 0,
240
- lastProcessedAttachmentsIdsList: [],
241
- },
242
- };
243
- adapter['uploader'].getAttachmentsFromArtifactId = jest
244
- .fn()
245
- .mockResolvedValue({
246
- attachments: [{ id: 'attachment1' }],
247
- });
248
- adapter.initializeRepos = jest.fn();
249
- // Act
250
- const result = await adapter.streamAttachments({
251
- stream: mockStream,
252
- processors: {
253
- reducer: mockReducer,
254
- iterator: mockIterator,
255
- },
256
- });
257
- // Assert
258
- expect(mockReducer).toHaveBeenCalledWith({
259
- attachments: [{ id: 'attachment1' }],
260
- adapter: adapter,
261
- batchSize: 1,
262
- });
263
- expect(mockIterator).toHaveBeenCalledWith({
264
- reducedAttachments: ['custom-reduced'],
265
- adapter: adapter,
266
- stream: mockStream,
267
- });
268
- expect(result).toBeUndefined();
269
- });
270
- it('should handle rate limiting from iterator', async () => {
271
- // Arrange
272
- const mockStream = jest.fn();
273
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
274
- streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
275
- }));
276
- adapterState.sdkState.toDevRev = {
277
- attachmentsMetadata: {
278
- artifactIds: ['artifact1'],
279
- lastProcessed: 0,
280
- lastProcessedAttachmentsIdsList: [],
281
- },
282
- };
283
- adapter['uploader'].getAttachmentsFromArtifactId = jest
284
- .fn()
285
- .mockResolvedValue({
286
- attachments: [{ id: 'attachment1' }],
287
- });
288
- adapter.initializeRepos = jest.fn();
289
- // Act
290
- const result = await adapter.streamAttachments({
291
- stream: mockStream,
292
- });
293
- // Assert
294
- expect(result).toEqual({ delay: 30 });
295
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
296
- });
297
- it('should handle error from iterator', async () => {
298
- // Arrange
299
- const mockStream = jest.fn();
300
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
301
- streamAll: jest.fn().mockResolvedValue({
302
- error: 'Mock error',
303
- }),
304
- }));
305
- adapterState.sdkState.toDevRev = {
306
- attachmentsMetadata: {
307
- artifactIds: ['artifact1'],
308
- lastProcessed: 0,
309
- lastProcessedAttachmentsIdsList: [],
310
- },
311
- };
312
- adapter['uploader'].getAttachmentsFromArtifactId = jest
313
- .fn()
314
- .mockResolvedValue({
315
- attachments: [{ id: 'attachment1' }],
316
- });
317
- adapter.initializeRepos = jest.fn();
318
- // Act
319
- const result = await adapter.streamAttachments({
320
- stream: mockStream,
321
- });
322
- // Assert
323
- expect(result).toEqual({ error: 'Mock error' });
324
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
325
- });
326
- it('should emit progress event and exit process on timeout, preserving state for resumption', async () => {
327
- // Arrange
328
- const mockStream = jest.fn();
329
- const exitSpy = jest
330
- .spyOn(process, 'exit')
331
- .mockImplementation(() => undefined);
332
- adapterState.sdkState.toDevRev = {
333
- attachmentsMetadata: {
334
- artifactIds: ['artifact1', 'artifact2', 'artifact3'],
335
- lastProcessed: 0,
336
- lastProcessedAttachmentsIdsList: [],
337
- },
338
- };
339
- adapter['uploader'].getAttachmentsFromArtifactId = jest
340
- .fn()
341
- .mockResolvedValue({
342
- attachments: [
343
- {
344
- url: 'http://example.com/file1.pdf',
345
- id: 'attachment1',
346
- file_name: 'file1.pdf',
347
- parent_id: 'parent1',
348
- },
349
- ],
350
- });
351
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
352
- streamAll: jest.fn().mockImplementation(() => {
353
- adapter.isTimeout = true;
354
- return {};
355
- }),
356
- }));
357
- adapter.initializeRepos = jest.fn();
358
- const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
359
- // Act
360
- await adapter.streamAttachments({
361
- stream: mockStream,
362
- });
363
- // Assert
364
- expect(emitSpy).toHaveBeenCalledWith(types_1.ExtractorEventType.AttachmentExtractionProgress);
365
- expect(exitSpy).toHaveBeenCalledWith(0);
366
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1', 'artifact2', 'artifact3']);
367
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(1);
368
- exitSpy.mockRestore();
369
- });
370
- it('should stop after the timeout flips between batches and preserve unprocessed artifacts for resumption', async () => {
371
- // Arrange: three artifacts. The first batch's streamAll completes
372
- // successfully; the second sets isTimeout=true mid-run. The third batch
373
- // must never be reached.
374
- const mockStream = jest.fn();
375
- const exitSpy = jest
376
- .spyOn(process, 'exit')
377
- .mockImplementation(() => undefined);
378
- adapterState.sdkState.toDevRev = {
379
- attachmentsMetadata: {
380
- artifactIds: ['artifact1', 'artifact2', 'artifact3'],
381
- lastProcessed: 0,
382
- lastProcessedAttachmentsIdsList: [],
383
- },
384
- };
385
- adapter['uploader'].getAttachmentsFromArtifactId = jest
386
- .fn()
387
- .mockResolvedValue({
388
- attachments: [
389
- {
390
- url: 'http://example.com/file.pdf',
391
- id: 'attachment-x',
392
- file_name: 'file.pdf',
393
- parent_id: 'parent-x',
394
- },
395
- ],
396
- });
397
- // First call: clean streamAll. Second call: flip isTimeout AFTER streaming.
398
- attachments_streaming_pool_1.AttachmentsStreamingPool
399
- .mockImplementationOnce(() => ({
400
- streamAll: jest.fn().mockResolvedValue({}),
401
- }))
402
- .mockImplementationOnce(() => ({
403
- streamAll: jest.fn().mockImplementation(() => {
404
- adapter.isTimeout = true;
405
- return {};
406
- }),
407
- }));
408
- adapter.initializeRepos = jest.fn();
409
- const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
410
- // Act
411
- await adapter.streamAttachments({ stream: mockStream });
412
- // Assert
413
- // - Fetched attachments for the first two artifacts only; the third never ran
414
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
415
- // - Progress emitted and process.exit(0) called once the timeout was detected
416
- expect(emitSpy).toHaveBeenCalledWith(types_1.ExtractorEventType.AttachmentExtractionProgress);
417
- expect(exitSpy).toHaveBeenCalledWith(0);
418
- // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught
419
- // before its shift) along with the untouched artifact 3
420
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact2', 'artifact3']);
421
- exitSpy.mockRestore();
422
- });
423
- it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
424
- // Arrange
425
- const mockStream = jest.fn();
426
- adapterState.sdkState.toDevRev = {
427
- attachmentsMetadata: {
428
- artifactIds: ['artifact1'],
429
- lastProcessed: 0,
430
- lastProcessedAttachmentsIdsList: [],
431
- },
432
- };
433
- adapter['uploader'].getAttachmentsFromArtifactId = jest
434
- .fn()
435
- .mockResolvedValueOnce({
436
- attachments: [
437
- {
438
- url: 'http://example.com/file1.pdf',
439
- id: 'attachment1',
440
- file_name: 'file1.pdf',
441
- parent_id: 'parent1',
442
- },
443
- {
444
- url: 'http://example.com/file2.pdf',
445
- id: 'attachment2',
446
- file_name: 'file2.pdf',
447
- parent_id: 'parent2',
448
- },
449
- {
450
- url: 'http://example.com/file3.pdf',
451
- id: 'attachment3',
452
- file_name: 'file3.pdf',
453
- parent_id: 'parent3',
454
- },
455
- ],
456
- });
457
- adapter.processAttachment = jest.fn().mockResolvedValue(null);
458
- // Act
459
- await adapter.streamAttachments({
460
- stream: mockStream,
461
- });
462
- // Assert
463
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
464
- expect(adapterState.sdkState.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
465
- });
466
- });
467
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.processAttachment`, () => {
468
- let adapter;
469
- beforeEach(() => {
470
- jest.clearAllMocks();
471
- ({ adapter } = makeAdapter(types_1.EventType.StartExtractingAttachments));
472
- });
473
- afterEach(() => {
474
- jest.restoreAllMocks();
475
- });
476
- const createMockHttpStream = (headers = {}) => ({
477
- headers,
478
- data: { destroy: jest.fn() },
479
- });
480
- const makeAttachment = (overrides = {}) => (Object.assign({ id: 'att-1', url: 'https://example.com/file.pdf', file_name: 'file.pdf', parent_id: 'parent-1', content_type: 'application/pdf' }, overrides));
481
- function setupUploaderHappyPath() {
482
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
483
- response: {
484
- artifact_id: 'art_1',
485
- upload_url: 'https://upload',
486
- form_data: [],
487
- },
488
- });
489
- adapter['uploader'].streamArtifact = jest
490
- .fn()
491
- .mockResolvedValue({ response: {} });
492
- adapter['uploader'].confirmArtifactUpload = jest
493
- .fn()
494
- .mockResolvedValue({ response: {} });
495
- const pushMock = jest.fn().mockResolvedValue(undefined);
496
- adapter.getRepo = jest.fn().mockReturnValue({ push: pushMock });
497
- return pushMock;
498
- }
499
- // ---- content-type resolution ----
500
- it('should use attachment.content_type when provided, ignoring HTTP header', async () => {
501
- // Arrange
502
- setupUploaderHappyPath();
503
- const mockStream = jest.fn().mockResolvedValue({
504
- httpStream: createMockHttpStream({
505
- 'content-type': 'text/plain',
506
- 'content-length': '100',
507
- }),
508
- });
509
- // Act
510
- await adapter.processAttachment(makeAttachment({ content_type: 'application/pdf' }), mockStream);
511
- // Assert
512
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('file.pdf', 'application/pdf', 100);
513
- });
514
- it('should use HTTP header content-type when attachment.content_type is not set', async () => {
515
- // Arrange
516
- setupUploaderHappyPath();
517
- const mockStream = jest.fn().mockResolvedValue({
518
- httpStream: createMockHttpStream({
519
- 'content-type': 'image/jpeg',
520
- 'content-length': '200',
521
- }),
522
- });
523
- const attachment = {
524
- id: 'att-2',
525
- url: 'https://example.com/photo.jpg',
526
- file_name: 'photo.jpg',
527
- parent_id: 'parent-2',
528
- };
529
- // Act
530
- await adapter.processAttachment(attachment, mockStream);
531
- // Assert
532
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('photo.jpg', 'image/jpeg', 200);
533
- });
534
- it('should fall back to application/octet-stream when neither content_type nor HTTP header is set', async () => {
535
- // Arrange
536
- setupUploaderHappyPath();
537
- const mockStream = jest.fn().mockResolvedValue({
538
- httpStream: createMockHttpStream({}),
539
- });
540
- const attachment = {
541
- id: 'att-3',
542
- url: 'https://example.com/file.bin',
543
- file_name: 'file.bin',
544
- parent_id: 'parent-3',
545
- };
546
- // Act
547
- await adapter.processAttachment(attachment, mockStream);
548
- // Assert
549
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('file.bin', 'application/octet-stream', undefined);
550
- });
551
- // ---- error paths ----
552
- it('should return the stream error directly when the stream function returns an error', async () => {
553
- // Arrange
554
- const stream = jest
555
- .fn()
556
- .mockResolvedValue({ error: new Error('stream failed') });
557
- // Act
558
- const result = await adapter.processAttachment(makeAttachment(), stream);
559
- // Assert
560
- expect(result === null || result === void 0 ? void 0 : result.error).toBeDefined();
561
- });
562
- it('should propagate a rate-limit delay from the stream function', async () => {
563
- // Arrange
564
- const stream = jest.fn().mockResolvedValue({ delay: 5 });
565
- // Act
566
- const result = await adapter.processAttachment(makeAttachment(), stream);
567
- // Assert
568
- expect(result === null || result === void 0 ? void 0 : result.delay).toBe(5);
569
- });
570
- it('should return an error containing the attachment ID when getArtifactUploadUrl fails', async () => {
571
- var _a, _b;
572
- // Arrange
573
- const stream = jest
574
- .fn()
575
- .mockResolvedValue({ httpStream: createMockHttpStream() });
576
- adapter['uploader'].getArtifactUploadUrl = jest
577
- .fn()
578
- .mockResolvedValue({ error: new Error('upload url failed') });
579
- // Act
580
- const result = await adapter.processAttachment(makeAttachment(), stream);
581
- // Assert
582
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('att-1');
583
- expect((_b = result === null || result === void 0 ? void 0 : result.error) === null || _b === void 0 ? void 0 : _b.message).toContain('preparing artifact');
584
- });
585
- it('should return an error when streamArtifact fails', async () => {
586
- var _a;
587
- // Arrange
588
- const stream = jest
589
- .fn()
590
- .mockResolvedValue({ httpStream: createMockHttpStream() });
591
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
592
- response: {
593
- artifact_id: 'art-1',
594
- upload_url: 'https://upload',
595
- form_data: [],
596
- },
597
- });
598
- adapter['uploader'].streamArtifact = jest
599
- .fn()
600
- .mockResolvedValue({ error: new Error('stream failed') });
601
- // Act
602
- const result = await adapter.processAttachment(makeAttachment(), stream);
603
- // Assert
604
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('streaming to artifact');
605
- });
606
- it('should return an error when confirmArtifactUpload fails', async () => {
607
- var _a;
608
- // Arrange
609
- const stream = jest
610
- .fn()
611
- .mockResolvedValue({ httpStream: createMockHttpStream() });
612
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
613
- response: {
614
- artifact_id: 'art-1',
615
- upload_url: 'https://upload',
616
- form_data: [],
617
- },
618
- });
619
- adapter['uploader'].streamArtifact = jest
620
- .fn()
621
- .mockResolvedValue({ response: {} });
622
- adapter['uploader'].confirmArtifactUpload = jest
623
- .fn()
624
- .mockResolvedValue({ error: new Error('confirm failed') });
625
- // Act
626
- const result = await adapter.processAttachment(makeAttachment(), stream);
627
- // Assert
628
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('confirming upload');
629
- });
630
- it.each([
631
- { inline: true, expected: true },
632
- { inline: false, expected: false },
633
- ])('should set inline=$expected on the ssorAttachment when attachment.inline=$inline', async ({ inline, expected }) => {
634
- // Arrange
635
- const pushMock = setupUploaderHappyPath();
636
- const stream = jest
637
- .fn()
638
- .mockResolvedValue({ httpStream: createMockHttpStream() });
639
- // Act
640
- await adapter.processAttachment(makeAttachment({ inline }), stream);
641
- // Assert
642
- const ssorItem = pushMock.mock.calls[0][0][0];
643
- expect(ssorItem.inline).toBe(expected);
644
- });
645
- it('should return a descriptive error when the stream function returns no httpStream', async () => {
646
- var _a;
647
- // Arrange
648
- const stream = jest.fn().mockResolvedValue({ httpStream: null });
649
- // Act
650
- const result = await adapter.processAttachment(makeAttachment(), stream);
651
- // Assert
652
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Error while opening attachment stream');
653
- });
654
- });
655
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.initializeRepos — event size threshold`, () => {
656
- it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => {
657
- // Arrange
658
- const { adapter } = makeAdapter();
659
- let capturedOnUpload;
660
- const { Repo } = require('../../repo/repo');
661
- Repo.mockImplementationOnce((opts) => {
662
- capturedOnUpload = opts.onUpload;
663
- return { itemType: 'issues', upload: jest.fn(), uploadedArtifacts: [] };
664
- });
665
- // Act
666
- adapter.initializeRepos([{ itemType: 'issues' }]);
667
- expect(capturedOnUpload).toBeDefined();
668
- capturedOnUpload({
669
- id: 'artifact-x',
670
- item_count: 1,
671
- item_type: 'x'.repeat(200000),
672
- });
673
- // Assert
674
- expect(adapter.isTimeout).toBe(true);
675
- });
676
- });
677
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.getRepo`, () => {
678
- it('should return undefined when the requested repo was never initialised', () => {
679
- // Arrange
680
- const { adapter } = makeAdapter();
681
- // Act
682
- const result = adapter.getRepo('non-existent-type');
683
- // Assert
684
- expect(result).toBeUndefined();
685
- });
686
- });
687
- describe(`${extraction_adapter_1.ExtractionAdapter.name}.destroyHttpStream`, () => {
688
- let adapter;
689
- beforeEach(() => {
690
- ({ adapter } = makeAdapter());
691
- });
692
- afterEach(() => {
693
- jest.restoreAllMocks();
694
- });
695
- it.each([
696
- {
697
- label: 'calls destroy() when available',
698
- data: { destroy: jest.fn(), close: jest.fn() },
699
- expectDestroy: true,
700
- expectClose: false,
701
- },
702
- {
703
- label: 'calls close() when destroy is not present',
704
- data: { close: jest.fn() },
705
- expectDestroy: false,
706
- expectClose: true,
707
- },
708
- {
709
- label: 'does not throw when neither method is present',
710
- data: {},
711
- expectDestroy: false,
712
- expectClose: false,
713
- },
714
- {
715
- label: 'does not throw when data is null',
716
- data: null,
717
- expectDestroy: false,
718
- expectClose: false,
719
- },
720
- ])('$label', ({ data, expectDestroy, expectClose }) => {
721
- // Arrange
722
- const httpStream = { data };
723
- // Act & Assert
724
- expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow();
725
- if (expectDestroy) {
726
- expect(data.destroy).toHaveBeenCalled();
727
- }
728
- if (expectClose) {
729
- expect(data.close).toHaveBeenCalled();
730
- }
731
- });
732
- it('should not re-throw when destroy() itself throws', () => {
733
- // Arrange
734
- const httpStream = {
735
- data: {
736
- destroy: () => {
737
- throw new Error('stream error');
738
- },
739
- },
740
- };
741
- // Act & Assert
742
- expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow();
743
- });
744
- });
745
- describe(`${extraction_adapter_1.ExtractionAdapter.name} — extractionScope`, () => {
746
- it('should return empty object by default', () => {
747
- const { adapter } = makeAdapter();
748
- expect(adapter.extractionScope).toEqual({});
749
- });
750
- it('should return extraction scope from adapter state', () => {
751
- const { adapter, adapterState } = makeAdapter();
752
- const extractionScope = {
753
- tasks: { extract: true },
754
- users: { extract: false },
755
- };
756
- adapterState._extractionScope = extractionScope;
757
- expect(adapter.extractionScope).toEqual(extractionScope);
758
- });
759
- });
760
- describe(`${extraction_adapter_1.ExtractionAdapter.name} — shouldExtract`, () => {
761
- it('should return true when extraction scope is empty', () => {
762
- const { adapter } = makeAdapter();
763
- expect(adapter.shouldExtract('tasks')).toBe(true);
764
- expect(adapter.shouldExtract('users')).toBe(true);
765
- });
766
- it('should return true when item type is not in scope', () => {
767
- const { adapter, adapterState } = makeAdapter();
768
- adapterState._extractionScope = {
769
- tasks: { extract: true },
770
- };
771
- expect(adapter.shouldExtract('users')).toBe(true);
772
- });
773
- it('should return true when item type has extract: true', () => {
774
- const { adapter, adapterState } = makeAdapter();
775
- adapterState._extractionScope = {
776
- tasks: { extract: true },
777
- };
778
- expect(adapter.shouldExtract('tasks')).toBe(true);
779
- });
780
- it('should return false when item type has extract: false', () => {
781
- const { adapter, adapterState } = makeAdapter();
782
- adapterState._extractionScope = {
783
- tasks: { extract: false },
784
- users: { extract: true },
785
- };
786
- expect(adapter.shouldExtract('tasks')).toBe(false);
787
- expect(adapter.shouldExtract('users')).toBe(true);
788
- });
789
- });