@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +5 -11
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +1 -8
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +4 -3
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +11 -39
  7. package/dist/common/constants.d.ts +1 -0
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +2 -1
  10. package/dist/common/helpers.d.ts +0 -14
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +0 -36
  13. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  14. package/dist/http/client.d.ts.map +1 -0
  15. package/dist/index.d.ts +25 -13
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +51 -30
  18. package/dist/logger/logger.d.ts +14 -0
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.js +55 -8
  21. package/dist/mappers/mappers.d.ts +9 -10
  22. package/dist/mappers/mappers.d.ts.map +1 -1
  23. package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
  24. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  25. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
  26. package/dist/mappers/mappers.js +13 -9
  27. package/dist/multithreading/adapters/base-adapter.d.ts +71 -0
  28. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  29. package/dist/multithreading/adapters/base-adapter.js +138 -0
  30. package/dist/multithreading/adapters/extraction-adapter.d.ts +57 -0
  31. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  32. package/dist/multithreading/adapters/extraction-adapter.js +323 -0
  33. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  34. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  35. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  36. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  37. package/dist/multithreading/adapters/loading-adapter.js +431 -0
  38. package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
  39. package/dist/multithreading/emit.d.ts.map +1 -0
  40. package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
  41. package/dist/multithreading/process-task.d.ts +16 -1
  42. package/dist/multithreading/process-task.d.ts.map +1 -1
  43. package/dist/multithreading/process-task.js +115 -50
  44. package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
  45. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  46. package/dist/multithreading/spawn/spawn.helpers.js +152 -2
  47. package/dist/multithreading/spawn/spawn.js +7 -7
  48. package/dist/multithreading/worker.js +1 -2
  49. package/dist/repo/repo.js +2 -2
  50. package/dist/state/base-state.d.ts +75 -0
  51. package/dist/state/base-state.d.ts.map +1 -0
  52. package/dist/state/base-state.js +261 -0
  53. package/dist/state/extraction-state.d.ts +29 -0
  54. package/dist/state/extraction-state.d.ts.map +1 -0
  55. package/dist/state/extraction-state.js +122 -0
  56. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  57. package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
  58. package/dist/state/loading-state.d.ts +18 -0
  59. package/dist/state/loading-state.d.ts.map +1 -0
  60. package/dist/state/loading-state.js +39 -0
  61. package/dist/state/state.d.ts +3 -35
  62. package/dist/state/state.d.ts.map +1 -1
  63. package/dist/state/state.interfaces.d.ts +20 -3
  64. package/dist/state/state.interfaces.d.ts.map +1 -1
  65. package/dist/state/state.interfaces.js +19 -1
  66. package/dist/state/state.js +9 -263
  67. package/dist/{common → state}/time-value-resolver.d.ts +4 -5
  68. package/dist/state/time-value-resolver.d.ts.map +1 -0
  69. package/dist/{common → state}/time-value-resolver.js +9 -8
  70. package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
  71. package/dist/testing/mock-event.d.ts.map +1 -0
  72. package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
  73. package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
  74. package/dist/testing/mock-server.d.ts.map +1 -0
  75. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  76. package/dist/{mock-server → testing}/mock-server.js +6 -1
  77. package/dist/types/errors.d.ts +9 -0
  78. package/dist/types/errors.d.ts.map +1 -0
  79. package/dist/types/extraction.d.ts +73 -18
  80. package/dist/types/extraction.d.ts.map +1 -1
  81. package/dist/types/extraction.js +4 -6
  82. package/dist/types/loading.d.ts +1 -2
  83. package/dist/types/loading.d.ts.map +1 -1
  84. package/dist/types/loading.js +0 -1
  85. package/dist/types/workers.d.ts +64 -20
  86. package/dist/types/workers.d.ts.map +1 -1
  87. package/dist/uploader/uploader.d.ts +5 -4
  88. package/dist/uploader/uploader.d.ts.map +1 -1
  89. package/dist/uploader/uploader.interfaces.d.ts +1 -1
  90. package/dist/uploader/uploader.js +10 -10
  91. package/package.json +3 -3
  92. package/dist/airsync-sdk.d.ts +0 -1802
  93. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  94. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  95. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -516
  96. package/dist/common/control-protocol.d.ts.map +0 -1
  97. package/dist/common/errors.d.ts +0 -6
  98. package/dist/common/errors.d.ts.map +0 -1
  99. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  100. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  101. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  102. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  103. package/dist/common/test-utils.d.ts.map +0 -1
  104. package/dist/common/time-value-resolver.d.ts.map +0 -1
  105. package/dist/common/time-value-resolver.test.d.ts +0 -2
  106. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  107. package/dist/common/time-value-resolver.test.js +0 -268
  108. package/dist/http/axios-client-internal.d.ts.map +0 -1
  109. package/dist/http/axios-client-internal.test.d.ts +0 -2
  110. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  111. package/dist/http/axios-client-internal.test.js +0 -175
  112. package/dist/http/axios-client.d.ts +0 -27
  113. package/dist/http/axios-client.d.ts.map +0 -1
  114. package/dist/http/axios-client.js +0 -60
  115. package/dist/http/constants.d.ts +0 -4
  116. package/dist/http/constants.d.ts.map +0 -1
  117. package/dist/http/constants.js +0 -6
  118. package/dist/http/index.d.ts +0 -2
  119. package/dist/http/index.d.ts.map +0 -1
  120. package/dist/http/index.js +0 -17
  121. package/dist/logger/logger.constants.d.ts +0 -6
  122. package/dist/logger/logger.constants.d.ts.map +0 -1
  123. package/dist/logger/logger.constants.js +0 -13
  124. package/dist/logger/logger.context.d.ts +0 -58
  125. package/dist/logger/logger.context.d.ts.map +0 -1
  126. package/dist/logger/logger.context.js +0 -86
  127. package/dist/logger/logger.test.d.ts +0 -2
  128. package/dist/logger/logger.test.d.ts.map +0 -1
  129. package/dist/logger/logger.test.js +0 -537
  130. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  131. package/dist/mappers/mappers.test.d.ts +0 -2
  132. package/dist/mappers/mappers.test.d.ts.map +0 -1
  133. package/dist/mappers/mappers.test.js +0 -105
  134. package/dist/mock-server/mock-server.d.ts.map +0 -1
  135. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  136. package/dist/multithreading/create-worker.test.d.ts +0 -2
  137. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  138. package/dist/multithreading/create-worker.test.js +0 -108
  139. package/dist/multithreading/process-task.test.d.ts +0 -2
  140. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  141. package/dist/multithreading/process-task.test.js +0 -146
  142. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  143. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  144. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  145. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  146. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  147. package/dist/multithreading/spawn/spawn.test.js +0 -232
  148. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  149. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  150. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +0 -175
  151. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
  152. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  153. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
  154. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
  155. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +0 -390
  156. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
  157. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
  158. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +0 -799
  159. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  160. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  161. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  162. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -514
  163. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
  164. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
  165. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
  166. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +0 -596
  167. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
  168. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
  169. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +0 -71
  170. package/dist/repo/repo.test.d.ts +0 -2
  171. package/dist/repo/repo.test.d.ts.map +0 -1
  172. package/dist/repo/repo.test.js +0 -184
  173. package/dist/state/state.extract-window.test.d.ts +0 -2
  174. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  175. package/dist/state/state.extract-window.test.js +0 -163
  176. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  177. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  178. package/dist/state/state.pending-boundaries.test.js +0 -189
  179. package/dist/state/state.post-state.test.d.ts +0 -2
  180. package/dist/state/state.post-state.test.d.ts.map +0 -1
  181. package/dist/state/state.post-state.test.js +0 -77
  182. package/dist/state/state.test.d.ts +0 -2
  183. package/dist/state/state.test.d.ts.map +0 -1
  184. package/dist/state/state.test.js +0 -268
  185. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  186. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  187. package/dist/state/state.time-value-resolution.test.js +0 -175
  188. package/dist/tsdoc-metadata.json +0 -11
  189. package/dist/types/extraction.test.d.ts +0 -2
  190. package/dist/types/extraction.test.d.ts.map +0 -1
  191. package/dist/types/extraction.test.js +0 -96
  192. package/dist/types/index.d.ts +0 -11
  193. package/dist/types/index.d.ts.map +0 -1
  194. package/dist/types/index.js +0 -21
  195. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  196. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  197. package/dist/uploader/uploader.helpers.test.js +0 -256
  198. package/dist/uploader/uploader.test.d.ts +0 -2
  199. package/dist/uploader/uploader.test.d.ts.map +0 -1
  200. package/dist/uploader/uploader.test.js +0 -601
  201. /package/dist/http/{axios-client-internal.js → client.js} +0 -0
  202. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
  203. /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
  204. /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
  205. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  206. /package/dist/{common → types}/errors.js +0 -0
@@ -1,799 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
4
- const state_1 = require("../../state/state");
5
- const jest_setup_1 = require("../../tests/jest.setup");
6
- const test_utils_1 = require("../../common/test-utils");
7
- const types_1 = require("../../types");
8
- const worker_adapter_1 = require("./worker-adapter");
9
- /* eslint-disable @typescript-eslint/no-require-imports */
10
- jest.mock('../../common/control-protocol', () => ({
11
- emit: jest.fn().mockResolvedValue({}),
12
- }));
13
- jest.mock('../../mappers/mappers');
14
- jest.mock('../../uploader/uploader');
15
- jest.mock('../../repo/repo');
16
- jest.mock('node:worker_threads', () => ({
17
- parentPort: { postMessage: jest.fn() },
18
- }));
19
- jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
20
- AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
21
- streamAll: jest.fn().mockResolvedValue(undefined),
22
- })),
23
- }));
24
- function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
25
- const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
26
- payload: { event_type: eventType },
27
- });
28
- const initialState = {
29
- attachments: { completed: false },
30
- snapInVersionId: '',
31
- toDevRev: {
32
- attachmentsMetadata: {
33
- artifactIds: [],
34
- lastProcessed: 0,
35
- lastProcessedAttachmentsIdsList: [],
36
- },
37
- },
38
- };
39
- const adapterState = new state_1.State({ event, initialState });
40
- const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
41
- return { adapter, event, adapterState };
42
- }
43
- describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
44
- let adapter;
45
- beforeEach(() => {
46
- jest.clearAllMocks();
47
- ({ adapter } = makeAdapter());
48
- });
49
- it('should process all artifact batches successfully', async () => {
50
- // Arrange
51
- const mockStream = jest.fn();
52
- adapter.state.toDevRev = {
53
- attachmentsMetadata: {
54
- artifactIds: ['artifact1', 'artifact2'],
55
- lastProcessed: 0,
56
- lastProcessedAttachmentsIdsList: [],
57
- },
58
- };
59
- adapter['uploader'].getAttachmentsFromArtifactId = jest
60
- .fn()
61
- .mockResolvedValueOnce({
62
- attachments: [
63
- {
64
- url: 'http://example.com/file1.pdf',
65
- id: 'attachment1',
66
- file_name: 'file1.pdf',
67
- parent_id: 'parent1',
68
- },
69
- {
70
- url: 'http://example.com/file2.pdf',
71
- id: 'attachment2',
72
- file_name: 'file2.pdf',
73
- parent_id: 'parent2',
74
- },
75
- ],
76
- })
77
- .mockResolvedValueOnce({
78
- attachments: [
79
- {
80
- url: 'http://example.com/file3.pdf',
81
- id: 'attachment3',
82
- file_name: 'file3.pdf',
83
- parent_id: 'parent3',
84
- },
85
- ],
86
- });
87
- adapter.initializeRepos = jest.fn();
88
- // Act
89
- const result = await adapter.streamAttachments({
90
- stream: mockStream,
91
- });
92
- // Assert
93
- expect(adapter.initializeRepos).toHaveBeenCalledWith([
94
- { itemType: 'ssor_attachment' },
95
- ]);
96
- expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
97
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
98
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
99
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
100
- expect(result).toBeUndefined();
101
- });
102
- it('[edge] should handle invalid batch size by using 1 instead', async () => {
103
- // Arrange
104
- const mockStream = jest.fn();
105
- adapter.state.toDevRev = {
106
- attachmentsMetadata: {
107
- artifactIds: ['artifact1'],
108
- lastProcessed: 0,
109
- lastProcessedAttachmentsIdsList: [],
110
- },
111
- };
112
- adapter['uploader'].getAttachmentsFromArtifactId = jest
113
- .fn()
114
- .mockResolvedValue({
115
- attachments: [
116
- {
117
- url: 'http://example.com/file1.pdf',
118
- id: 'attachment1',
119
- file_name: 'file1.pdf',
120
- parent_id: 'parent1',
121
- },
122
- ],
123
- });
124
- adapter.initializeRepos = jest.fn();
125
- // Act
126
- const result = await adapter.streamAttachments({
127
- stream: mockStream,
128
- batchSize: 0,
129
- });
130
- // Assert
131
- expect(result).toBeUndefined();
132
- });
133
- it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
134
- // Arrange
135
- const mockStream = jest.fn();
136
- adapter.state.toDevRev = {
137
- attachmentsMetadata: {
138
- artifactIds: ['artifact1'],
139
- lastProcessed: 0,
140
- lastProcessedAttachmentsIdsList: [],
141
- },
142
- };
143
- adapter['uploader'].getAttachmentsFromArtifactId = jest
144
- .fn()
145
- .mockResolvedValue({
146
- attachments: [
147
- {
148
- url: 'http://example.com/file1.pdf',
149
- id: 'attachment1',
150
- file_name: 'file1.pdf',
151
- parent_id: 'parent1',
152
- },
153
- ],
154
- });
155
- adapter.initializeRepos = jest.fn();
156
- // Act
157
- const result = await adapter.streamAttachments({
158
- stream: mockStream,
159
- batchSize: 100,
160
- });
161
- // Assert
162
- expect(result).toBeUndefined();
163
- });
164
- it('[edge] should handle empty attachments metadata artifact IDs', async () => {
165
- // Arrange
166
- const mockStream = jest.fn();
167
- adapter.state.toDevRev = {
168
- attachmentsMetadata: {
169
- artifactIds: [],
170
- lastProcessed: 0,
171
- },
172
- };
173
- // Act
174
- const result = await adapter.streamAttachments({
175
- stream: mockStream,
176
- });
177
- // Assert
178
- expect(result).toBeUndefined();
179
- });
180
- it('[edge] should handle errors when getting attachments', async () => {
181
- // Arrange
182
- const mockStream = jest.fn();
183
- adapter.state.toDevRev = {
184
- attachmentsMetadata: {
185
- artifactIds: ['artifact1'],
186
- lastProcessed: 0,
187
- lastProcessedAttachmentsIdsList: [],
188
- },
189
- };
190
- const mockError = new Error('Failed to get attachments');
191
- adapter['uploader'].getAttachmentsFromArtifactId = jest
192
- .fn()
193
- .mockResolvedValue({
194
- error: mockError,
195
- });
196
- adapter.initializeRepos = jest.fn();
197
- // Act
198
- const result = await adapter.streamAttachments({
199
- stream: mockStream,
200
- });
201
- // Assert
202
- expect(result).toEqual({
203
- error: mockError,
204
- });
205
- });
206
- it('[edge] should handle empty attachments array from artifact', async () => {
207
- // Arrange
208
- const mockStream = jest.fn();
209
- adapter.state.toDevRev = {
210
- attachmentsMetadata: {
211
- artifactIds: ['artifact1'],
212
- lastProcessed: 0,
213
- lastProcessedAttachmentsIdsList: [],
214
- },
215
- };
216
- adapter['uploader'].getAttachmentsFromArtifactId = jest
217
- .fn()
218
- .mockResolvedValue({
219
- attachments: [],
220
- });
221
- adapter.initializeRepos = jest.fn();
222
- // Act
223
- const result = await adapter.streamAttachments({
224
- stream: mockStream,
225
- });
226
- // Assert
227
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
228
- expect(result).toBeUndefined();
229
- });
230
- it('should use custom processors when provided', async () => {
231
- // Arrange
232
- const mockStream = jest.fn();
233
- const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
234
- const mockIterator = jest.fn().mockResolvedValue({});
235
- adapter.state.toDevRev = {
236
- attachmentsMetadata: {
237
- artifactIds: ['artifact1'],
238
- lastProcessed: 0,
239
- lastProcessedAttachmentsIdsList: [],
240
- },
241
- };
242
- adapter['uploader'].getAttachmentsFromArtifactId = jest
243
- .fn()
244
- .mockResolvedValue({
245
- attachments: [{ id: 'attachment1' }],
246
- });
247
- adapter.initializeRepos = jest.fn();
248
- // Act
249
- const result = await adapter.streamAttachments({
250
- stream: mockStream,
251
- processors: {
252
- reducer: mockReducer,
253
- iterator: mockIterator,
254
- },
255
- });
256
- // Assert
257
- expect(mockReducer).toHaveBeenCalledWith({
258
- attachments: [{ id: 'attachment1' }],
259
- adapter: adapter,
260
- batchSize: 1,
261
- });
262
- expect(mockIterator).toHaveBeenCalledWith({
263
- reducedAttachments: ['custom-reduced'],
264
- adapter: adapter,
265
- stream: mockStream,
266
- });
267
- expect(result).toBeUndefined();
268
- });
269
- it('should handle rate limiting from iterator', async () => {
270
- // Arrange
271
- const mockStream = jest.fn();
272
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
273
- streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
274
- }));
275
- adapter.state.toDevRev = {
276
- attachmentsMetadata: {
277
- artifactIds: ['artifact1'],
278
- lastProcessed: 0,
279
- lastProcessedAttachmentsIdsList: [],
280
- },
281
- };
282
- adapter['uploader'].getAttachmentsFromArtifactId = jest
283
- .fn()
284
- .mockResolvedValue({
285
- attachments: [{ id: 'attachment1' }],
286
- });
287
- adapter.initializeRepos = jest.fn();
288
- // Act
289
- const result = await adapter.streamAttachments({
290
- stream: mockStream,
291
- });
292
- // Assert
293
- expect(result).toEqual({ delay: 30 });
294
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
295
- 'artifact1',
296
- ]);
297
- });
298
- it('should handle error from iterator', async () => {
299
- // Arrange
300
- const mockStream = jest.fn();
301
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
302
- streamAll: jest.fn().mockResolvedValue({
303
- error: 'Mock error',
304
- }),
305
- }));
306
- adapter.state.toDevRev = {
307
- attachmentsMetadata: {
308
- artifactIds: ['artifact1'],
309
- lastProcessed: 0,
310
- lastProcessedAttachmentsIdsList: [],
311
- },
312
- };
313
- adapter['uploader'].getAttachmentsFromArtifactId = jest
314
- .fn()
315
- .mockResolvedValue({
316
- attachments: [{ id: 'attachment1' }],
317
- });
318
- adapter.initializeRepos = jest.fn();
319
- // Act
320
- const result = await adapter.streamAttachments({
321
- stream: mockStream,
322
- });
323
- // Assert
324
- expect(result).toEqual({ error: 'Mock error' });
325
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
326
- 'artifact1',
327
- ]);
328
- });
329
- it('should emit progress event and exit process on timeout, preserving state for resumption', async () => {
330
- // Arrange
331
- const mockStream = jest.fn();
332
- const exitSpy = jest
333
- .spyOn(process, 'exit')
334
- .mockImplementation(() => undefined);
335
- adapter.state.toDevRev = {
336
- attachmentsMetadata: {
337
- artifactIds: ['artifact1', 'artifact2', 'artifact3'],
338
- lastProcessed: 0,
339
- lastProcessedAttachmentsIdsList: [],
340
- },
341
- };
342
- adapter['uploader'].getAttachmentsFromArtifactId = jest
343
- .fn()
344
- .mockResolvedValue({
345
- attachments: [
346
- {
347
- url: 'http://example.com/file1.pdf',
348
- id: 'attachment1',
349
- file_name: 'file1.pdf',
350
- parent_id: 'parent1',
351
- },
352
- ],
353
- });
354
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
355
- streamAll: jest.fn().mockImplementation(() => {
356
- adapter.isTimeout = true;
357
- return {};
358
- }),
359
- }));
360
- adapter.initializeRepos = jest.fn();
361
- const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
362
- // Act
363
- await adapter.streamAttachments({
364
- stream: mockStream,
365
- });
366
- // Assert
367
- expect(emitSpy).toHaveBeenCalledWith(types_1.ExtractorEventType.AttachmentExtractionProgress);
368
- expect(exitSpy).toHaveBeenCalledWith(0);
369
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
370
- 'artifact1',
371
- 'artifact2',
372
- 'artifact3',
373
- ]);
374
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(1);
375
- exitSpy.mockRestore();
376
- });
377
- it('should stop after the timeout flips between batches and preserve unprocessed artifacts for resumption', async () => {
378
- // Arrange: three artifacts. The first batch's streamAll completes
379
- // successfully; the second sets isTimeout=true mid-run. The third batch
380
- // must never be reached.
381
- const mockStream = jest.fn();
382
- const exitSpy = jest
383
- .spyOn(process, 'exit')
384
- .mockImplementation(() => undefined);
385
- adapter.state.toDevRev = {
386
- attachmentsMetadata: {
387
- artifactIds: ['artifact1', 'artifact2', 'artifact3'],
388
- lastProcessed: 0,
389
- lastProcessedAttachmentsIdsList: [],
390
- },
391
- };
392
- adapter['uploader'].getAttachmentsFromArtifactId = jest
393
- .fn()
394
- .mockResolvedValue({
395
- attachments: [
396
- {
397
- url: 'http://example.com/file.pdf',
398
- id: 'attachment-x',
399
- file_name: 'file.pdf',
400
- parent_id: 'parent-x',
401
- },
402
- ],
403
- });
404
- // First call: clean streamAll. Second call: flip isTimeout AFTER streaming.
405
- attachments_streaming_pool_1.AttachmentsStreamingPool
406
- .mockImplementationOnce(() => ({
407
- streamAll: jest.fn().mockResolvedValue({}),
408
- }))
409
- .mockImplementationOnce(() => ({
410
- streamAll: jest.fn().mockImplementation(() => {
411
- adapter.isTimeout = true;
412
- return {};
413
- }),
414
- }));
415
- adapter.initializeRepos = jest.fn();
416
- const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue();
417
- // Act
418
- await adapter.streamAttachments({ stream: mockStream });
419
- // Assert
420
- // - Fetched attachments for the first two artifacts only; the third never ran
421
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
422
- // - Progress emitted and process.exit(0) called once the timeout was detected
423
- expect(emitSpy).toHaveBeenCalledWith(types_1.ExtractorEventType.AttachmentExtractionProgress);
424
- expect(exitSpy).toHaveBeenCalledWith(0);
425
- // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught
426
- // before its shift) along with the untouched artifact 3
427
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
428
- 'artifact2',
429
- 'artifact3',
430
- ]);
431
- exitSpy.mockRestore();
432
- });
433
- it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
434
- // Arrange
435
- const mockStream = jest.fn();
436
- adapter.state.toDevRev = {
437
- attachmentsMetadata: {
438
- artifactIds: ['artifact1'],
439
- lastProcessed: 0,
440
- lastProcessedAttachmentsIdsList: [],
441
- },
442
- };
443
- adapter['uploader'].getAttachmentsFromArtifactId = jest
444
- .fn()
445
- .mockResolvedValueOnce({
446
- attachments: [
447
- {
448
- url: 'http://example.com/file1.pdf',
449
- id: 'attachment1',
450
- file_name: 'file1.pdf',
451
- parent_id: 'parent1',
452
- },
453
- {
454
- url: 'http://example.com/file2.pdf',
455
- id: 'attachment2',
456
- file_name: 'file2.pdf',
457
- parent_id: 'parent2',
458
- },
459
- {
460
- url: 'http://example.com/file3.pdf',
461
- id: 'attachment3',
462
- file_name: 'file3.pdf',
463
- parent_id: 'parent3',
464
- },
465
- ],
466
- });
467
- adapter.processAttachment = jest.fn().mockResolvedValue(null);
468
- // Act
469
- await adapter.streamAttachments({
470
- stream: mockStream,
471
- });
472
- // Assert
473
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
474
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
475
- });
476
- });
477
- describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
478
- let adapter;
479
- beforeEach(() => {
480
- jest.clearAllMocks();
481
- ({ adapter } = makeAdapter(types_1.EventType.StartExtractingAttachments));
482
- });
483
- afterEach(() => {
484
- jest.restoreAllMocks();
485
- });
486
- const createMockHttpStream = (headers = {}) => ({
487
- headers,
488
- data: { destroy: jest.fn() },
489
- });
490
- const makeAttachment = (overrides = {}) => (Object.assign({ id: 'att-1', url: 'https://example.com/file.pdf', file_name: 'file.pdf', parent_id: 'parent-1', content_type: 'application/pdf' }, overrides));
491
- function setupUploaderHappyPath() {
492
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
493
- response: {
494
- artifact_id: 'art_1',
495
- upload_url: 'https://upload',
496
- form_data: [],
497
- },
498
- });
499
- adapter['uploader'].streamArtifact = jest
500
- .fn()
501
- .mockResolvedValue({ response: {} });
502
- adapter['uploader'].confirmArtifactUpload = jest
503
- .fn()
504
- .mockResolvedValue({ response: {} });
505
- const pushMock = jest.fn().mockResolvedValue(undefined);
506
- adapter.getRepo = jest.fn().mockReturnValue({ push: pushMock });
507
- return pushMock;
508
- }
509
- // ---- content-type resolution ----
510
- it('should use attachment.content_type when provided, ignoring HTTP header', async () => {
511
- // Arrange
512
- setupUploaderHappyPath();
513
- const mockStream = jest.fn().mockResolvedValue({
514
- httpStream: createMockHttpStream({
515
- 'content-type': 'text/plain',
516
- 'content-length': '100',
517
- }),
518
- });
519
- // Act
520
- await adapter.processAttachment(makeAttachment({ content_type: 'application/pdf' }), mockStream);
521
- // Assert
522
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('file.pdf', 'application/pdf', 100);
523
- });
524
- it('should use HTTP header content-type when attachment.content_type is not set', async () => {
525
- // Arrange
526
- setupUploaderHappyPath();
527
- const mockStream = jest.fn().mockResolvedValue({
528
- httpStream: createMockHttpStream({
529
- 'content-type': 'image/jpeg',
530
- 'content-length': '200',
531
- }),
532
- });
533
- const attachment = {
534
- id: 'att-2',
535
- url: 'https://example.com/photo.jpg',
536
- file_name: 'photo.jpg',
537
- parent_id: 'parent-2',
538
- };
539
- // Act
540
- await adapter.processAttachment(attachment, mockStream);
541
- // Assert
542
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('photo.jpg', 'image/jpeg', 200);
543
- });
544
- it('should fall back to application/octet-stream when neither content_type nor HTTP header is set', async () => {
545
- // Arrange
546
- setupUploaderHappyPath();
547
- const mockStream = jest.fn().mockResolvedValue({
548
- httpStream: createMockHttpStream({}),
549
- });
550
- const attachment = {
551
- id: 'att-3',
552
- url: 'https://example.com/file.bin',
553
- file_name: 'file.bin',
554
- parent_id: 'parent-3',
555
- };
556
- // Act
557
- await adapter.processAttachment(attachment, mockStream);
558
- // Assert
559
- expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith('file.bin', 'application/octet-stream', undefined);
560
- });
561
- // ---- error paths ----
562
- it('should return the stream error directly when the stream function returns an error', async () => {
563
- // Arrange
564
- const stream = jest
565
- .fn()
566
- .mockResolvedValue({ error: new Error('stream failed') });
567
- // Act
568
- const result = await adapter.processAttachment(makeAttachment(), stream);
569
- // Assert
570
- expect(result === null || result === void 0 ? void 0 : result.error).toBeDefined();
571
- });
572
- it('should propagate a rate-limit delay from the stream function', async () => {
573
- // Arrange
574
- const stream = jest.fn().mockResolvedValue({ delay: 5 });
575
- // Act
576
- const result = await adapter.processAttachment(makeAttachment(), stream);
577
- // Assert
578
- expect(result === null || result === void 0 ? void 0 : result.delay).toBe(5);
579
- });
580
- it('should return an error containing the attachment ID when getArtifactUploadUrl fails', async () => {
581
- var _a, _b;
582
- // Arrange
583
- const stream = jest
584
- .fn()
585
- .mockResolvedValue({ httpStream: createMockHttpStream() });
586
- adapter['uploader'].getArtifactUploadUrl = jest
587
- .fn()
588
- .mockResolvedValue({ error: new Error('upload url failed') });
589
- // Act
590
- const result = await adapter.processAttachment(makeAttachment(), stream);
591
- // Assert
592
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('att-1');
593
- expect((_b = result === null || result === void 0 ? void 0 : result.error) === null || _b === void 0 ? void 0 : _b.message).toContain('preparing artifact');
594
- });
595
- it('should return an error when streamArtifact fails', async () => {
596
- var _a;
597
- // Arrange
598
- const stream = jest
599
- .fn()
600
- .mockResolvedValue({ httpStream: createMockHttpStream() });
601
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
602
- response: {
603
- artifact_id: 'art-1',
604
- upload_url: 'https://upload',
605
- form_data: [],
606
- },
607
- });
608
- adapter['uploader'].streamArtifact = jest
609
- .fn()
610
- .mockResolvedValue({ error: new Error('stream failed') });
611
- // Act
612
- const result = await adapter.processAttachment(makeAttachment(), stream);
613
- // Assert
614
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('streaming to artifact');
615
- });
616
- it('should return an error when confirmArtifactUpload fails', async () => {
617
- var _a;
618
- // Arrange
619
- const stream = jest
620
- .fn()
621
- .mockResolvedValue({ httpStream: createMockHttpStream() });
622
- adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({
623
- response: {
624
- artifact_id: 'art-1',
625
- upload_url: 'https://upload',
626
- form_data: [],
627
- },
628
- });
629
- adapter['uploader'].streamArtifact = jest
630
- .fn()
631
- .mockResolvedValue({ response: {} });
632
- adapter['uploader'].confirmArtifactUpload = jest
633
- .fn()
634
- .mockResolvedValue({ error: new Error('confirm failed') });
635
- // Act
636
- const result = await adapter.processAttachment(makeAttachment(), stream);
637
- // Assert
638
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('confirming upload');
639
- });
640
- it.each([
641
- { inline: true, expected: true },
642
- { inline: false, expected: false },
643
- ])('should set inline=$expected on the ssorAttachment when attachment.inline=$inline', async ({ inline, expected }) => {
644
- // Arrange
645
- const pushMock = setupUploaderHappyPath();
646
- const stream = jest
647
- .fn()
648
- .mockResolvedValue({ httpStream: createMockHttpStream() });
649
- // Act
650
- await adapter.processAttachment(makeAttachment({ inline }), stream);
651
- // Assert
652
- const ssorItem = pushMock.mock.calls[0][0][0];
653
- expect(ssorItem.inline).toBe(expected);
654
- });
655
- it('should return a descriptive error when the stream function returns no httpStream', async () => {
656
- var _a;
657
- // Arrange
658
- const stream = jest.fn().mockResolvedValue({ httpStream: null });
659
- // Act
660
- const result = await adapter.processAttachment(makeAttachment(), stream);
661
- // Assert
662
- expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Error while opening attachment stream');
663
- });
664
- });
665
- describe(`${worker_adapter_1.WorkerAdapter.name}.initializeRepos — event size threshold`, () => {
666
- it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => {
667
- // Arrange
668
- const { adapter } = makeAdapter();
669
- let capturedOnUpload;
670
- const { Repo } = require('../../repo/repo');
671
- Repo.mockImplementationOnce((opts) => {
672
- capturedOnUpload = opts.onUpload;
673
- return { itemType: 'issues', upload: jest.fn(), uploadedArtifacts: [] };
674
- });
675
- // Act
676
- adapter.initializeRepos([{ itemType: 'issues' }]);
677
- expect(capturedOnUpload).toBeDefined();
678
- capturedOnUpload({
679
- id: 'artifact-x',
680
- item_count: 1,
681
- item_type: 'x'.repeat(200000),
682
- });
683
- // Assert
684
- expect(adapter.isTimeout).toBe(true);
685
- });
686
- });
687
- describe(`${worker_adapter_1.WorkerAdapter.name}.getRepo`, () => {
688
- it('should return undefined when the requested repo was never initialised', () => {
689
- // Arrange
690
- const { adapter } = makeAdapter();
691
- // Act
692
- const result = adapter.getRepo('non-existent-type');
693
- // Assert
694
- expect(result).toBeUndefined();
695
- });
696
- });
697
- describe(`${worker_adapter_1.WorkerAdapter.name}.destroyHttpStream`, () => {
698
- let adapter;
699
- beforeEach(() => {
700
- ({ adapter } = makeAdapter());
701
- });
702
- afterEach(() => {
703
- jest.restoreAllMocks();
704
- });
705
- it.each([
706
- {
707
- label: 'calls destroy() when available',
708
- data: { destroy: jest.fn(), close: jest.fn() },
709
- expectDestroy: true,
710
- expectClose: false,
711
- },
712
- {
713
- label: 'calls close() when destroy is not present',
714
- data: { close: jest.fn() },
715
- expectDestroy: false,
716
- expectClose: true,
717
- },
718
- {
719
- label: 'does not throw when neither method is present',
720
- data: {},
721
- expectDestroy: false,
722
- expectClose: false,
723
- },
724
- {
725
- label: 'does not throw when data is null',
726
- data: null,
727
- expectDestroy: false,
728
- expectClose: false,
729
- },
730
- ])('$label', ({ data, expectDestroy, expectClose }) => {
731
- // Arrange
732
- const httpStream = { data };
733
- // Act & Assert
734
- expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow();
735
- if (expectDestroy) {
736
- expect(data.destroy).toHaveBeenCalled();
737
- }
738
- if (expectClose) {
739
- expect(data.close).toHaveBeenCalled();
740
- }
741
- });
742
- it('should not re-throw when destroy() itself throws', () => {
743
- // Arrange
744
- const httpStream = {
745
- data: {
746
- destroy: () => {
747
- throw new Error('stream error');
748
- },
749
- },
750
- };
751
- // Act & Assert
752
- expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow();
753
- });
754
- });
755
- describe(`${worker_adapter_1.WorkerAdapter.name} — extractionScope`, () => {
756
- it('should return empty object by default', () => {
757
- const { adapter } = makeAdapter();
758
- expect(adapter.extractionScope).toEqual({});
759
- });
760
- it('should return extraction scope from adapter state', () => {
761
- const { adapter, adapterState } = makeAdapter();
762
- const extractionScope = {
763
- tasks: { extract: true },
764
- users: { extract: false },
765
- };
766
- adapterState._extractionScope = extractionScope;
767
- expect(adapter.extractionScope).toEqual(extractionScope);
768
- });
769
- });
770
- describe(`${worker_adapter_1.WorkerAdapter.name} — shouldExtract`, () => {
771
- it('should return true when extraction scope is empty', () => {
772
- const { adapter } = makeAdapter();
773
- expect(adapter.shouldExtract('tasks')).toBe(true);
774
- expect(adapter.shouldExtract('users')).toBe(true);
775
- });
776
- it('should return true when item type is not in scope', () => {
777
- const { adapter, adapterState } = makeAdapter();
778
- adapterState._extractionScope = {
779
- tasks: { extract: true },
780
- };
781
- expect(adapter.shouldExtract('users')).toBe(true);
782
- });
783
- it('should return true when item type has extract: true', () => {
784
- const { adapter, adapterState } = makeAdapter();
785
- adapterState._extractionScope = {
786
- tasks: { extract: true },
787
- };
788
- expect(adapter.shouldExtract('tasks')).toBe(true);
789
- });
790
- it('should return false when item type has extract: false', () => {
791
- const { adapter, adapterState } = makeAdapter();
792
- adapterState._extractionScope = {
793
- tasks: { extract: false },
794
- users: { extract: true },
795
- };
796
- expect(adapter.shouldExtract('tasks')).toBe(false);
797
- expect(adapter.shouldExtract('users')).toBe(true);
798
- });
799
- });