@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +5 -11
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -10
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +15 -43
  7. package/dist/common/constants.d.ts +1 -0
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +2 -1
  10. package/dist/common/helpers.d.ts +0 -14
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +0 -36
  13. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  14. package/dist/http/client.d.ts.map +1 -0
  15. package/dist/index.d.ts +22 -11
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +46 -28
  18. package/dist/logger/logger.d.ts +14 -0
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.js +55 -8
  21. package/dist/mappers/mappers.d.ts +9 -10
  22. package/dist/mappers/mappers.d.ts.map +1 -1
  23. package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
  24. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  25. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
  26. package/dist/mappers/mappers.js +13 -9
  27. package/dist/multithreading/adapters/base-adapter.d.ts +23 -8
  28. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  29. package/dist/multithreading/adapters/base-adapter.js +41 -8
  30. package/dist/multithreading/adapters/extraction-adapter.d.ts +8 -9
  31. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  32. package/dist/multithreading/adapters/extraction-adapter.js +24 -26
  33. package/dist/multithreading/adapters/loading-adapter.d.ts +7 -7
  34. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  35. package/dist/multithreading/adapters/loading-adapter.js +53 -83
  36. package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
  37. package/dist/multithreading/emit.d.ts.map +1 -0
  38. package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
  39. package/dist/multithreading/process-task.d.ts.map +1 -1
  40. package/dist/multithreading/process-task.js +26 -9
  41. package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
  42. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  43. package/dist/multithreading/spawn/spawn.helpers.js +152 -2
  44. package/dist/multithreading/spawn/spawn.js +7 -7
  45. package/dist/multithreading/worker.js +1 -2
  46. package/dist/repo/repo.js +2 -2
  47. package/dist/state/base-state.d.ts +5 -5
  48. package/dist/state/base-state.d.ts.map +1 -1
  49. package/dist/state/base-state.js +7 -7
  50. package/dist/state/extraction-state.d.ts +9 -9
  51. package/dist/state/extraction-state.d.ts.map +1 -1
  52. package/dist/state/extraction-state.js +10 -10
  53. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  54. package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
  55. package/dist/state/loading-state.d.ts +7 -8
  56. package/dist/state/loading-state.d.ts.map +1 -1
  57. package/dist/state/loading-state.js +6 -7
  58. package/dist/state/state.d.ts +4 -0
  59. package/dist/state/state.d.ts.map +1 -0
  60. package/dist/state/state.interfaces.d.ts +28 -18
  61. package/dist/state/state.interfaces.d.ts.map +1 -1
  62. package/dist/state/state.interfaces.js +13 -6
  63. package/dist/state/state.js +11 -0
  64. package/dist/{common → state}/time-value-resolver.d.ts +6 -7
  65. package/dist/state/time-value-resolver.d.ts.map +1 -0
  66. package/dist/{common → state}/time-value-resolver.js +9 -8
  67. package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
  68. package/dist/testing/mock-event.d.ts.map +1 -0
  69. package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
  70. package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
  71. package/dist/testing/mock-server.d.ts.map +1 -0
  72. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  73. package/dist/{mock-server → testing}/mock-server.js +6 -1
  74. package/dist/types/errors.d.ts +9 -0
  75. package/dist/types/errors.d.ts.map +1 -0
  76. package/dist/types/extraction.d.ts +70 -15
  77. package/dist/types/extraction.d.ts.map +1 -1
  78. package/dist/types/extraction.js +4 -6
  79. package/dist/types/loading.d.ts +1 -2
  80. package/dist/types/loading.d.ts.map +1 -1
  81. package/dist/types/loading.js +0 -1
  82. package/dist/types/workers.d.ts +66 -23
  83. package/dist/types/workers.d.ts.map +1 -1
  84. package/dist/uploader/uploader.d.ts +5 -4
  85. package/dist/uploader/uploader.d.ts.map +1 -1
  86. package/dist/uploader/uploader.interfaces.d.ts +1 -1
  87. package/dist/uploader/uploader.js +10 -10
  88. package/package.json +3 -3
  89. package/dist/airsync-sdk.d.ts +0 -2059
  90. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  91. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  92. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
  93. package/dist/common/control-protocol.d.ts.map +0 -1
  94. package/dist/common/errors.d.ts +0 -6
  95. package/dist/common/errors.d.ts.map +0 -1
  96. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  97. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  98. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  99. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  100. package/dist/common/test-utils.d.ts.map +0 -1
  101. package/dist/common/time-value-resolver.d.ts.map +0 -1
  102. package/dist/common/time-value-resolver.test.d.ts +0 -2
  103. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  104. package/dist/common/time-value-resolver.test.js +0 -268
  105. package/dist/http/axios-client-internal.d.ts.map +0 -1
  106. package/dist/http/axios-client-internal.test.d.ts +0 -2
  107. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  108. package/dist/http/axios-client-internal.test.js +0 -175
  109. package/dist/http/axios-client.d.ts +0 -27
  110. package/dist/http/axios-client.d.ts.map +0 -1
  111. package/dist/http/axios-client.js +0 -60
  112. package/dist/http/constants.d.ts +0 -4
  113. package/dist/http/constants.d.ts.map +0 -1
  114. package/dist/http/constants.js +0 -6
  115. package/dist/http/index.d.ts +0 -2
  116. package/dist/http/index.d.ts.map +0 -1
  117. package/dist/http/index.js +0 -17
  118. package/dist/logger/logger.constants.d.ts +0 -6
  119. package/dist/logger/logger.constants.d.ts.map +0 -1
  120. package/dist/logger/logger.constants.js +0 -13
  121. package/dist/logger/logger.context.d.ts +0 -58
  122. package/dist/logger/logger.context.d.ts.map +0 -1
  123. package/dist/logger/logger.context.js +0 -86
  124. package/dist/logger/logger.test.d.ts +0 -2
  125. package/dist/logger/logger.test.d.ts.map +0 -1
  126. package/dist/logger/logger.test.js +0 -537
  127. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  128. package/dist/mappers/mappers.test.d.ts +0 -2
  129. package/dist/mappers/mappers.test.d.ts.map +0 -1
  130. package/dist/mappers/mappers.test.js +0 -105
  131. package/dist/mock-server/mock-server.d.ts.map +0 -1
  132. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  133. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
  134. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
  135. package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
  136. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
  137. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
  138. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
  139. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
  140. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
  141. package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
  142. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
  143. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
  144. package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
  145. package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
  146. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
  147. package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
  148. package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
  149. package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
  150. package/dist/multithreading/adapters/serialization.test.js +0 -71
  151. package/dist/multithreading/create-worker.test.d.ts +0 -2
  152. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  153. package/dist/multithreading/create-worker.test.js +0 -108
  154. package/dist/multithreading/process-task.test.d.ts +0 -2
  155. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  156. package/dist/multithreading/process-task.test.js +0 -146
  157. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  158. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  159. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  160. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  161. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  162. package/dist/multithreading/spawn/spawn.test.js +0 -232
  163. package/dist/repo/repo.test.d.ts +0 -2
  164. package/dist/repo/repo.test.d.ts.map +0 -1
  165. package/dist/repo/repo.test.js +0 -184
  166. package/dist/state/state.extract-window.test.d.ts +0 -2
  167. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  168. package/dist/state/state.extract-window.test.js +0 -163
  169. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  170. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  171. package/dist/state/state.pending-boundaries.test.js +0 -189
  172. package/dist/state/state.post-state.test.d.ts +0 -2
  173. package/dist/state/state.post-state.test.d.ts.map +0 -1
  174. package/dist/state/state.post-state.test.js +0 -78
  175. package/dist/state/state.test.d.ts +0 -2
  176. package/dist/state/state.test.d.ts.map +0 -1
  177. package/dist/state/state.test.js +0 -271
  178. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  179. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  180. package/dist/state/state.time-value-resolution.test.js +0 -175
  181. package/dist/tsdoc-metadata.json +0 -11
  182. package/dist/types/extraction.test.d.ts +0 -2
  183. package/dist/types/extraction.test.d.ts.map +0 -1
  184. package/dist/types/extraction.test.js +0 -96
  185. package/dist/types/index.d.ts +0 -10
  186. package/dist/types/index.d.ts.map +0 -1
  187. package/dist/types/index.js +0 -21
  188. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  189. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  190. package/dist/uploader/uploader.helpers.test.js +0 -256
  191. package/dist/uploader/uploader.test.d.ts +0 -2
  192. package/dist/uploader/uploader.test.d.ts.map +0 -1
  193. package/dist/uploader/uploader.test.js +0 -601
  194. /package/dist/http/{axios-client-internal.js → client.js} +0 -0
  195. /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
  196. /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
  197. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  198. /package/dist/{common → types}/errors.js +0 -0
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=attachments-streaming-pool.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"attachments-streaming-pool.test.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.test.ts"],"names":[],"mappings":""}
@@ -1,540 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
4
- /* eslint-disable @typescript-eslint/no-explicit-any */
5
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
6
- let mockAdapter;
7
- let mockStream;
8
- let mockAttachments;
9
- let attachmentsMetadata;
10
- beforeEach(() => {
11
- // SDK-owned attachments bookkeeping, passed into the pool directly.
12
- attachmentsMetadata = {
13
- artifactIds: [],
14
- lastProcessed: 0,
15
- lastProcessedAttachmentsIdsList: [],
16
- };
17
- // Create mock adapter
18
- mockAdapter = {
19
- state: {
20
- attachments: { completed: false },
21
- },
22
- processAttachment: jest
23
- .fn()
24
- .mockResolvedValue({}),
25
- };
26
- // Create mock stream function
27
- mockStream = jest.fn().mockResolvedValue({ success: true });
28
- // Create mock attachments
29
- mockAttachments = [
30
- {
31
- id: 'attachment-1',
32
- url: 'https://example.com/file1.pdf',
33
- file_name: 'file1.pdf',
34
- parent_id: 'parent-1',
35
- },
36
- {
37
- id: 'attachment-2',
38
- url: 'https://example.com/file2.jpg',
39
- file_name: 'file2.jpg',
40
- parent_id: 'parent-2',
41
- },
42
- {
43
- id: 'attachment-3',
44
- url: 'https://example.com/file3.doc',
45
- file_name: 'file3.doc',
46
- parent_id: 'parent-3',
47
- },
48
- ];
49
- });
50
- afterEach(() => {
51
- jest.clearAllMocks();
52
- jest.restoreAllMocks();
53
- });
54
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.constructor.name, () => {
55
- it('should initialize with default values', () => {
56
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
57
- adapter: mockAdapter,
58
- attachmentsMetadata,
59
- attachments: mockAttachments,
60
- stream: mockStream,
61
- });
62
- expect(pool['adapter']).toBe(mockAdapter);
63
- expect(pool['attachments']).toEqual(mockAttachments);
64
- expect(pool['batchSize']).toBe(10);
65
- expect(pool['stream']).toBe(mockStream);
66
- });
67
- it('should initialize with custom batch size', () => {
68
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
69
- adapter: mockAdapter,
70
- attachmentsMetadata,
71
- attachments: mockAttachments,
72
- batchSize: 5,
73
- stream: mockStream,
74
- });
75
- expect(pool['batchSize']).toBe(5);
76
- });
77
- it('should create a copy of attachments array', () => {
78
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
79
- adapter: mockAdapter,
80
- attachmentsMetadata,
81
- attachments: mockAttachments,
82
- stream: mockStream,
83
- });
84
- expect(pool['attachments']).toEqual(mockAttachments);
85
- expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
86
- });
87
- });
88
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.streamAll.name, () => {
89
- it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
90
- attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined;
91
- mockAdapter.processAttachment.mockResolvedValue({});
92
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
93
- adapter: mockAdapter,
94
- attachmentsMetadata,
95
- attachments: mockAttachments,
96
- stream: mockStream,
97
- });
98
- // Mock startPoolStreaming to avoid actual processing
99
- jest
100
- .spyOn(pool, 'startPoolStreaming')
101
- .mockResolvedValue(undefined);
102
- await pool.streamAll();
103
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
104
- });
105
- it('should process all attachments successfully', async () => {
106
- mockAdapter.processAttachment.mockResolvedValue({});
107
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
108
- adapter: mockAdapter,
109
- attachmentsMetadata,
110
- attachments: mockAttachments,
111
- stream: mockStream,
112
- });
113
- const result = await pool.streamAll();
114
- expect(result).toEqual({});
115
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
116
- });
117
- it('should handle empty attachments array', async () => {
118
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
119
- adapter: mockAdapter,
120
- attachmentsMetadata,
121
- attachments: [],
122
- stream: mockStream,
123
- });
124
- const result = await pool.streamAll();
125
- expect(result).toEqual({});
126
- expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
127
- });
128
- it('should return delay when rate limit is hit', async () => {
129
- const delayResponse = { delay: 5000 };
130
- mockAdapter.processAttachment.mockResolvedValue(delayResponse);
131
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
132
- adapter: mockAdapter,
133
- attachmentsMetadata,
134
- attachments: mockAttachments,
135
- stream: mockStream,
136
- });
137
- const result = await pool.streamAll();
138
- expect(result).toEqual({ delay: 5000 });
139
- });
140
- it('should resume attachment extraction if it encounters old ids', async () => {
141
- // Test migration from old string[] format to new ProcessedAttachment[] format
142
- // Using 'as any' because we're intentionally testing legacy data format
143
- attachmentsMetadata.lastProcessedAttachmentsIdsList = [
144
- 'attachment-1',
145
- 'attachment-2',
146
- ];
147
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
148
- adapter: mockAdapter,
149
- attachmentsMetadata,
150
- attachments: mockAttachments,
151
- stream: mockStream,
152
- });
153
- const result = await pool.streamAll();
154
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
155
- { id: 'attachment-1', parent_id: '' },
156
- { id: 'attachment-2', parent_id: '' },
157
- { id: 'attachment-1', parent_id: 'parent-1' },
158
- { id: 'attachment-2', parent_id: 'parent-2' },
159
- { id: 'attachment-3', parent_id: 'parent-3' },
160
- ]);
161
- expect(result).toEqual({});
162
- });
163
- it('should handle all attachments failing with different file types and sizes', async () => {
164
- const largeAttachments = [
165
- {
166
- id: 'attachment-image',
167
- url: 'https://example.com/photo.jpg',
168
- file_name: 'photo.jpg',
169
- parent_id: 'parent-1',
170
- },
171
- {
172
- id: 'attachment-pdf',
173
- url: 'https://example.com/document.pdf',
174
- file_name: 'document.pdf',
175
- parent_id: 'parent-2',
176
- },
177
- {
178
- id: 'attachment-video',
179
- url: 'https://example.com/video.mp4',
180
- file_name: 'video.mp4',
181
- parent_id: 'parent-3',
182
- },
183
- ];
184
- const imageError = new Error('Image upload failed: File too large');
185
- const pdfError = new Error('PDF upload failed: Unsupported format');
186
- const videoError = new Error('Video upload failed: Network error');
187
- mockAdapter.processAttachment
188
- .mockRejectedValueOnce(imageError)
189
- .mockRejectedValueOnce(pdfError)
190
- .mockRejectedValueOnce(videoError);
191
- const warnSpy = jest.spyOn(console, 'warn');
192
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
193
- adapter: mockAdapter,
194
- attachmentsMetadata,
195
- attachments: largeAttachments,
196
- stream: mockStream,
197
- });
198
- await pool.streamAll();
199
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
200
- // Since there are 3 parallel workers (one per attachment), each logs its own failed attachment
201
- expect(warnSpy).toHaveBeenCalledTimes(3);
202
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
203
- });
204
- });
205
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
206
- it('should skip already processed attachments', async () => {
207
- attachmentsMetadata.lastProcessedAttachmentsIdsList = [
208
- { id: 'attachment-1', parent_id: 'parent-1' },
209
- ];
210
- mockAdapter.processAttachment.mockResolvedValue({});
211
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
212
- adapter: mockAdapter,
213
- attachmentsMetadata,
214
- attachments: mockAttachments,
215
- stream: mockStream,
216
- });
217
- await pool.streamAll();
218
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
219
- });
220
- it('should add successfully processed attachment IDs to the list', async () => {
221
- mockAdapter.processAttachment.mockResolvedValue({});
222
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
223
- adapter: mockAdapter,
224
- attachmentsMetadata,
225
- attachments: mockAttachments,
226
- stream: mockStream,
227
- });
228
- await pool.streamAll();
229
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
230
- { id: 'attachment-1', parent_id: 'parent-1' },
231
- { id: 'attachment-2', parent_id: 'parent-2' },
232
- { id: 'attachment-3', parent_id: 'parent-3' },
233
- ]);
234
- });
235
- it('should handle processing errors gracefully', async () => {
236
- const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
237
- const error = new Error('Processing failed');
238
- mockAdapter.processAttachment
239
- .mockResolvedValueOnce({}) // First attachment succeeds
240
- .mockRejectedValueOnce(error) // Second attachment fails
241
- .mockResolvedValueOnce({}); // Third attachment succeeds
242
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
243
- adapter: mockAdapter,
244
- attachmentsMetadata,
245
- attachments: mockAttachments,
246
- stream: mockStream,
247
- });
248
- await pool.streamAll();
249
- expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg due to error in processAttachment function', error);
250
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
251
- {
252
- id: 'attachment-1',
253
- parent_id: 'parent-1',
254
- },
255
- {
256
- id: 'attachment-3',
257
- parent_id: 'parent-3',
258
- },
259
- ]);
260
- });
261
- it('should stop processing when rate limit delay is encountered', async () => {
262
- mockAdapter.processAttachment
263
- .mockResolvedValueOnce({}) // First attachment succeeds
264
- .mockResolvedValueOnce({ delay: 5000 }) // Second attachment triggers rate limit
265
- .mockResolvedValueOnce({}); // Third attachment succeeds
266
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
267
- adapter: mockAdapter,
268
- attachmentsMetadata,
269
- attachments: mockAttachments,
270
- stream: mockStream,
271
- });
272
- await pool.streamAll();
273
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
274
- expect(attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
275
- { id: 'attachment-1', parent_id: 'parent-1' },
276
- { id: 'attachment-3', parent_id: 'parent-3' },
277
- ]);
278
- });
279
- it('should pass correct parameters to processAttachment', async () => {
280
- mockAdapter.processAttachment.mockResolvedValue({});
281
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
282
- adapter: mockAdapter,
283
- attachmentsMetadata,
284
- attachments: [mockAttachments[0]],
285
- stream: mockStream,
286
- });
287
- await pool.streamAll();
288
- expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
289
- });
290
- });
291
- it('[edge] should handle single attachment', async () => {
292
- mockAdapter.processAttachment.mockResolvedValue({});
293
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
294
- adapter: mockAdapter,
295
- attachmentsMetadata,
296
- attachments: [mockAttachments[0]],
297
- stream: mockStream,
298
- });
299
- const result = await pool.streamAll();
300
- expect(result).toEqual({});
301
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
302
- });
303
- it('[edge] should handle batch size larger than attachments array', async () => {
304
- mockAdapter.processAttachment.mockResolvedValue({});
305
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
306
- adapter: mockAdapter,
307
- attachmentsMetadata,
308
- attachments: mockAttachments,
309
- batchSize: 100,
310
- stream: mockStream,
311
- });
312
- await pool.streamAll();
313
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
314
- });
315
- it('[edge] should upload attachments with same id, but different parent_id', async () => {
316
- mockAdapter.processAttachment.mockResolvedValue({});
317
- mockAttachments.push({
318
- id: 'attachment-1',
319
- url: 'http://example.com/file5.jpg',
320
- file_name: 'file5.jpg',
321
- parent_id: 'parent-4',
322
- });
323
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
324
- adapter: mockAdapter,
325
- attachmentsMetadata,
326
- attachments: mockAttachments,
327
- batchSize: 1,
328
- stream: mockStream,
329
- });
330
- await pool.streamAll();
331
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(4);
332
- });
333
- it('[edge] should handle batch size of 1', async () => {
334
- mockAdapter.processAttachment.mockResolvedValue({});
335
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
336
- adapter: mockAdapter,
337
- attachmentsMetadata,
338
- attachments: mockAttachments,
339
- batchSize: 1,
340
- stream: mockStream,
341
- });
342
- await pool.streamAll();
343
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
344
- });
345
- describe('content_type handling', () => {
346
- it('should pass attachment with content_type to processAttachment', async () => {
347
- mockAdapter.processAttachment.mockResolvedValue({});
348
- const attachmentWithContentType = {
349
- id: 'attachment-ct',
350
- url: 'https://example.com/report.pdf',
351
- file_name: 'report.pdf',
352
- parent_id: 'parent-ct',
353
- content_type: 'application/pdf',
354
- };
355
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
356
- adapter: mockAdapter,
357
- attachmentsMetadata,
358
- attachments: [attachmentWithContentType],
359
- stream: mockStream,
360
- });
361
- await pool.streamAll();
362
- expect(mockAdapter.processAttachment).toHaveBeenCalledWith(attachmentWithContentType, mockStream);
363
- expect(mockAdapter.processAttachment.mock.calls[0][0].content_type).toBe('application/pdf');
364
- });
365
- it('should handle mixed attachments with and without content_type', async () => {
366
- mockAdapter.processAttachment.mockResolvedValue({});
367
- const mixedAttachments = [
368
- {
369
- id: 'att-with-ct',
370
- url: 'https://example.com/image.png',
371
- file_name: 'image.png',
372
- parent_id: 'parent-1',
373
- content_type: 'image/png',
374
- },
375
- {
376
- id: 'att-without-ct',
377
- url: 'https://example.com/file.bin',
378
- file_name: 'file.bin',
379
- parent_id: 'parent-2',
380
- },
381
- {
382
- id: 'att-with-ct-2',
383
- url: 'https://example.com/doc.pdf',
384
- file_name: 'doc.pdf',
385
- parent_id: 'parent-3',
386
- content_type: 'application/pdf',
387
- },
388
- ];
389
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
390
- adapter: mockAdapter,
391
- attachmentsMetadata,
392
- attachments: mixedAttachments,
393
- batchSize: 1,
394
- stream: mockStream,
395
- });
396
- await pool.streamAll();
397
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
398
- expect(mockAdapter.processAttachment.mock.calls[0][0].content_type).toBe('image/png');
399
- expect(mockAdapter.processAttachment.mock.calls[1][0].content_type).toBeUndefined();
400
- expect(mockAdapter.processAttachment.mock.calls[2][0].content_type).toBe('application/pdf');
401
- });
402
- it('should include content_type in error log when processAttachment returns error', async () => {
403
- mockAdapter.processAttachment.mockResolvedValue({
404
- error: { message: 'Upload failed' },
405
- });
406
- const warnSpy = jest.spyOn(console, 'warn');
407
- const attachmentWithContentType = {
408
- id: 'att-error-ct',
409
- url: 'https://example.com/file.pdf',
410
- file_name: 'file.pdf',
411
- parent_id: 'parent-err',
412
- content_type: 'application/pdf',
413
- };
414
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
415
- adapter: mockAdapter,
416
- attachmentsMetadata,
417
- attachments: [attachmentWithContentType],
418
- batchSize: 1,
419
- stream: mockStream,
420
- });
421
- await pool.streamAll();
422
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('content_type application/pdf'), expect.any(String));
423
- });
424
- it('should include content_type in error log when processAttachment throws', async () => {
425
- const error = new Error('Processing crashed');
426
- mockAdapter.processAttachment.mockRejectedValue(error);
427
- const warnSpy = jest.spyOn(console, 'warn');
428
- const attachmentWithContentType = {
429
- id: 'att-throw-ct',
430
- url: 'https://example.com/file.png',
431
- file_name: 'file.png',
432
- parent_id: 'parent-throw',
433
- content_type: 'image/png',
434
- };
435
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
436
- adapter: mockAdapter,
437
- attachmentsMetadata,
438
- attachments: [attachmentWithContentType],
439
- batchSize: 1,
440
- stream: mockStream,
441
- });
442
- await pool.streamAll();
443
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('content_type image/png'), error);
444
- });
445
- it('should not include content_type in error log when content_type is not set', async () => {
446
- mockAdapter.processAttachment.mockResolvedValue({
447
- error: { message: 'Upload failed' },
448
- });
449
- const warnSpy = jest.spyOn(console, 'warn');
450
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
451
- adapter: mockAdapter,
452
- attachmentsMetadata,
453
- attachments: [mockAttachments[0]],
454
- batchSize: 1,
455
- stream: mockStream,
456
- });
457
- await pool.streamAll();
458
- expect(warnSpy).toHaveBeenCalledWith(expect.not.stringContaining('content_type'), expect.any(String));
459
- });
460
- });
461
- describe('concurrency behavior', () => {
462
- it('should process attachments concurrently within batch size', async () => {
463
- let processCallCount = 0;
464
- const processPromises = [];
465
- mockAdapter.processAttachment.mockImplementation(async () => {
466
- const promise = new Promise((resolve) => {
467
- setTimeout(() => {
468
- processCallCount++;
469
- resolve({});
470
- }, 100);
471
- });
472
- processPromises.push(promise);
473
- return promise;
474
- });
475
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
476
- adapter: mockAdapter,
477
- attachmentsMetadata,
478
- attachments: mockAttachments,
479
- batchSize: 2,
480
- stream: mockStream,
481
- });
482
- const startTime = Date.now();
483
- await pool.streamAll();
484
- const endTime = Date.now();
485
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
486
- expect(processCallCount).toBe(3);
487
- // Should complete in roughly 200ms (2 batches of 100ms each) rather than 300ms (sequential)
488
- expect(endTime - startTime).toBeLessThan(250);
489
- });
490
- });
491
- describe('log context attribution', () => {
492
- it('should emit SDK-generated logs from pool (Starting download message)', async () => {
493
- const logSpy = jest.spyOn(console, 'log');
494
- mockAdapter.processAttachment.mockResolvedValue({});
495
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
496
- adapter: mockAdapter,
497
- attachmentsMetadata,
498
- attachments: mockAttachments.slice(0, 1),
499
- stream: mockStream,
500
- });
501
- // Call streamAll - it should log "Starting download of N attachments..."
502
- await pool.streamAll();
503
- // Verify the SDK-generated "Starting download" message was logged
504
- expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Starting download of 1 attachments'));
505
- });
506
- it('should process user stream callback correctly while maintaining context isolation', async () => {
507
- let userCallbackExecuted = false;
508
- // Mock stream to capture context info
509
- const mockStreamFn = jest
510
- .fn()
511
- .mockImplementation(async () => {
512
- await Promise.resolve();
513
- userCallbackExecuted = true;
514
- // Record that the callback executed
515
- return Promise.resolve({
516
- httpStream: undefined,
517
- error: undefined,
518
- });
519
- });
520
- mockAdapter.processAttachment.mockImplementation(async (attachment, stream) => {
521
- // processAttachment should be called with the user's stream function
522
- const result = await stream({
523
- item: attachment,
524
- event: {},
525
- });
526
- return result;
527
- });
528
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
529
- adapter: mockAdapter,
530
- attachmentsMetadata,
531
- attachments: mockAttachments.slice(0, 1),
532
- stream: mockStreamFn,
533
- });
534
- await pool.streamAll();
535
- // Verify the user callback was executed
536
- expect(userCallbackExecuted).toBe(true);
537
- expect(mockStreamFn).toHaveBeenCalled();
538
- });
539
- });
540
- });
@@ -1 +0,0 @@
1
- {"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB;AAED,eAAO,MAAM,IAAI,GAAU,6BAIxB,aAAa,KAAG,OAAO,CAAC,aAAa,CA4BvC,CAAC"}
@@ -1,6 +0,0 @@
1
- export declare const enum ExtractionCommonError {
2
- EXTERNAL_SYNC_UNIT_DELETED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED",
3
- EXTERNAL_SYNC_UNIT_DEACTIVATED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED",
4
- USER_DELETED = "ERROR_CODE=USER_DELETED"
5
- }
6
- //# sourceMappingURL=errors.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/common/errors.ts"],"names":[],"mappings":"AAGA,0BAAkB,qBAAqB;IACrC,0BAA0B,0CAAgE;IAC1F,8BAA8B,8CAAoE;IAClG,YAAY,4BAAkD;CAC/D"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"install-initial-domain-mapping.d.ts","sourceRoot":"","sources":["../../src/common/install-initial-domain-mapping.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,YAAY,EACnB,wBAAwB,EAAE,oBAAoB,GAC7C,OAAO,CAAC,IAAI,CAAC,CA0Ff"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=install-initial-domain-mapping.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"install-initial-domain-mapping.test.d.ts","sourceRoot":"","sources":["../../src/common/install-initial-domain-mapping.test.ts"],"names":[],"mappings":""}