@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +5 -11
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +1 -8
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +4 -3
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +11 -39
  7. package/dist/common/constants.d.ts +1 -0
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +2 -1
  10. package/dist/common/helpers.d.ts +0 -14
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +0 -36
  13. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  14. package/dist/http/client.d.ts.map +1 -0
  15. package/dist/index.d.ts +25 -13
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +51 -30
  18. package/dist/logger/logger.d.ts +14 -0
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.js +55 -8
  21. package/dist/mappers/mappers.d.ts +9 -10
  22. package/dist/mappers/mappers.d.ts.map +1 -1
  23. package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
  24. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  25. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
  26. package/dist/mappers/mappers.js +13 -9
  27. package/dist/multithreading/adapters/base-adapter.d.ts +71 -0
  28. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  29. package/dist/multithreading/adapters/base-adapter.js +138 -0
  30. package/dist/multithreading/adapters/extraction-adapter.d.ts +57 -0
  31. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  32. package/dist/multithreading/adapters/extraction-adapter.js +323 -0
  33. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  34. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  35. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  36. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  37. package/dist/multithreading/adapters/loading-adapter.js +431 -0
  38. package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
  39. package/dist/multithreading/emit.d.ts.map +1 -0
  40. package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
  41. package/dist/multithreading/process-task.d.ts +16 -1
  42. package/dist/multithreading/process-task.d.ts.map +1 -1
  43. package/dist/multithreading/process-task.js +115 -50
  44. package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
  45. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  46. package/dist/multithreading/spawn/spawn.helpers.js +152 -2
  47. package/dist/multithreading/spawn/spawn.js +7 -7
  48. package/dist/multithreading/worker.js +1 -2
  49. package/dist/repo/repo.js +2 -2
  50. package/dist/state/base-state.d.ts +75 -0
  51. package/dist/state/base-state.d.ts.map +1 -0
  52. package/dist/state/base-state.js +261 -0
  53. package/dist/state/extraction-state.d.ts +29 -0
  54. package/dist/state/extraction-state.d.ts.map +1 -0
  55. package/dist/state/extraction-state.js +122 -0
  56. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  57. package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
  58. package/dist/state/loading-state.d.ts +18 -0
  59. package/dist/state/loading-state.d.ts.map +1 -0
  60. package/dist/state/loading-state.js +39 -0
  61. package/dist/state/state.d.ts +3 -35
  62. package/dist/state/state.d.ts.map +1 -1
  63. package/dist/state/state.interfaces.d.ts +20 -3
  64. package/dist/state/state.interfaces.d.ts.map +1 -1
  65. package/dist/state/state.interfaces.js +19 -1
  66. package/dist/state/state.js +9 -263
  67. package/dist/{common → state}/time-value-resolver.d.ts +4 -5
  68. package/dist/state/time-value-resolver.d.ts.map +1 -0
  69. package/dist/{common → state}/time-value-resolver.js +9 -8
  70. package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
  71. package/dist/testing/mock-event.d.ts.map +1 -0
  72. package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
  73. package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
  74. package/dist/testing/mock-server.d.ts.map +1 -0
  75. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  76. package/dist/{mock-server → testing}/mock-server.js +6 -1
  77. package/dist/types/errors.d.ts +9 -0
  78. package/dist/types/errors.d.ts.map +1 -0
  79. package/dist/types/extraction.d.ts +73 -18
  80. package/dist/types/extraction.d.ts.map +1 -1
  81. package/dist/types/extraction.js +4 -6
  82. package/dist/types/loading.d.ts +1 -2
  83. package/dist/types/loading.d.ts.map +1 -1
  84. package/dist/types/loading.js +0 -1
  85. package/dist/types/workers.d.ts +64 -20
  86. package/dist/types/workers.d.ts.map +1 -1
  87. package/dist/uploader/uploader.d.ts +5 -4
  88. package/dist/uploader/uploader.d.ts.map +1 -1
  89. package/dist/uploader/uploader.interfaces.d.ts +1 -1
  90. package/dist/uploader/uploader.js +10 -10
  91. package/package.json +3 -3
  92. package/dist/airsync-sdk.d.ts +0 -1802
  93. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  94. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  95. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -516
  96. package/dist/common/control-protocol.d.ts.map +0 -1
  97. package/dist/common/errors.d.ts +0 -6
  98. package/dist/common/errors.d.ts.map +0 -1
  99. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  100. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  101. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  102. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  103. package/dist/common/test-utils.d.ts.map +0 -1
  104. package/dist/common/time-value-resolver.d.ts.map +0 -1
  105. package/dist/common/time-value-resolver.test.d.ts +0 -2
  106. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  107. package/dist/common/time-value-resolver.test.js +0 -268
  108. package/dist/http/axios-client-internal.d.ts.map +0 -1
  109. package/dist/http/axios-client-internal.test.d.ts +0 -2
  110. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  111. package/dist/http/axios-client-internal.test.js +0 -175
  112. package/dist/http/axios-client.d.ts +0 -27
  113. package/dist/http/axios-client.d.ts.map +0 -1
  114. package/dist/http/axios-client.js +0 -60
  115. package/dist/http/constants.d.ts +0 -4
  116. package/dist/http/constants.d.ts.map +0 -1
  117. package/dist/http/constants.js +0 -6
  118. package/dist/http/index.d.ts +0 -2
  119. package/dist/http/index.d.ts.map +0 -1
  120. package/dist/http/index.js +0 -17
  121. package/dist/logger/logger.constants.d.ts +0 -6
  122. package/dist/logger/logger.constants.d.ts.map +0 -1
  123. package/dist/logger/logger.constants.js +0 -13
  124. package/dist/logger/logger.context.d.ts +0 -58
  125. package/dist/logger/logger.context.d.ts.map +0 -1
  126. package/dist/logger/logger.context.js +0 -86
  127. package/dist/logger/logger.test.d.ts +0 -2
  128. package/dist/logger/logger.test.d.ts.map +0 -1
  129. package/dist/logger/logger.test.js +0 -537
  130. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  131. package/dist/mappers/mappers.test.d.ts +0 -2
  132. package/dist/mappers/mappers.test.d.ts.map +0 -1
  133. package/dist/mappers/mappers.test.js +0 -105
  134. package/dist/mock-server/mock-server.d.ts.map +0 -1
  135. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  136. package/dist/multithreading/create-worker.test.d.ts +0 -2
  137. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  138. package/dist/multithreading/create-worker.test.js +0 -108
  139. package/dist/multithreading/process-task.test.d.ts +0 -2
  140. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  141. package/dist/multithreading/process-task.test.js +0 -146
  142. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  143. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  144. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  145. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  146. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  147. package/dist/multithreading/spawn/spawn.test.js +0 -232
  148. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  149. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  150. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +0 -175
  151. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
  152. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  153. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
  154. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
  155. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +0 -390
  156. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
  157. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
  158. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +0 -799
  159. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  160. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  161. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  162. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -514
  163. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
  164. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
  165. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
  166. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +0 -596
  167. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
  168. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
  169. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +0 -71
  170. package/dist/repo/repo.test.d.ts +0 -2
  171. package/dist/repo/repo.test.d.ts.map +0 -1
  172. package/dist/repo/repo.test.js +0 -184
  173. package/dist/state/state.extract-window.test.d.ts +0 -2
  174. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  175. package/dist/state/state.extract-window.test.js +0 -163
  176. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  177. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  178. package/dist/state/state.pending-boundaries.test.js +0 -189
  179. package/dist/state/state.post-state.test.d.ts +0 -2
  180. package/dist/state/state.post-state.test.d.ts.map +0 -1
  181. package/dist/state/state.post-state.test.js +0 -77
  182. package/dist/state/state.test.d.ts +0 -2
  183. package/dist/state/state.test.d.ts.map +0 -1
  184. package/dist/state/state.test.js +0 -268
  185. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  186. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  187. package/dist/state/state.time-value-resolution.test.js +0 -175
  188. package/dist/tsdoc-metadata.json +0 -11
  189. package/dist/types/extraction.test.d.ts +0 -2
  190. package/dist/types/extraction.test.d.ts.map +0 -1
  191. package/dist/types/extraction.test.js +0 -96
  192. package/dist/types/index.d.ts +0 -11
  193. package/dist/types/index.d.ts.map +0 -1
  194. package/dist/types/index.js +0 -21
  195. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  196. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  197. package/dist/uploader/uploader.helpers.test.js +0 -256
  198. package/dist/uploader/uploader.test.d.ts +0 -2
  199. package/dist/uploader/uploader.test.d.ts.map +0 -1
  200. package/dist/uploader/uploader.test.js +0 -601
  201. /package/dist/http/{axios-client-internal.js → client.js} +0 -0
  202. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
  203. /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
  204. /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
  205. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  206. /package/dist/{common → types}/errors.js +0 -0
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=attachments-streaming-pool.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"attachments-streaming-pool.test.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.test.ts"],"names":[],"mappings":""}
@@ -1,516 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
4
- /* eslint-disable @typescript-eslint/no-explicit-any */
5
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
6
- let mockAdapter;
7
- let mockStream;
8
- let mockAttachments;
9
- beforeEach(() => {
10
- // Create mock adapter
11
- mockAdapter = {
12
- state: {
13
- attachments: { completed: false },
14
- toDevRev: {
15
- attachmentsMetadata: {
16
- lastProcessedAttachmentsIdsList: [],
17
- },
18
- },
19
- },
20
- processAttachment: jest
21
- .fn()
22
- .mockResolvedValue({}),
23
- };
24
- // Create mock stream function
25
- mockStream = jest.fn().mockResolvedValue({ success: true });
26
- // Create mock attachments
27
- mockAttachments = [
28
- {
29
- id: 'attachment-1',
30
- url: 'https://example.com/file1.pdf',
31
- file_name: 'file1.pdf',
32
- parent_id: 'parent-1',
33
- },
34
- {
35
- id: 'attachment-2',
36
- url: 'https://example.com/file2.jpg',
37
- file_name: 'file2.jpg',
38
- parent_id: 'parent-2',
39
- },
40
- {
41
- id: 'attachment-3',
42
- url: 'https://example.com/file3.doc',
43
- file_name: 'file3.doc',
44
- parent_id: 'parent-3',
45
- },
46
- ];
47
- });
48
- afterEach(() => {
49
- jest.clearAllMocks();
50
- jest.restoreAllMocks();
51
- });
52
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.constructor.name, () => {
53
- it('should initialize with default values', () => {
54
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
55
- adapter: mockAdapter,
56
- attachments: mockAttachments,
57
- stream: mockStream,
58
- });
59
- expect(pool['adapter']).toBe(mockAdapter);
60
- expect(pool['attachments']).toEqual(mockAttachments);
61
- expect(pool['batchSize']).toBe(10);
62
- expect(pool['stream']).toBe(mockStream);
63
- });
64
- it('should initialize with custom batch size', () => {
65
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
66
- adapter: mockAdapter,
67
- attachments: mockAttachments,
68
- batchSize: 5,
69
- stream: mockStream,
70
- });
71
- expect(pool['batchSize']).toBe(5);
72
- });
73
- it('should create a copy of attachments array', () => {
74
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
75
- adapter: mockAdapter,
76
- attachments: mockAttachments,
77
- stream: mockStream,
78
- });
79
- expect(pool['attachments']).toEqual(mockAttachments);
80
- expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
81
- });
82
- });
83
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.streamAll.name, () => {
84
- it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
85
- mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
86
- undefined;
87
- mockAdapter.processAttachment.mockResolvedValue({});
88
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
89
- adapter: mockAdapter,
90
- attachments: mockAttachments,
91
- stream: mockStream,
92
- });
93
- // Mock startPoolStreaming to avoid actual processing
94
- jest
95
- .spyOn(pool, 'startPoolStreaming')
96
- .mockResolvedValue(undefined);
97
- await pool.streamAll();
98
- expect(mockAdapter.state.toDevRev.attachmentsMetadata
99
- .lastProcessedAttachmentsIdsList).toEqual([]);
100
- });
101
- it('should process all attachments successfully', async () => {
102
- mockAdapter.processAttachment.mockResolvedValue({});
103
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
104
- adapter: mockAdapter,
105
- attachments: mockAttachments,
106
- stream: mockStream,
107
- });
108
- const result = await pool.streamAll();
109
- expect(result).toEqual({});
110
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
111
- });
112
- it('should handle empty attachments array', async () => {
113
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
114
- adapter: mockAdapter,
115
- attachments: [],
116
- stream: mockStream,
117
- });
118
- const result = await pool.streamAll();
119
- expect(result).toEqual({});
120
- expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
121
- });
122
- it('should return delay when rate limit is hit', async () => {
123
- const delayResponse = { delay: 5000 };
124
- mockAdapter.processAttachment.mockResolvedValue(delayResponse);
125
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
126
- adapter: mockAdapter,
127
- attachments: mockAttachments,
128
- stream: mockStream,
129
- });
130
- const result = await pool.streamAll();
131
- expect(result).toEqual({ delay: 5000 });
132
- });
133
- it('should resume attachment extraction if it encounters old ids', async () => {
134
- var _a;
135
- // Test migration from old string[] format to new ProcessedAttachment[] format
136
- // Using 'as any' because we're intentionally testing legacy data format
137
- mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
138
- ['attachment-1', 'attachment-2'];
139
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
140
- adapter: mockAdapter,
141
- attachments: mockAttachments,
142
- stream: mockStream,
143
- });
144
- const result = await pool.streamAll();
145
- expect((_a = mockAdapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
146
- { id: 'attachment-1', parent_id: '' },
147
- { id: 'attachment-2', parent_id: '' },
148
- { id: 'attachment-1', parent_id: 'parent-1' },
149
- { id: 'attachment-2', parent_id: 'parent-2' },
150
- { id: 'attachment-3', parent_id: 'parent-3' },
151
- ]);
152
- expect(result).toEqual({});
153
- });
154
- it('should handle all attachments failing with different file types and sizes', async () => {
155
- const largeAttachments = [
156
- {
157
- id: 'attachment-image',
158
- url: 'https://example.com/photo.jpg',
159
- file_name: 'photo.jpg',
160
- parent_id: 'parent-1',
161
- },
162
- {
163
- id: 'attachment-pdf',
164
- url: 'https://example.com/document.pdf',
165
- file_name: 'document.pdf',
166
- parent_id: 'parent-2',
167
- },
168
- {
169
- id: 'attachment-video',
170
- url: 'https://example.com/video.mp4',
171
- file_name: 'video.mp4',
172
- parent_id: 'parent-3',
173
- },
174
- ];
175
- const imageError = new Error('Image upload failed: File too large');
176
- const pdfError = new Error('PDF upload failed: Unsupported format');
177
- const videoError = new Error('Video upload failed: Network error');
178
- mockAdapter.processAttachment
179
- .mockRejectedValueOnce(imageError)
180
- .mockRejectedValueOnce(pdfError)
181
- .mockRejectedValueOnce(videoError);
182
- const warnSpy = jest.spyOn(console, 'warn');
183
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
184
- adapter: mockAdapter,
185
- attachments: largeAttachments,
186
- stream: mockStream,
187
- });
188
- await pool.streamAll();
189
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
190
- // Since there are 3 parallel workers (one per attachment), each logs its own failed attachment
191
- expect(warnSpy).toHaveBeenCalledTimes(3);
192
- expect(mockAdapter.state.toDevRev.attachmentsMetadata
193
- .lastProcessedAttachmentsIdsList).toEqual([]);
194
- });
195
- });
196
- describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
197
- it('should skip already processed attachments', async () => {
198
- mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
199
- [{ id: 'attachment-1', parent_id: 'parent-1' }];
200
- mockAdapter.processAttachment.mockResolvedValue({});
201
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
202
- adapter: mockAdapter,
203
- attachments: mockAttachments,
204
- stream: mockStream,
205
- });
206
- await pool.streamAll();
207
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
208
- });
209
- it('should add successfully processed attachment IDs to the list', async () => {
210
- mockAdapter.processAttachment.mockResolvedValue({});
211
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
212
- adapter: mockAdapter,
213
- attachments: mockAttachments,
214
- stream: mockStream,
215
- });
216
- await pool.streamAll();
217
- expect(mockAdapter.state.toDevRev.attachmentsMetadata
218
- .lastProcessedAttachmentsIdsList).toEqual([
219
- { id: 'attachment-1', parent_id: 'parent-1' },
220
- { id: 'attachment-2', parent_id: 'parent-2' },
221
- { id: 'attachment-3', parent_id: 'parent-3' },
222
- ]);
223
- });
224
- it('should handle processing errors gracefully', async () => {
225
- const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
226
- const error = new Error('Processing failed');
227
- mockAdapter.processAttachment
228
- .mockResolvedValueOnce({}) // First attachment succeeds
229
- .mockRejectedValueOnce(error) // Second attachment fails
230
- .mockResolvedValueOnce({}); // Third attachment succeeds
231
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
232
- adapter: mockAdapter,
233
- attachments: mockAttachments,
234
- stream: mockStream,
235
- });
236
- await pool.streamAll();
237
- expect(warnSpy).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 with extension jpg due to error in processAttachment function', error);
238
- expect(mockAdapter.state.toDevRev.attachmentsMetadata
239
- .lastProcessedAttachmentsIdsList).toEqual([
240
- {
241
- id: 'attachment-1',
242
- parent_id: 'parent-1',
243
- },
244
- {
245
- id: 'attachment-3',
246
- parent_id: 'parent-3',
247
- },
248
- ]);
249
- });
250
- it('should stop processing when rate limit delay is encountered', async () => {
251
- mockAdapter.processAttachment
252
- .mockResolvedValueOnce({}) // First attachment succeeds
253
- .mockResolvedValueOnce({ delay: 5000 }) // Second attachment triggers rate limit
254
- .mockResolvedValueOnce({}); // Third attachment succeeds
255
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
256
- adapter: mockAdapter,
257
- attachments: mockAttachments,
258
- stream: mockStream,
259
- });
260
- await pool.streamAll();
261
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
262
- expect(mockAdapter.state.toDevRev.attachmentsMetadata
263
- .lastProcessedAttachmentsIdsList).toEqual([
264
- { id: 'attachment-1', parent_id: 'parent-1' },
265
- { id: 'attachment-3', parent_id: 'parent-3' },
266
- ]);
267
- });
268
- it('should pass correct parameters to processAttachment', async () => {
269
- mockAdapter.processAttachment.mockResolvedValue({});
270
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
271
- adapter: mockAdapter,
272
- attachments: [mockAttachments[0]],
273
- stream: mockStream,
274
- });
275
- await pool.streamAll();
276
- expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
277
- });
278
- });
279
- it('[edge] should handle single attachment', async () => {
280
- mockAdapter.processAttachment.mockResolvedValue({});
281
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
282
- adapter: mockAdapter,
283
- attachments: [mockAttachments[0]],
284
- stream: mockStream,
285
- });
286
- const result = await pool.streamAll();
287
- expect(result).toEqual({});
288
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
289
- });
290
- it('[edge] should handle batch size larger than attachments array', async () => {
291
- mockAdapter.processAttachment.mockResolvedValue({});
292
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
293
- adapter: mockAdapter,
294
- attachments: mockAttachments,
295
- batchSize: 100,
296
- stream: mockStream,
297
- });
298
- await pool.streamAll();
299
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
300
- });
301
- it('[edge] should upload attachments with same id, but different parent_id', async () => {
302
- mockAdapter.processAttachment.mockResolvedValue({});
303
- mockAttachments.push({
304
- id: 'attachment-1',
305
- url: 'http://example.com/file5.jpg',
306
- file_name: 'file5.jpg',
307
- parent_id: 'parent-4',
308
- });
309
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
310
- adapter: mockAdapter,
311
- attachments: mockAttachments,
312
- batchSize: 1,
313
- stream: mockStream,
314
- });
315
- await pool.streamAll();
316
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(4);
317
- });
318
- it('[edge] should handle batch size of 1', async () => {
319
- mockAdapter.processAttachment.mockResolvedValue({});
320
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
321
- adapter: mockAdapter,
322
- attachments: mockAttachments,
323
- batchSize: 1,
324
- stream: mockStream,
325
- });
326
- await pool.streamAll();
327
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
328
- });
329
- describe('content_type handling', () => {
330
- it('should pass attachment with content_type to processAttachment', async () => {
331
- mockAdapter.processAttachment.mockResolvedValue({});
332
- const attachmentWithContentType = {
333
- id: 'attachment-ct',
334
- url: 'https://example.com/report.pdf',
335
- file_name: 'report.pdf',
336
- parent_id: 'parent-ct',
337
- content_type: 'application/pdf',
338
- };
339
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
340
- adapter: mockAdapter,
341
- attachments: [attachmentWithContentType],
342
- stream: mockStream,
343
- });
344
- await pool.streamAll();
345
- expect(mockAdapter.processAttachment).toHaveBeenCalledWith(attachmentWithContentType, mockStream);
346
- expect(mockAdapter.processAttachment.mock.calls[0][0].content_type).toBe('application/pdf');
347
- });
348
- it('should handle mixed attachments with and without content_type', async () => {
349
- mockAdapter.processAttachment.mockResolvedValue({});
350
- const mixedAttachments = [
351
- {
352
- id: 'att-with-ct',
353
- url: 'https://example.com/image.png',
354
- file_name: 'image.png',
355
- parent_id: 'parent-1',
356
- content_type: 'image/png',
357
- },
358
- {
359
- id: 'att-without-ct',
360
- url: 'https://example.com/file.bin',
361
- file_name: 'file.bin',
362
- parent_id: 'parent-2',
363
- },
364
- {
365
- id: 'att-with-ct-2',
366
- url: 'https://example.com/doc.pdf',
367
- file_name: 'doc.pdf',
368
- parent_id: 'parent-3',
369
- content_type: 'application/pdf',
370
- },
371
- ];
372
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
373
- adapter: mockAdapter,
374
- attachments: mixedAttachments,
375
- batchSize: 1,
376
- stream: mockStream,
377
- });
378
- await pool.streamAll();
379
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
380
- expect(mockAdapter.processAttachment.mock.calls[0][0].content_type).toBe('image/png');
381
- expect(mockAdapter.processAttachment.mock.calls[1][0].content_type).toBeUndefined();
382
- expect(mockAdapter.processAttachment.mock.calls[2][0].content_type).toBe('application/pdf');
383
- });
384
- it('should include content_type in error log when processAttachment returns error', async () => {
385
- mockAdapter.processAttachment.mockResolvedValue({
386
- error: { message: 'Upload failed' },
387
- });
388
- const warnSpy = jest.spyOn(console, 'warn');
389
- const attachmentWithContentType = {
390
- id: 'att-error-ct',
391
- url: 'https://example.com/file.pdf',
392
- file_name: 'file.pdf',
393
- parent_id: 'parent-err',
394
- content_type: 'application/pdf',
395
- };
396
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
397
- adapter: mockAdapter,
398
- attachments: [attachmentWithContentType],
399
- batchSize: 1,
400
- stream: mockStream,
401
- });
402
- await pool.streamAll();
403
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('content_type application/pdf'), expect.any(String));
404
- });
405
- it('should include content_type in error log when processAttachment throws', async () => {
406
- const error = new Error('Processing crashed');
407
- mockAdapter.processAttachment.mockRejectedValue(error);
408
- const warnSpy = jest.spyOn(console, 'warn');
409
- const attachmentWithContentType = {
410
- id: 'att-throw-ct',
411
- url: 'https://example.com/file.png',
412
- file_name: 'file.png',
413
- parent_id: 'parent-throw',
414
- content_type: 'image/png',
415
- };
416
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
417
- adapter: mockAdapter,
418
- attachments: [attachmentWithContentType],
419
- batchSize: 1,
420
- stream: mockStream,
421
- });
422
- await pool.streamAll();
423
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('content_type image/png'), error);
424
- });
425
- it('should not include content_type in error log when content_type is not set', async () => {
426
- mockAdapter.processAttachment.mockResolvedValue({
427
- error: { message: 'Upload failed' },
428
- });
429
- const warnSpy = jest.spyOn(console, 'warn');
430
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
431
- adapter: mockAdapter,
432
- attachments: [mockAttachments[0]],
433
- batchSize: 1,
434
- stream: mockStream,
435
- });
436
- await pool.streamAll();
437
- expect(warnSpy).toHaveBeenCalledWith(expect.not.stringContaining('content_type'), expect.any(String));
438
- });
439
- });
440
- describe('concurrency behavior', () => {
441
- it('should process attachments concurrently within batch size', async () => {
442
- let processCallCount = 0;
443
- const processPromises = [];
444
- mockAdapter.processAttachment.mockImplementation(async () => {
445
- const promise = new Promise((resolve) => {
446
- setTimeout(() => {
447
- processCallCount++;
448
- resolve({});
449
- }, 100);
450
- });
451
- processPromises.push(promise);
452
- return promise;
453
- });
454
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
455
- adapter: mockAdapter,
456
- attachments: mockAttachments,
457
- batchSize: 2,
458
- stream: mockStream,
459
- });
460
- const startTime = Date.now();
461
- await pool.streamAll();
462
- const endTime = Date.now();
463
- expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
464
- expect(processCallCount).toBe(3);
465
- // Should complete in roughly 200ms (2 batches of 100ms each) rather than 300ms (sequential)
466
- expect(endTime - startTime).toBeLessThan(250);
467
- });
468
- });
469
- describe('log context attribution', () => {
470
- it('should emit SDK-generated logs from pool (Starting download message)', async () => {
471
- const logSpy = jest.spyOn(console, 'log');
472
- mockAdapter.processAttachment.mockResolvedValue({});
473
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
474
- adapter: mockAdapter,
475
- attachments: mockAttachments.slice(0, 1),
476
- stream: mockStream,
477
- });
478
- // Call streamAll - it should log "Starting download of N attachments..."
479
- await pool.streamAll();
480
- // Verify the SDK-generated "Starting download" message was logged
481
- expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Starting download of 1 attachments'));
482
- });
483
- it('should process user stream callback correctly while maintaining context isolation', async () => {
484
- let userCallbackExecuted = false;
485
- // Mock stream to capture context info
486
- const mockStreamFn = jest
487
- .fn()
488
- .mockImplementation(async () => {
489
- await Promise.resolve();
490
- userCallbackExecuted = true;
491
- // Record that the callback executed
492
- return Promise.resolve({
493
- httpStream: undefined,
494
- error: undefined,
495
- });
496
- });
497
- mockAdapter.processAttachment.mockImplementation(async (attachment, stream) => {
498
- // processAttachment should be called with the user's stream function
499
- const result = await stream({
500
- item: attachment,
501
- event: {},
502
- });
503
- return result;
504
- });
505
- const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
506
- adapter: mockAdapter,
507
- attachments: mockAttachments.slice(0, 1),
508
- stream: mockStreamFn,
509
- });
510
- await pool.streamAll();
511
- // Verify the user callback was executed
512
- expect(userCallbackExecuted).toBe(true);
513
- expect(mockStreamFn).toHaveBeenCalled();
514
- });
515
- });
516
- });
@@ -1 +0,0 @@
1
- {"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB;AAED,eAAO,MAAM,IAAI,GAAU,6BAIxB,aAAa,KAAG,OAAO,CAAC,aAAa,CA4BvC,CAAC"}
@@ -1,6 +0,0 @@
1
- export declare const enum ExtractionCommonError {
2
- EXTERNAL_SYNC_UNIT_DELETED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED",
3
- EXTERNAL_SYNC_UNIT_DEACTIVATED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED",
4
- USER_DELETED = "ERROR_CODE=USER_DELETED"
5
- }
6
- //# sourceMappingURL=errors.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/common/errors.ts"],"names":[],"mappings":"AAGA,0BAAkB,qBAAqB;IACrC,0BAA0B,0CAAgE;IAC1F,8BAA8B,8CAAoE;IAClG,YAAY,4BAAkD;CAC/D"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"install-initial-domain-mapping.d.ts","sourceRoot":"","sources":["../../src/common/install-initial-domain-mapping.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,YAAY,EACnB,wBAAwB,EAAE,oBAAoB,GAC7C,OAAO,CAAC,IAAI,CAAC,CA0Ff"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=install-initial-domain-mapping.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"install-initial-domain-mapping.test.d.ts","sourceRoot":"","sources":["../../src/common/install-initial-domain-mapping.test.ts"],"names":[],"mappings":""}