@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +2059 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +8 -2
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +27 -16
  8. package/dist/attachments-streaming/attachments-streaming-pool.test.js +246 -27
  9. package/dist/common/constants.d.ts +16 -5
  10. package/dist/common/constants.d.ts.map +1 -1
  11. package/dist/common/constants.js +20 -6
  12. package/dist/common/control-protocol.d.ts +2 -2
  13. package/dist/common/control-protocol.d.ts.map +1 -1
  14. package/dist/common/control-protocol.js +1 -5
  15. package/dist/common/helpers.d.ts +8 -0
  16. package/dist/common/helpers.d.ts.map +1 -1
  17. package/dist/common/helpers.js +15 -0
  18. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  19. package/dist/common/install-initial-domain-mapping.test.js +5 -2
  20. package/dist/common/test-utils.d.ts +19 -0
  21. package/dist/common/test-utils.d.ts.map +1 -0
  22. package/dist/common/test-utils.js +101 -0
  23. package/dist/common/time-value-resolver.d.ts +47 -0
  24. package/dist/common/time-value-resolver.d.ts.map +1 -0
  25. package/dist/common/time-value-resolver.js +141 -0
  26. package/dist/common/time-value-resolver.test.d.ts +2 -0
  27. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  28. package/dist/common/time-value-resolver.test.js +268 -0
  29. package/dist/http/axios-client-internal.d.ts.map +1 -1
  30. package/dist/http/axios-client-internal.js +7 -0
  31. package/dist/http/axios-client-internal.test.js +21 -0
  32. package/dist/http/axios-client.d.ts.map +1 -1
  33. package/dist/http/axios-client.js +6 -3
  34. package/dist/http/index.d.ts +0 -1
  35. package/dist/http/index.d.ts.map +1 -1
  36. package/dist/http/index.js +0 -1
  37. package/dist/index.d.ts +12 -10
  38. package/dist/index.d.ts.map +1 -1
  39. package/dist/index.js +18 -13
  40. package/dist/logger/logger.d.ts +1 -17
  41. package/dist/logger/logger.d.ts.map +1 -1
  42. package/dist/logger/logger.interfaces.d.ts +2 -2
  43. package/dist/logger/logger.js +35 -27
  44. package/dist/logger/logger.test.js +66 -19
  45. package/dist/mappers/mappers.interface.d.ts +5 -5
  46. package/dist/mappers/mappers.interface.js +3 -3
  47. package/dist/mappers/mappers.test.js +8 -10
  48. package/dist/mock-server/mock-server.d.ts +47 -0
  49. package/dist/mock-server/mock-server.d.ts.map +1 -0
  50. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  51. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  52. package/dist/mock-server/mock-server.interfaces.js +4 -0
  53. package/dist/mock-server/mock-server.js +278 -0
  54. package/dist/multithreading/adapters/base-adapter.d.ts +56 -0
  55. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  56. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +2 -0
  57. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +1 -0
  58. package/dist/multithreading/adapters/base-adapter.emit.test.js +403 -0
  59. package/dist/multithreading/adapters/base-adapter.js +105 -0
  60. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +2 -0
  61. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +1 -0
  62. package/dist/multithreading/{worker-adapter/worker-adapter.artifacts.test.js → adapters/extraction-adapter.artifacts.test.js} +53 -5
  63. package/dist/multithreading/adapters/extraction-adapter.d.ts +58 -0
  64. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  65. package/dist/multithreading/adapters/extraction-adapter.js +325 -0
  66. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +2 -0
  67. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +1 -0
  68. package/dist/multithreading/adapters/extraction-adapter.test.js +789 -0
  69. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  70. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  71. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  72. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  73. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +2 -0
  74. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +1 -0
  75. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.test.js → adapters/loading-adapter.helpers.test.js} +26 -26
  76. package/dist/multithreading/adapters/loading-adapter.js +461 -0
  77. package/dist/multithreading/adapters/loading-adapter.test.d.ts +2 -0
  78. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +1 -0
  79. package/dist/multithreading/adapters/loading-adapter.test.js +589 -0
  80. package/dist/multithreading/adapters/serialization.test.d.ts +2 -0
  81. package/dist/multithreading/adapters/serialization.test.d.ts.map +1 -0
  82. package/dist/multithreading/adapters/serialization.test.js +71 -0
  83. package/dist/multithreading/create-worker.test.js +36 -17
  84. package/dist/multithreading/process-task.d.ts +16 -1
  85. package/dist/multithreading/process-task.d.ts.map +1 -1
  86. package/dist/multithreading/process-task.js +97 -82
  87. package/dist/multithreading/process-task.test.d.ts +2 -0
  88. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  89. package/dist/multithreading/process-task.test.js +146 -0
  90. package/dist/multithreading/spawn/spawn.d.ts +3 -3
  91. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  92. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  93. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  94. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  95. package/dist/multithreading/spawn/spawn.js +20 -25
  96. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  97. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  98. package/dist/multithreading/spawn/spawn.test.js +232 -0
  99. package/dist/repo/repo.d.ts.map +1 -1
  100. package/dist/repo/repo.interfaces.d.ts +4 -2
  101. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  102. package/dist/repo/repo.js +4 -3
  103. package/dist/repo/repo.test.js +60 -7
  104. package/dist/state/base-state.d.ts +75 -0
  105. package/dist/state/base-state.d.ts.map +1 -0
  106. package/dist/state/base-state.js +261 -0
  107. package/dist/state/extraction-state.d.ts +29 -0
  108. package/dist/state/extraction-state.d.ts.map +1 -0
  109. package/dist/state/extraction-state.js +122 -0
  110. package/dist/state/loading-state.d.ts +19 -0
  111. package/dist/state/loading-state.d.ts.map +1 -0
  112. package/dist/state/loading-state.js +40 -0
  113. package/dist/state/state.extract-window.test.d.ts +2 -0
  114. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  115. package/dist/state/state.extract-window.test.js +163 -0
  116. package/dist/state/state.interfaces.d.ts +44 -27
  117. package/dist/state/state.interfaces.d.ts.map +1 -1
  118. package/dist/state/state.interfaces.js +18 -5
  119. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  120. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  121. package/dist/state/state.pending-boundaries.test.js +189 -0
  122. package/dist/state/state.post-state.test.d.ts +2 -0
  123. package/dist/state/state.post-state.test.d.ts.map +1 -0
  124. package/dist/state/state.post-state.test.js +78 -0
  125. package/dist/state/state.test.js +135 -88
  126. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  127. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  128. package/dist/state/state.time-value-resolution.test.js +175 -0
  129. package/dist/tsdoc-metadata.json +11 -0
  130. package/dist/types/common.d.ts +0 -25
  131. package/dist/types/common.d.ts.map +1 -1
  132. package/dist/types/common.js +1 -11
  133. package/dist/types/external-domain-metadata.d.ts +20 -1
  134. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  135. package/dist/types/extraction.d.ts +92 -221
  136. package/dist/types/extraction.d.ts.map +1 -1
  137. package/dist/types/extraction.js +56 -124
  138. package/dist/types/extraction.test.js +69 -43
  139. package/dist/types/index.d.ts +3 -3
  140. package/dist/types/index.d.ts.map +1 -1
  141. package/dist/types/index.js +6 -3
  142. package/dist/types/loading.d.ts +4 -24
  143. package/dist/types/loading.d.ts.map +1 -1
  144. package/dist/types/loading.js +1 -22
  145. package/dist/types/workers.d.ts +47 -29
  146. package/dist/types/workers.d.ts.map +1 -1
  147. package/dist/types/workers.js +1 -0
  148. package/dist/uploader/uploader.d.ts +1 -0
  149. package/dist/uploader/uploader.d.ts.map +1 -1
  150. package/dist/uploader/uploader.helpers.test.js +0 -11
  151. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  152. package/dist/uploader/uploader.js +53 -20
  153. package/dist/uploader/uploader.test.js +22 -10
  154. package/package.json +10 -13
  155. package/dist/common/event-type-translation.d.ts +0 -24
  156. package/dist/common/event-type-translation.d.ts.map +0 -1
  157. package/dist/common/event-type-translation.js +0 -117
  158. package/dist/deprecated/adapter/index.d.ts +0 -62
  159. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  160. package/dist/deprecated/adapter/index.js +0 -151
  161. package/dist/deprecated/common/helpers.d.ts +0 -7
  162. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  163. package/dist/deprecated/common/helpers.js +0 -47
  164. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  165. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  166. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  167. package/dist/deprecated/demo-extractor/index.js +0 -161
  168. package/dist/deprecated/http/client.d.ts +0 -22
  169. package/dist/deprecated/http/client.d.ts.map +0 -1
  170. package/dist/deprecated/http/client.js +0 -161
  171. package/dist/deprecated/uploader/index.d.ts +0 -35
  172. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  173. package/dist/deprecated/uploader/index.js +0 -161
  174. package/dist/http/types.d.ts +0 -17
  175. package/dist/http/types.d.ts.map +0 -1
  176. package/dist/http/types.js +0 -2
  177. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  178. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  179. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -91
  180. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  181. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  182. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  183. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  184. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -745
  185. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  186. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  187. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
  188. package/dist/state/state.d.ts +0 -30
  189. package/dist/state/state.d.ts.map +0 -1
  190. package/dist/state/state.js +0 -166
  191. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.test.ts"],"names":[],"mappings":""}
@@ -1,483 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
4
- const state_1 = require("../../state/state");
5
- const test_helpers_1 = require("../../tests/test-helpers");
6
- const types_1 = require("../../types");
7
- const worker_adapter_1 = require("./worker-adapter");
8
- /* eslint-disable @typescript-eslint/no-require-imports */
9
- // Mock dependencies
10
- jest.mock('../../common/control-protocol', () => ({
11
- emit: jest.fn().mockResolvedValue({}),
12
- }));
13
- // const mockPostState = jest.spyOn(State.prototype, 'postState').mockResolvedValue(); // Mock to resolve void
14
- // const mockFetchState = jest.spyOn(State.prototype, 'fetchState').mockResolvedValue({}); // Mock to resolve a default state
15
- jest.mock('../../mappers/mappers');
16
- jest.mock('../../uploader/uploader');
17
- // jest.mock('../../state/state');
18
- jest.mock('../../repo/repo');
19
- jest.mock('node:worker_threads', () => ({
20
- parentPort: {
21
- postMessage: jest.fn(),
22
- },
23
- }));
24
- jest.mock('../../attachments-streaming/attachments-streaming-pool', () => {
25
- return {
26
- AttachmentsStreamingPool: jest.fn().mockImplementation(() => {
27
- return {
28
- streamAll: jest.fn().mockResolvedValue(undefined),
29
- };
30
- }),
31
- };
32
- });
33
- describe(worker_adapter_1.WorkerAdapter.name, () => {
34
- let adapter;
35
- let mockEvent;
36
- let mockAdapterState;
37
- beforeEach(() => {
38
- // Reset all mocks
39
- jest.clearAllMocks();
40
- // Create mock objects
41
- mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.StartExtractingData });
42
- const initialState = {
43
- attachments: { completed: false },
44
- lastSyncStarted: '',
45
- lastSuccessfulSyncStarted: '',
46
- snapInVersionId: '',
47
- toDevRev: {
48
- attachmentsMetadata: {
49
- artifactIds: [],
50
- lastProcessed: 0,
51
- lastProcessedAttachmentsIdsList: [],
52
- },
53
- },
54
- };
55
- mockAdapterState = new state_1.State({
56
- event: mockEvent,
57
- initialState: initialState,
58
- });
59
- // Create the adapter instance
60
- adapter = new worker_adapter_1.WorkerAdapter({
61
- event: mockEvent,
62
- adapterState: mockAdapterState,
63
- });
64
- });
65
- describe(worker_adapter_1.WorkerAdapter.prototype.streamAttachments.name, () => {
66
- it('should process all artifact batches successfully', async () => {
67
- const mockStream = jest.fn();
68
- // Set up adapter state with artifact IDs
69
- adapter.state.toDevRev = {
70
- attachmentsMetadata: {
71
- artifactIds: ['artifact1', 'artifact2'],
72
- lastProcessed: 0,
73
- lastProcessedAttachmentsIdsList: [],
74
- },
75
- };
76
- // Mock getting attachments from each artifact
77
- adapter['uploader'].getAttachmentsFromArtifactId = jest
78
- .fn()
79
- .mockResolvedValueOnce({
80
- attachments: [
81
- {
82
- url: 'http://example.com/file1.pdf',
83
- id: 'attachment1',
84
- file_name: 'file1.pdf',
85
- parent_id: 'parent1',
86
- },
87
- {
88
- url: 'http://example.com/file2.pdf',
89
- id: 'attachment2',
90
- file_name: 'file2.pdf',
91
- parent_id: 'parent2',
92
- },
93
- ],
94
- })
95
- .mockResolvedValueOnce({
96
- attachments: [
97
- {
98
- url: 'http://example.com/file3.pdf',
99
- id: 'attachment3',
100
- file_name: 'file3.pdf',
101
- parent_id: 'parent3',
102
- },
103
- ],
104
- });
105
- // Mock the initializeRepos method
106
- adapter.initializeRepos = jest.fn();
107
- const result = await adapter.streamAttachments({
108
- stream: mockStream,
109
- });
110
- expect(adapter.initializeRepos).toHaveBeenCalledWith([
111
- { itemType: 'ssor_attachment' },
112
- ]);
113
- expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
114
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
115
- // Verify state was updated correctly
116
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
117
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
118
- expect(result).toBeUndefined();
119
- });
120
- it('[edge] should handle invalid batch size by using 1 instead', async () => {
121
- const mockStream = jest.fn();
122
- // Set up adapter state with artifact IDs
123
- adapter.state.toDevRev = {
124
- attachmentsMetadata: {
125
- artifactIds: ['artifact1'],
126
- lastProcessed: 0,
127
- lastProcessedAttachmentsIdsList: [],
128
- },
129
- };
130
- // Mock getting attachments
131
- adapter['uploader'].getAttachmentsFromArtifactId = jest
132
- .fn()
133
- .mockResolvedValue({
134
- attachments: [
135
- {
136
- url: 'http://example.com/file1.pdf',
137
- id: 'attachment1',
138
- file_name: 'file1.pdf',
139
- parent_id: 'parent1',
140
- },
141
- ],
142
- });
143
- adapter.initializeRepos = jest.fn();
144
- const result = await adapter.streamAttachments({
145
- stream: mockStream,
146
- batchSize: 0,
147
- });
148
- expect(result).toBeUndefined();
149
- });
150
- it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
151
- const mockStream = jest.fn();
152
- // Set up adapter state with artifact IDs
153
- adapter.state.toDevRev = {
154
- attachmentsMetadata: {
155
- artifactIds: ['artifact1'],
156
- lastProcessed: 0,
157
- lastProcessedAttachmentsIdsList: [],
158
- },
159
- };
160
- // Mock getting attachments
161
- adapter['uploader'].getAttachmentsFromArtifactId = jest
162
- .fn()
163
- .mockResolvedValue({
164
- attachments: [
165
- {
166
- url: 'http://example.com/file1.pdf',
167
- id: 'attachment1',
168
- file_name: 'file1.pdf',
169
- parent_id: 'parent1',
170
- },
171
- ],
172
- });
173
- // Mock the required methods
174
- adapter.initializeRepos = jest.fn();
175
- const result = await adapter.streamAttachments({
176
- stream: mockStream,
177
- batchSize: 100, // Set batch size greater than 50
178
- });
179
- expect(result).toBeUndefined();
180
- });
181
- it('[edge] should handle empty attachments metadata artifact IDs', async () => {
182
- const mockStream = jest.fn();
183
- // Set up adapter state with no artifact IDs
184
- adapter.state.toDevRev = {
185
- attachmentsMetadata: {
186
- artifactIds: [],
187
- lastProcessed: 0,
188
- },
189
- };
190
- const result = await adapter.streamAttachments({
191
- stream: mockStream,
192
- });
193
- expect(result).toBeUndefined();
194
- });
195
- it('[edge] should handle errors when getting attachments', async () => {
196
- const mockStream = jest.fn();
197
- // Set up adapter state with artifact IDs
198
- adapter.state.toDevRev = {
199
- attachmentsMetadata: {
200
- artifactIds: ['artifact1'],
201
- lastProcessed: 0,
202
- lastProcessedAttachmentsIdsList: [],
203
- },
204
- };
205
- // Mock error when getting attachments
206
- const mockError = new Error('Failed to get attachments');
207
- adapter['uploader'].getAttachmentsFromArtifactId = jest
208
- .fn()
209
- .mockResolvedValue({
210
- error: mockError,
211
- });
212
- // Mock methods
213
- adapter.initializeRepos = jest.fn();
214
- const result = await adapter.streamAttachments({
215
- stream: mockStream,
216
- });
217
- expect(result).toEqual({
218
- error: mockError,
219
- });
220
- });
221
- it('[edge] should handle empty attachments array from artifact', async () => {
222
- const mockStream = jest.fn();
223
- // Set up adapter state with artifact IDs
224
- adapter.state.toDevRev = {
225
- attachmentsMetadata: {
226
- artifactIds: ['artifact1'],
227
- lastProcessed: 0,
228
- lastProcessedAttachmentsIdsList: [],
229
- },
230
- };
231
- // Mock getting empty attachments
232
- adapter['uploader'].getAttachmentsFromArtifactId = jest
233
- .fn()
234
- .mockResolvedValue({
235
- attachments: [],
236
- });
237
- // Mock methods
238
- adapter.initializeRepos = jest.fn();
239
- const result = await adapter.streamAttachments({
240
- stream: mockStream,
241
- });
242
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
243
- expect(result).toBeUndefined();
244
- });
245
- it('should use custom processors when provided', async () => {
246
- const mockStream = jest.fn();
247
- const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
248
- const mockIterator = jest.fn().mockResolvedValue({});
249
- // Set up adapter state with artifact IDs
250
- adapter.state.toDevRev = {
251
- attachmentsMetadata: {
252
- artifactIds: ['artifact1'],
253
- lastProcessed: 0,
254
- lastProcessedAttachmentsIdsList: [],
255
- },
256
- };
257
- // Mock getting attachments
258
- adapter['uploader'].getAttachmentsFromArtifactId = jest
259
- .fn()
260
- .mockResolvedValue({
261
- attachments: [{ id: 'attachment1' }],
262
- });
263
- // Mock methods
264
- adapter.initializeRepos = jest.fn();
265
- const result = await adapter.streamAttachments({
266
- stream: mockStream,
267
- processors: {
268
- reducer: mockReducer,
269
- iterator: mockIterator,
270
- },
271
- });
272
- expect(mockReducer).toHaveBeenCalledWith({
273
- attachments: [{ id: 'attachment1' }],
274
- adapter: adapter,
275
- batchSize: 1,
276
- });
277
- expect(mockIterator).toHaveBeenCalledWith({
278
- reducedAttachments: ['custom-reduced'],
279
- adapter: adapter,
280
- stream: mockStream,
281
- });
282
- expect(result).toBeUndefined();
283
- });
284
- it('should handle rate limiting from iterator', async () => {
285
- const mockStream = jest.fn();
286
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
287
- return {
288
- // Return an object with a `streamAll` method that resolves to your desired value.
289
- streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
290
- };
291
- });
292
- // Set up adapter state with artifact IDs
293
- adapter.state.toDevRev = {
294
- attachmentsMetadata: {
295
- artifactIds: ['artifact1'],
296
- lastProcessed: 0,
297
- lastProcessedAttachmentsIdsList: [],
298
- },
299
- };
300
- // Mock getting attachments
301
- adapter['uploader'].getAttachmentsFromArtifactId = jest
302
- .fn()
303
- .mockResolvedValue({
304
- attachments: [{ id: 'attachment1' }],
305
- });
306
- // Mock methods
307
- adapter.initializeRepos = jest.fn();
308
- const result = await adapter.streamAttachments({
309
- stream: mockStream,
310
- });
311
- expect(result).toEqual({
312
- delay: 30,
313
- });
314
- // The artifactIds array should remain unchanged
315
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
316
- 'artifact1',
317
- ]);
318
- });
319
- it('should handle error from iterator', async () => {
320
- const mockStream = jest.fn();
321
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
322
- return {
323
- // Return an object with a `streamAll` method that resolves to your desired value.
324
- streamAll: jest.fn().mockResolvedValue({
325
- error: 'Mock error',
326
- }),
327
- };
328
- });
329
- // Set up adapter state with artifact IDs
330
- adapter.state.toDevRev = {
331
- attachmentsMetadata: {
332
- artifactIds: ['artifact1'],
333
- lastProcessed: 0,
334
- lastProcessedAttachmentsIdsList: [],
335
- },
336
- };
337
- // Mock getting attachments
338
- adapter['uploader'].getAttachmentsFromArtifactId = jest
339
- .fn()
340
- .mockResolvedValue({
341
- attachments: [{ id: 'attachment1' }],
342
- });
343
- // Mock methods
344
- adapter.initializeRepos = jest.fn();
345
- const result = await adapter.streamAttachments({
346
- stream: mockStream,
347
- });
348
- expect(result).toEqual({
349
- error: 'Mock error',
350
- });
351
- // The artifactIds array should remain unchanged
352
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
353
- 'artifact1',
354
- ]);
355
- });
356
- it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
357
- const mockStream = jest.fn();
358
- adapter.state.toDevRev = {
359
- attachmentsMetadata: {
360
- artifactIds: ['artifact1'],
361
- lastProcessed: 0,
362
- lastProcessedAttachmentsIdsList: [],
363
- },
364
- };
365
- adapter['uploader'].getAttachmentsFromArtifactId = jest
366
- .fn()
367
- .mockResolvedValueOnce({
368
- attachments: [
369
- {
370
- url: 'http://example.com/file1.pdf',
371
- id: 'attachment1',
372
- file_name: 'file1.pdf',
373
- parent_id: 'parent1',
374
- },
375
- {
376
- url: 'http://example.com/file2.pdf',
377
- id: 'attachment2',
378
- file_name: 'file2.pdf',
379
- parent_id: 'parent2',
380
- },
381
- {
382
- url: 'http://example.com/file3.pdf',
383
- id: 'attachment3',
384
- file_name: 'file3.pdf',
385
- parent_id: 'parent3',
386
- },
387
- ],
388
- });
389
- adapter.processAttachment = jest.fn().mockResolvedValue(null);
390
- await adapter.streamAttachments({
391
- stream: mockStream,
392
- });
393
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
394
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
395
- });
396
- });
397
- describe(worker_adapter_1.WorkerAdapter.prototype.emit.name, () => {
398
- let counter;
399
- let mockPostMessage;
400
- beforeEach(() => {
401
- counter = { counter: 0 };
402
- // Import the worker_threads module and spy on parentPort.postMessage
403
- const workerThreads = require('node:worker_threads');
404
- mockPostMessage = jest.fn().mockImplementation(() => {
405
- counter.counter += 1;
406
- });
407
- // Spy on the parentPort.postMessage method
408
- if (workerThreads.parentPort) {
409
- jest
410
- .spyOn(workerThreads.parentPort, 'postMessage')
411
- .mockImplementation(mockPostMessage);
412
- }
413
- else {
414
- // If parentPort is null (not in worker context), create a mock
415
- workerThreads.parentPort = {
416
- postMessage: mockPostMessage,
417
- };
418
- }
419
- });
420
- afterEach(() => {
421
- // Restore all mocks
422
- jest.restoreAllMocks();
423
- });
424
- it('should emit only one event when multiple events of same type are sent', async () => {
425
- adapter['adapterState'].postState = jest
426
- .fn()
427
- .mockResolvedValue(undefined);
428
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
429
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
430
- reports: [],
431
- processed_files: [],
432
- });
433
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
434
- reports: [],
435
- processed_files: [],
436
- });
437
- expect(counter.counter).toBe(1);
438
- });
439
- it('should emit event when different event type is sent after previous events', async () => {
440
- adapter['adapterState'].postState = jest
441
- .fn()
442
- .mockResolvedValue(undefined);
443
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
444
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
445
- reports: [],
446
- processed_files: [],
447
- });
448
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
449
- reports: [],
450
- processed_files: [],
451
- });
452
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
453
- reports: [],
454
- processed_files: [],
455
- });
456
- expect(counter.counter).toBe(1);
457
- });
458
- it('should correctly emit one event even if postState errors', async () => {
459
- adapter['adapterState'].postState = jest
460
- .fn()
461
- .mockRejectedValue(new Error('postState error'));
462
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
463
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
464
- reports: [],
465
- processed_files: [],
466
- });
467
- expect(counter.counter).toBe(1);
468
- });
469
- it('should correctly emit one event even if uploadAllRepos errors', async () => {
470
- adapter['adapterState'].postState = jest
471
- .fn()
472
- .mockResolvedValue(undefined);
473
- adapter.uploadAllRepos = jest
474
- .fn()
475
- .mockRejectedValue(new Error('uploadAllRepos error'));
476
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
477
- reports: [],
478
- processed_files: [],
479
- });
480
- expect(counter.counter).toBe(1);
481
- });
482
- });
483
- });
@@ -1,30 +0,0 @@
1
- import { AdapterState, StateInterface } from './state.interfaces';
2
- export declare function createAdapterState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
3
- export declare class State<ConnectorState> {
4
- private _state;
5
- private initialSdkState;
6
- private workerUrl;
7
- private devrevToken;
8
- private syncUnitId;
9
- private requestId;
10
- constructor({ event, initialState }: StateInterface<ConnectorState>);
11
- get state(): AdapterState<ConnectorState>;
12
- set state(value: AdapterState<ConnectorState>);
13
- /**
14
- * Initializes the state for this adapter instance by fetching from API
15
- * or creating an initial state if none exists (404).
16
- * @param initialState The initial connector state provided by the spawn function
17
- */
18
- init(initialState: ConnectorState): Promise<void>;
19
- /**
20
- * Updates the state of the adapter by posting to API.
21
- * @param {object} state - The state to be updated
22
- */
23
- postState(state?: AdapterState<ConnectorState>): Promise<void>;
24
- /**
25
- * Fetches the state of the adapter from API.
26
- * @return The raw state data from API
27
- */
28
- fetchState(): Promise<string>;
29
- }
30
- //# sourceMappingURL=state.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AASA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA0DjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAuCpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;CAkBpC"}
@@ -1,166 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.State = void 0;
7
- exports.createAdapterState = createAdapterState;
8
- const axios_1 = __importDefault(require("axios"));
9
- const constants_1 = require("../common/constants");
10
- const install_initial_domain_mapping_1 = require("../common/install-initial-domain-mapping");
11
- const axios_client_internal_1 = require("../http/axios-client-internal");
12
- const logger_1 = require("../logger/logger");
13
- const common_1 = require("../types/common");
14
- const extraction_1 = require("../types/extraction");
15
- const state_interfaces_1 = require("./state.interfaces");
16
- async function createAdapterState({ event, initialState, initialDomainMapping, options, }) {
17
- // Deep clone the initial state to avoid mutating the original state
18
- const deepCloneInitialState = structuredClone(initialState);
19
- const as = new State({
20
- event,
21
- initialState: deepCloneInitialState,
22
- initialDomainMapping,
23
- options,
24
- });
25
- if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
26
- await as.init(deepCloneInitialState);
27
- // Check if IDM needs to be updated
28
- const snapInVersionId = event.context.snap_in_version_id;
29
- const hasSnapInVersionInState = 'snapInVersionId' in as.state;
30
- const shouldUpdateIDM = !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId;
31
- if (!shouldUpdateIDM) {
32
- console.log(`Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.`);
33
- }
34
- else {
35
- try {
36
- console.log(`Snap-in version in state "${as.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.`);
37
- if (initialDomainMapping) {
38
- await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(event, initialDomainMapping);
39
- as.state.snapInVersionId = snapInVersionId;
40
- }
41
- else {
42
- throw new Error('No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.');
43
- }
44
- }
45
- catch (error) {
46
- console.error('Error while installing initial domain mapping.', (0, logger_1.serializeError)(error));
47
- process.exit(1);
48
- }
49
- }
50
- // Set lastSyncStarted if the event type is StartExtractingData
51
- if (event.payload.event_type === extraction_1.EventType.StartExtractingData &&
52
- !as.state.lastSyncStarted) {
53
- as.state.lastSyncStarted = new Date().toISOString();
54
- console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`);
55
- }
56
- }
57
- return as;
58
- }
59
- class State {
60
- constructor({ event, initialState }) {
61
- this.initialSdkState =
62
- event.payload.event_context.mode === common_1.SyncMode.LOADING
63
- ? state_interfaces_1.loadingSdkState
64
- : state_interfaces_1.extractionSdkState;
65
- this._state = Object.assign(Object.assign({}, initialState), this.initialSdkState);
66
- this.workerUrl = event.payload.event_context.worker_data_url;
67
- this.devrevToken = event.context.secrets.service_account_token;
68
- this.syncUnitId = event.payload.event_context.sync_unit_id;
69
- this.requestId = event.payload.event_context.request_id_adaas;
70
- }
71
- get state() {
72
- return this._state;
73
- }
74
- set state(value) {
75
- this._state = value;
76
- }
77
- /**
78
- * Initializes the state for this adapter instance by fetching from API
79
- * or creating an initial state if none exists (404).
80
- * @param initialState The initial connector state provided by the spawn function
81
- */
82
- async init(initialState) {
83
- var _a;
84
- try {
85
- const stringifiedState = await this.fetchState();
86
- if (!stringifiedState) {
87
- throw new Error('No state found in response.');
88
- }
89
- let parsedState;
90
- try {
91
- parsedState = JSON.parse(stringifiedState);
92
- }
93
- catch (error) {
94
- throw new Error(`Failed to parse state. ${error}`);
95
- }
96
- this.state = parsedState;
97
- console.log('State fetched successfully. Current state', (0, logger_1.getPrintableState)(this.state));
98
- }
99
- catch (error) {
100
- if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
101
- console.log('State not found. Initializing state with initial state.');
102
- const initialAdapterState = Object.assign(Object.assign({}, initialState), this.initialSdkState);
103
- this.state = initialAdapterState;
104
- await this.postState(initialAdapterState);
105
- }
106
- else {
107
- console.error('Failed to init state.', (0, logger_1.serializeError)(error));
108
- process.exit(1);
109
- }
110
- }
111
- }
112
- /**
113
- * Updates the state of the adapter by posting to API.
114
- * @param {object} state - The state to be updated
115
- */
116
- async postState(state) {
117
- const url = this.workerUrl + '.update';
118
- this.state = state || this.state;
119
- let stringifiedState;
120
- try {
121
- stringifiedState = JSON.stringify(this.state);
122
- }
123
- catch (error) {
124
- console.error('Failed to stringify state.', (0, logger_1.serializeError)(error));
125
- process.exit(1);
126
- }
127
- try {
128
- await axios_client_internal_1.axiosClient.post(url, {
129
- state: stringifiedState,
130
- }, {
131
- headers: {
132
- Authorization: this.devrevToken,
133
- },
134
- params: {
135
- sync_unit: this.syncUnitId,
136
- request_id: this.requestId,
137
- },
138
- });
139
- console.log('State updated successfully to', (0, logger_1.getPrintableState)(this.state));
140
- }
141
- catch (error) {
142
- console.error('Failed to update the state.', (0, logger_1.serializeError)(error));
143
- process.exit(1);
144
- }
145
- }
146
- /**
147
- * Fetches the state of the adapter from API.
148
- * @return The raw state data from API
149
- */
150
- async fetchState() {
151
- var _a;
152
- console.log(`Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.`);
153
- const url = this.workerUrl + '.get';
154
- const response = await axios_client_internal_1.axiosClient.get(url, {
155
- headers: {
156
- Authorization: this.devrevToken,
157
- },
158
- params: {
159
- sync_unit: this.syncUnitId,
160
- request_id: this.requestId,
161
- },
162
- });
163
- return (_a = response.data) === null || _a === void 0 ? void 0 : _a.state;
164
- }
165
- }
166
- exports.State = State;