@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +5 -11
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +1 -8
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +4 -3
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +11 -39
  7. package/dist/common/constants.d.ts +1 -0
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +2 -1
  10. package/dist/common/helpers.d.ts +0 -14
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +0 -36
  13. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  14. package/dist/http/client.d.ts.map +1 -0
  15. package/dist/index.d.ts +25 -13
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +51 -30
  18. package/dist/logger/logger.d.ts +14 -0
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.js +55 -8
  21. package/dist/mappers/mappers.d.ts +9 -10
  22. package/dist/mappers/mappers.d.ts.map +1 -1
  23. package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
  24. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  25. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
  26. package/dist/mappers/mappers.js +13 -9
  27. package/dist/multithreading/adapters/base-adapter.d.ts +71 -0
  28. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  29. package/dist/multithreading/adapters/base-adapter.js +138 -0
  30. package/dist/multithreading/adapters/extraction-adapter.d.ts +57 -0
  31. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  32. package/dist/multithreading/adapters/extraction-adapter.js +323 -0
  33. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  34. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  35. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  36. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  37. package/dist/multithreading/adapters/loading-adapter.js +431 -0
  38. package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
  39. package/dist/multithreading/emit.d.ts.map +1 -0
  40. package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
  41. package/dist/multithreading/process-task.d.ts +16 -1
  42. package/dist/multithreading/process-task.d.ts.map +1 -1
  43. package/dist/multithreading/process-task.js +115 -50
  44. package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
  45. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  46. package/dist/multithreading/spawn/spawn.helpers.js +152 -2
  47. package/dist/multithreading/spawn/spawn.js +7 -7
  48. package/dist/multithreading/worker.js +1 -2
  49. package/dist/repo/repo.js +2 -2
  50. package/dist/state/base-state.d.ts +75 -0
  51. package/dist/state/base-state.d.ts.map +1 -0
  52. package/dist/state/base-state.js +261 -0
  53. package/dist/state/extraction-state.d.ts +29 -0
  54. package/dist/state/extraction-state.d.ts.map +1 -0
  55. package/dist/state/extraction-state.js +122 -0
  56. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  57. package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
  58. package/dist/state/loading-state.d.ts +18 -0
  59. package/dist/state/loading-state.d.ts.map +1 -0
  60. package/dist/state/loading-state.js +39 -0
  61. package/dist/state/state.d.ts +3 -35
  62. package/dist/state/state.d.ts.map +1 -1
  63. package/dist/state/state.interfaces.d.ts +20 -3
  64. package/dist/state/state.interfaces.d.ts.map +1 -1
  65. package/dist/state/state.interfaces.js +19 -1
  66. package/dist/state/state.js +9 -263
  67. package/dist/{common → state}/time-value-resolver.d.ts +4 -5
  68. package/dist/state/time-value-resolver.d.ts.map +1 -0
  69. package/dist/{common → state}/time-value-resolver.js +9 -8
  70. package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
  71. package/dist/testing/mock-event.d.ts.map +1 -0
  72. package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
  73. package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
  74. package/dist/testing/mock-server.d.ts.map +1 -0
  75. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  76. package/dist/{mock-server → testing}/mock-server.js +6 -1
  77. package/dist/types/errors.d.ts +9 -0
  78. package/dist/types/errors.d.ts.map +1 -0
  79. package/dist/types/extraction.d.ts +73 -18
  80. package/dist/types/extraction.d.ts.map +1 -1
  81. package/dist/types/extraction.js +4 -6
  82. package/dist/types/loading.d.ts +1 -2
  83. package/dist/types/loading.d.ts.map +1 -1
  84. package/dist/types/loading.js +0 -1
  85. package/dist/types/workers.d.ts +64 -20
  86. package/dist/types/workers.d.ts.map +1 -1
  87. package/dist/uploader/uploader.d.ts +5 -4
  88. package/dist/uploader/uploader.d.ts.map +1 -1
  89. package/dist/uploader/uploader.interfaces.d.ts +1 -1
  90. package/dist/uploader/uploader.js +10 -10
  91. package/package.json +3 -3
  92. package/dist/airsync-sdk.d.ts +0 -1802
  93. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  94. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  95. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -516
  96. package/dist/common/control-protocol.d.ts.map +0 -1
  97. package/dist/common/errors.d.ts +0 -6
  98. package/dist/common/errors.d.ts.map +0 -1
  99. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  100. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  101. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  102. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  103. package/dist/common/test-utils.d.ts.map +0 -1
  104. package/dist/common/time-value-resolver.d.ts.map +0 -1
  105. package/dist/common/time-value-resolver.test.d.ts +0 -2
  106. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  107. package/dist/common/time-value-resolver.test.js +0 -268
  108. package/dist/http/axios-client-internal.d.ts.map +0 -1
  109. package/dist/http/axios-client-internal.test.d.ts +0 -2
  110. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  111. package/dist/http/axios-client-internal.test.js +0 -175
  112. package/dist/http/axios-client.d.ts +0 -27
  113. package/dist/http/axios-client.d.ts.map +0 -1
  114. package/dist/http/axios-client.js +0 -60
  115. package/dist/http/constants.d.ts +0 -4
  116. package/dist/http/constants.d.ts.map +0 -1
  117. package/dist/http/constants.js +0 -6
  118. package/dist/http/index.d.ts +0 -2
  119. package/dist/http/index.d.ts.map +0 -1
  120. package/dist/http/index.js +0 -17
  121. package/dist/logger/logger.constants.d.ts +0 -6
  122. package/dist/logger/logger.constants.d.ts.map +0 -1
  123. package/dist/logger/logger.constants.js +0 -13
  124. package/dist/logger/logger.context.d.ts +0 -58
  125. package/dist/logger/logger.context.d.ts.map +0 -1
  126. package/dist/logger/logger.context.js +0 -86
  127. package/dist/logger/logger.test.d.ts +0 -2
  128. package/dist/logger/logger.test.d.ts.map +0 -1
  129. package/dist/logger/logger.test.js +0 -537
  130. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  131. package/dist/mappers/mappers.test.d.ts +0 -2
  132. package/dist/mappers/mappers.test.d.ts.map +0 -1
  133. package/dist/mappers/mappers.test.js +0 -105
  134. package/dist/mock-server/mock-server.d.ts.map +0 -1
  135. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  136. package/dist/multithreading/create-worker.test.d.ts +0 -2
  137. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  138. package/dist/multithreading/create-worker.test.js +0 -108
  139. package/dist/multithreading/process-task.test.d.ts +0 -2
  140. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  141. package/dist/multithreading/process-task.test.js +0 -146
  142. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  143. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  144. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  145. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  146. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  147. package/dist/multithreading/spawn/spawn.test.js +0 -232
  148. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  149. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  150. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +0 -175
  151. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -97
  152. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  153. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +0 -2
  154. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +0 -1
  155. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +0 -390
  156. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +0 -2
  157. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +0 -1
  158. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +0 -799
  159. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  160. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  161. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  162. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -514
  163. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -846
  164. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +0 -2
  165. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +0 -1
  166. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +0 -596
  167. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +0 -2
  168. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +0 -1
  169. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +0 -71
  170. package/dist/repo/repo.test.d.ts +0 -2
  171. package/dist/repo/repo.test.d.ts.map +0 -1
  172. package/dist/repo/repo.test.js +0 -184
  173. package/dist/state/state.extract-window.test.d.ts +0 -2
  174. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  175. package/dist/state/state.extract-window.test.js +0 -163
  176. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  177. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  178. package/dist/state/state.pending-boundaries.test.js +0 -189
  179. package/dist/state/state.post-state.test.d.ts +0 -2
  180. package/dist/state/state.post-state.test.d.ts.map +0 -1
  181. package/dist/state/state.post-state.test.js +0 -77
  182. package/dist/state/state.test.d.ts +0 -2
  183. package/dist/state/state.test.d.ts.map +0 -1
  184. package/dist/state/state.test.js +0 -268
  185. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  186. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  187. package/dist/state/state.time-value-resolution.test.js +0 -175
  188. package/dist/tsdoc-metadata.json +0 -11
  189. package/dist/types/extraction.test.d.ts +0 -2
  190. package/dist/types/extraction.test.d.ts.map +0 -1
  191. package/dist/types/extraction.test.js +0 -96
  192. package/dist/types/index.d.ts +0 -11
  193. package/dist/types/index.d.ts.map +0 -1
  194. package/dist/types/index.js +0 -21
  195. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  196. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  197. package/dist/uploader/uploader.helpers.test.js +0 -256
  198. package/dist/uploader/uploader.test.d.ts +0 -2
  199. package/dist/uploader/uploader.test.d.ts.map +0 -1
  200. package/dist/uploader/uploader.test.js +0 -601
  201. /package/dist/http/{axios-client-internal.js → client.js} +0 -0
  202. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
  203. /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
  204. /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
  205. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  206. /package/dist/{common → types}/errors.js +0 -0
@@ -1,232 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const events_1 = require("events");
4
- const constants_1 = require("../../common/constants");
5
- const extraction_1 = require("../../types/extraction");
6
- const workers_1 = require("../../types/workers");
7
- const test_utils_1 = require("../../common/test-utils");
8
- // ---------------------------------------------------------------------------
9
- // Mocks
10
- // ---------------------------------------------------------------------------
11
- jest.mock('../create-worker', () => ({
12
- createWorker: jest.fn(),
13
- }));
14
- jest.mock('../../common/control-protocol', () => ({
15
- emit: jest.fn().mockResolvedValue({}),
16
- }));
17
- jest.mock('../../logger/logger', () => ({
18
- Logger: jest.fn().mockImplementation(() => ({
19
- log: jest.fn(),
20
- warn: jest.fn(),
21
- error: jest.fn(),
22
- info: jest.fn(),
23
- logFn: jest.fn(),
24
- })),
25
- serializeError: jest.fn((e) => String(e)),
26
- }));
27
- jest.mock('../../common/helpers', () => ({
28
- getLibraryVersion: jest.fn().mockReturnValue('1.0.0-test'),
29
- getMemoryUsage: jest.fn().mockReturnValue({
30
- formattedMessage: 'Memory: RSS 100/512MB (19.53%) [...]',
31
- rssUsedMB: '100.00',
32
- rssUsedPercent: '19.53%',
33
- heapUsedPercent: '30.00%',
34
- externalMB: '10.00',
35
- arrayBuffersMB: '5.00',
36
- }),
37
- sleep: jest.fn(),
38
- truncateFilename: jest.fn((f) => f),
39
- truncateMessage: jest.fn((m) => m),
40
- }));
41
- // ---------------------------------------------------------------------------
42
- // Imports after mocks
43
- // ---------------------------------------------------------------------------
44
- const spawn_1 = require("./spawn");
45
- const create_worker_1 = require("../create-worker");
46
- const control_protocol_1 = require("../../common/control-protocol");
47
- const helpers_1 = require("../../common/helpers");
48
- // ---------------------------------------------------------------------------
49
- // Factory for a fake worker (EventEmitter with postMessage + terminate)
50
- // ---------------------------------------------------------------------------
51
- function makeWorker() {
52
- const w = new events_1.EventEmitter();
53
- w.postMessage = jest.fn();
54
- w.terminate = jest.fn().mockResolvedValue(0);
55
- return w;
56
- }
57
- // ---------------------------------------------------------------------------
58
- // Helper: instantiate Spawn directly, injecting a mock logger via console swap
59
- // ---------------------------------------------------------------------------
60
- function buildSpawn(overrides) {
61
- var _a;
62
- const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
63
- payload: { event_type: extraction_1.EventType.StartExtractingData },
64
- });
65
- const mockLogger = {
66
- log: jest.fn(),
67
- warn: jest.fn(),
68
- error: jest.fn(),
69
- info: jest.fn(),
70
- logFn: jest.fn(),
71
- };
72
- const originalConsole = console;
73
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
74
- global.console = mockLogger;
75
- const s = new spawn_1.Spawn({
76
- event,
77
- worker: overrides.worker,
78
- options: overrides.options,
79
- resolve: (_a = overrides.resolve) !== null && _a !== void 0 ? _a : jest.fn(),
80
- originalConsole: originalConsole,
81
- });
82
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
83
- global.console = originalConsole;
84
- return s;
85
- }
86
- // ---------------------------------------------------------------------------
87
- // spawn() factory tests
88
- // ---------------------------------------------------------------------------
89
- describe('spawn() factory', () => {
90
- beforeEach(() => {
91
- jest.clearAllMocks();
92
- jest.useFakeTimers({ legacyFakeTimers: true });
93
- });
94
- afterEach(() => {
95
- jest.useRealTimers();
96
- });
97
- it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => {
98
- // Arrange
99
- const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
100
- payload: { event_type: extraction_1.EventType.UnknownEventType },
101
- });
102
- // Act
103
- await (0, spawn_1.spawn)({ event, initialState: {} });
104
- // Assert: no worker process should be started, but the platform should
105
- // still receive a terminal event (so the run doesn't hang).
106
- expect(create_worker_1.createWorker).not.toHaveBeenCalled();
107
- expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({ event, eventType: expect.any(String) }));
108
- });
109
- it('should reject the returned promise when createWorker throws', async () => {
110
- // Arrange
111
- create_worker_1.createWorker.mockRejectedValue(new Error('worker boom'));
112
- const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
113
- payload: { event_type: extraction_1.EventType.StartExtractingData },
114
- });
115
- // Act & Assert
116
- await expect((0, spawn_1.spawn)({
117
- event,
118
- initialState: {},
119
- baseWorkerPath: '/fake',
120
- options: {
121
- workerPathOverrides: {
122
- [extraction_1.EventType.StartExtractingData]: '/path.js',
123
- },
124
- },
125
- })).rejects.toThrow('worker boom');
126
- });
127
- });
128
- // ---------------------------------------------------------------------------
129
- // Spawn class — lifecycle tests
130
- // ---------------------------------------------------------------------------
131
- describe('Spawn class', () => {
132
- let worker;
133
- let resolveMock;
134
- beforeEach(() => {
135
- jest.clearAllMocks();
136
- jest.useFakeTimers({ legacyFakeTimers: true });
137
- worker = makeWorker();
138
- resolveMock = jest.fn();
139
- });
140
- afterEach(() => {
141
- jest.useRealTimers();
142
- });
143
- // -------------------------------------------------------------------------
144
- // WorkerMessageFailed captured and propagated in error emit
145
- // -------------------------------------------------------------------------
146
- it('should include the WorkerMessageFailed reason in the error event emitted to the platform', async () => {
147
- // Arrange
148
- buildSpawn({ worker, resolve: resolveMock });
149
- // Act
150
- worker.emit(workers_1.WorkerEvent.WorkerMessage, {
151
- subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
152
- payload: { message: 'connector exploded' },
153
- });
154
- worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
155
- await Promise.resolve();
156
- await Promise.resolve();
157
- // Assert: the platform receives an error event whose message contains
158
- // the reason sent by the worker — this is what operators see in the run log.
159
- expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({
160
- data: expect.objectContaining({
161
- error: expect.objectContaining({
162
- message: expect.stringContaining('connector exploded'),
163
- }),
164
- }),
165
- }));
166
- expect(resolveMock).toHaveBeenCalled();
167
- });
168
- it('should emit a data extraction error when the worker exits without ever emitting', async () => {
169
- // Arrange
170
- buildSpawn({ worker, resolve: resolveMock });
171
- // Act
172
- worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
173
- await Promise.resolve();
174
- await Promise.resolve();
175
- // Assert: a silent worker exit must surface a DataExtractionError to the
176
- // platform with a non-empty message. The event type is the upstream
177
- // contract; the exact message wording is implementation detail.
178
- expect(control_protocol_1.emit).toHaveBeenCalledTimes(1);
179
- const [emitted] = control_protocol_1.emit.mock.calls[0];
180
- expect(emitted.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
181
- expect(emitted.data.error.message).toEqual(expect.any(String));
182
- expect(emitted.data.error.message.length).toBeGreaterThan(0);
183
- expect(resolveMock).toHaveBeenCalled();
184
- });
185
- // Soft-timeout and hard-timeout timer behavior is covered end-to-end by
186
- // src/tests/timeout-handling/ (real workers, real timers). Unit tests with
187
- // fake timers only re-asserted the mocked setTimeout call and gave no signal.
188
- // -------------------------------------------------------------------------
189
- // Memory monitoring — error clears the interval
190
- // -------------------------------------------------------------------------
191
- it('should clear the memory monitoring interval when getMemoryUsage throws to prevent repeated crashes', async () => {
192
- // Arrange
193
- helpers_1.getMemoryUsage.mockImplementation(() => {
194
- throw new Error('OOM');
195
- });
196
- const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
197
- buildSpawn({ worker, resolve: resolveMock });
198
- // Act
199
- jest.advanceTimersByTime(30001); // MEMORY_LOG_INTERVAL = 30 s
200
- await Promise.resolve();
201
- // Assert
202
- expect(clearIntervalSpy).toHaveBeenCalled();
203
- });
204
- // -------------------------------------------------------------------------
205
- // Soft-timeout race: worker emits AFTER softTimeoutSent — no double-emit
206
- // -------------------------------------------------------------------------
207
- it('should NOT emit an error when the worker successfully emits just after receiving the soft-timeout signal', async () => {
208
- // Arrange
209
- buildSpawn({ worker, resolve: resolveMock });
210
- // Act: trigger soft timeout — sends WorkerMessageExit to the worker
211
- jest.advanceTimersByTime(constants_1.DEFAULT_LAMBDA_TIMEOUT + 1);
212
- await Promise.resolve();
213
- // Confirm the soft-timeout path actually fired — guards against a silent
214
- // pass if the timeout constant changes and the timer never runs.
215
- expect(worker.postMessage).toHaveBeenCalledWith(expect.objectContaining({
216
- subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
217
- }));
218
- // Worker responds: emits its event successfully, then exits normally
219
- worker.emit(workers_1.WorkerEvent.WorkerMessage, {
220
- subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
221
- });
222
- worker.emit(workers_1.WorkerEvent.WorkerExit, 0);
223
- // The exit handler defers via setImmediate when softTimeoutSent=true
224
- jest.runAllImmediates();
225
- await Promise.resolve();
226
- await Promise.resolve();
227
- // Assert: no error should reach the platform — the worker completed its job
228
- const errorEmits = control_protocol_1.emit.mock.calls.filter((call) => { var _a, _b; return (_b = (_a = call[0]) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.error; });
229
- expect(errorEmits).toHaveLength(0);
230
- expect(resolveMock).toHaveBeenCalled();
231
- });
232
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.artifacts.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts"],"names":[],"mappings":""}
@@ -1,175 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const constants_1 = require("../../common/constants");
4
- const state_1 = require("../../state/state");
5
- const jest_setup_1 = require("../../tests/jest.setup");
6
- const test_helpers_1 = require("../../tests/test-helpers");
7
- const test_utils_1 = require("../../common/test-utils");
8
- const types_1 = require("../../types");
9
- const worker_adapter_1 = require("./worker-adapter");
10
- // 1. Create a mock function for the method you want to override.
11
- const mockUpload = (itemType, objects) => {
12
- return {
13
- error: null,
14
- artifact: {
15
- id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`,
16
- item_type: itemType,
17
- item_count: objects.length,
18
- },
19
- };
20
- };
21
- // 2. Mock the entire 'uploader' module.
22
- // The factory function () => { ... } returns the mock implementation.
23
- jest.mock('../../uploader/uploader', () => {
24
- return {
25
- // The mocked Uploader class
26
- Uploader: jest.fn().mockImplementation(() => {
27
- // The constructor of the mocked Uploader returns an object
28
- // with the methods you want to control.
29
- return {
30
- upload: mockUpload,
31
- };
32
- }),
33
- };
34
- });
35
- function checkArtifactOrder(artifacts, expectedOrder) {
36
- let outerIndex = 0;
37
- for (const artifact of artifacts) {
38
- try {
39
- // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown
40
- while (artifact.item_type != expectedOrder[outerIndex].itemType) {
41
- outerIndex++;
42
- }
43
- }
44
- catch (e) {
45
- console.error('Error finding artifact type in repos:', e);
46
- return false;
47
- }
48
- }
49
- return true;
50
- }
51
- describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => {
52
- let testAdapter;
53
- beforeEach(() => {
54
- // Create a fresh adapter instance for this test to avoid mocking conflicts
55
- const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
56
- payload: { event_type: types_1.EventType.StartExtractingData },
57
- });
58
- const mockAdapterState = new state_1.State({
59
- event: mockEvent,
60
- initialState: { attachments: { completed: false } },
61
- });
62
- testAdapter = new worker_adapter_1.WorkerAdapter({
63
- event: mockEvent,
64
- adapterState: mockAdapterState,
65
- options: {
66
- batchSize: 50,
67
- },
68
- });
69
- });
70
- it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => {
71
- var _a, _b;
72
- const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
73
- // Initialize repos
74
- testAdapter.initializeRepos(repos);
75
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
76
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(105)));
77
- await testAdapter.uploadAllRepos();
78
- const artifacts = testAdapter.artifacts;
79
- expect(artifacts.length).toBe(4);
80
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
81
- });
82
- it('should work with more than 2 repos', async () => {
83
- var _a, _b, _c, _d;
84
- const repos = [
85
- { itemType: 'ItemTypeA' },
86
- { itemType: 'ItemTypeB' },
87
- { itemType: 'ItemTypeC' },
88
- { itemType: 'ItemTypeD' },
89
- ];
90
- // Initialize repos
91
- testAdapter.initializeRepos(repos);
92
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
93
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
94
- await ((_c = testAdapter.getRepo('ItemTypeC')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(103)));
95
- await ((_d = testAdapter.getRepo('ItemTypeD')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(104)));
96
- await testAdapter.uploadAllRepos();
97
- const artifacts = testAdapter.artifacts;
98
- expect(artifacts.length).toBe(12);
99
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
100
- });
101
- it('should maintain order with multiple pushes and uploads', async () => {
102
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
103
- const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
104
- // Initialize repos
105
- testAdapter.initializeRepos(repos);
106
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
107
- await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
108
- await ((_c = testAdapter.getRepo('ItemTypeA')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(101)));
109
- await ((_d = testAdapter.getRepo('ItemTypeB')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(102)));
110
- await ((_e = testAdapter.getRepo('ItemTypeA')) === null || _e === void 0 ? void 0 : _e.upload());
111
- await ((_f = testAdapter.getRepo('ItemTypeB')) === null || _f === void 0 ? void 0 : _f.upload());
112
- await ((_g = testAdapter.getRepo('ItemTypeA')) === null || _g === void 0 ? void 0 : _g.push((0, test_helpers_1.createItems)(101)));
113
- await ((_h = testAdapter.getRepo('ItemTypeB')) === null || _h === void 0 ? void 0 : _h.push((0, test_helpers_1.createItems)(102)));
114
- await ((_j = testAdapter.getRepo('ItemTypeA')) === null || _j === void 0 ? void 0 : _j.push((0, test_helpers_1.createItems)(101)));
115
- await ((_k = testAdapter.getRepo('ItemTypeB')) === null || _k === void 0 ? void 0 : _k.push((0, test_helpers_1.createItems)(102)));
116
- await testAdapter.uploadAllRepos();
117
- const artifacts = testAdapter.artifacts;
118
- expect(artifacts.length).toBe(20);
119
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
120
- });
121
- it('should not count artifacts if 0 items are pushed to the repo', async () => {
122
- var _a;
123
- const repos = [{ itemType: 'ItemTypeA' }];
124
- // Initialize repos
125
- testAdapter.initializeRepos(repos);
126
- await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push([]));
127
- await testAdapter.uploadAllRepos();
128
- const artifacts = testAdapter.artifacts;
129
- expect(artifacts.length).toBe(0);
130
- expect(checkArtifactOrder(artifacts, repos)).toBe(true);
131
- });
132
- });
133
- describe('External sync units splitting into artifacts', () => {
134
- let testAdapter;
135
- beforeEach(() => {
136
- const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
137
- payload: { event_type: types_1.EventType.StartExtractingExternalSyncUnits },
138
- });
139
- const mockAdapterState = new state_1.State({
140
- event: mockEvent,
141
- initialState: {},
142
- });
143
- testAdapter = new worker_adapter_1.WorkerAdapter({
144
- event: mockEvent,
145
- adapterState: mockAdapterState,
146
- });
147
- });
148
- it('should split 125k external sync units into 5 artifacts', async () => {
149
- const BATCH_SIZE = 25000;
150
- const TOTAL_UNITS = 125000;
151
- const externalSyncUnits = Array.from({ length: TOTAL_UNITS }, (_, i) => ({
152
- id: String(i),
153
- name: `Unit ${i}`,
154
- description: `Description ${i}`,
155
- }));
156
- testAdapter.initializeRepos([
157
- {
158
- itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
159
- overridenOptions: {
160
- batchSize: BATCH_SIZE,
161
- skipConfirmation: true,
162
- },
163
- },
164
- ]);
165
- const repo = testAdapter.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS);
166
- const chunkSize = 10000;
167
- for (let i = 0; i < TOTAL_UNITS; i += chunkSize) {
168
- await (repo === null || repo === void 0 ? void 0 : repo.push(externalSyncUnits.slice(i, i + chunkSize)));
169
- }
170
- await testAdapter.uploadAllRepos();
171
- expect(testAdapter.artifacts.length).toBe(TOTAL_UNITS / BATCH_SIZE // 5
172
- );
173
- expect(testAdapter.artifacts.every((a) => a.item_type === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)).toBe(true);
174
- });
175
- });
@@ -1,97 +0,0 @@
1
- import { Mappers } from '../../mappers/mappers';
2
- import { Repo } from '../../repo/repo';
3
- import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
4
- import { AdapterState } from '../../state/state.interfaces';
5
- import { AirSyncEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
6
- import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
7
- import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
8
- import { Artifact } from '../../uploader/uploader.interfaces';
9
- export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
10
- /**
11
- * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
12
- * as parameter in processTask and onTimeout functions. The class provides
13
- * utilities to emit control events to the platform, update the state of the connector,
14
- * and upload artifacts to the platform.
15
- * @class WorkerAdapter
16
- * @constructor
17
- * @param options - The options to create a new instance of WorkerAdapter class
18
- * @param event - The event object received from the platform
19
- * @param initialState - The initial state of the adapter
20
- * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development
21
- * @param workerPath - The path to the worker file
22
- *
23
- * @public
24
- */
25
- export declare class WorkerAdapter<ConnectorState> {
26
- readonly event: AirSyncEvent;
27
- readonly options?: WorkerAdapterOptions;
28
- isTimeout: boolean;
29
- hasWorkerEmitted: boolean;
30
- private adapterState;
31
- private _artifacts;
32
- private repos;
33
- private currentEventDataLength;
34
- private loaderReports;
35
- private _processedFiles;
36
- private _mappers;
37
- private uploader;
38
- constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
39
- get state(): AdapterState<ConnectorState>;
40
- set state(value: AdapterState<ConnectorState>);
41
- get reports(): LoaderReport[];
42
- get processedFiles(): string[];
43
- get mappers(): Mappers;
44
- get extractionScope(): import("../../types/workers").ExtractionScope;
45
- /**
46
- * Returns whether the given item type should be extracted.
47
- * Defaults to true if the scope is empty or the item type is not listed.
48
- */
49
- shouldExtract(itemType: string): boolean;
50
- initializeRepos(repos: RepoInterface[]): void;
51
- getRepo(itemType: string): Repo | undefined;
52
- postState(): Promise<void>;
53
- get artifacts(): Artifact[];
54
- set artifacts(artifacts: Artifact[]);
55
- /**
56
- * Emits an event to the platform.
57
- *
58
- * @param newEventType - The event type to be emitted
59
- * @param data - The data to be sent with the event
60
- */
61
- emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
62
- uploadAllRepos(): Promise<void>;
63
- loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
64
- getLoaderBatches({ supportedItemTypes, }: {
65
- supportedItemTypes: string[];
66
- }): Promise<FileToLoad[]>;
67
- loadAttachments({ create, }: {
68
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
69
- }): Promise<LoadItemTypesResponse>;
70
- loadItem({ item, itemTypeToLoad, }: {
71
- item: ExternalSystemItem;
72
- itemTypeToLoad: ItemTypeToLoad;
73
- }): Promise<LoadItemResponse>;
74
- processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
75
- /**
76
- * Destroys a stream to prevent memory leaks.
77
- * @param httpStream - The axios response stream to destroy
78
- */
79
- private destroyHttpStream;
80
- loadAttachment({ item, create, }: {
81
- item: ExternalSystemAttachment;
82
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
83
- }): Promise<LoadItemResponse>;
84
- /**
85
- * Streams the attachments to the DevRev platform.
86
- * The attachments are streamed to the platform and the artifact information is returned.
87
- * @param params - The parameters to stream the attachments
88
- * @returns The response object containing the ssorAttachment artifact information
89
- * or error information if there was an error
90
- */
91
- streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
92
- stream: ExternalSystemAttachmentStreamingFunction;
93
- processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
94
- batchSize?: number;
95
- }): Promise<StreamAttachmentsReturnType>;
96
- }
97
- //# sourceMappingURL=worker-adapter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmHV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA+GvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.emit.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.emit.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.emit.test.ts"],"names":[],"mappings":""}