@devrev/airsync-sdk 2.0.0-beta.0 → 2.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +1802 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -0
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.js +44 -4
  6. package/dist/attachments-streaming/attachments-streaming-pool.test.js +259 -10
  7. package/dist/common/constants.d.ts +16 -5
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +20 -6
  10. package/dist/common/control-protocol.d.ts +2 -2
  11. package/dist/common/control-protocol.d.ts.map +1 -1
  12. package/dist/common/control-protocol.js +1 -5
  13. package/dist/common/helpers.d.ts +8 -0
  14. package/dist/common/helpers.d.ts.map +1 -1
  15. package/dist/common/helpers.js +15 -0
  16. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  17. package/dist/common/install-initial-domain-mapping.test.js +7 -3
  18. package/dist/common/test-utils.d.ts +19 -0
  19. package/dist/common/test-utils.d.ts.map +1 -0
  20. package/dist/common/test-utils.js +101 -0
  21. package/dist/common/time-value-resolver.d.ts +47 -0
  22. package/dist/common/time-value-resolver.d.ts.map +1 -0
  23. package/dist/common/time-value-resolver.js +141 -0
  24. package/dist/common/time-value-resolver.test.d.ts +2 -0
  25. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  26. package/dist/common/time-value-resolver.test.js +268 -0
  27. package/dist/http/axios-client-internal.d.ts.map +1 -1
  28. package/dist/http/axios-client-internal.js +7 -0
  29. package/dist/http/axios-client-internal.test.js +27 -6
  30. package/dist/http/axios-client.d.ts.map +1 -1
  31. package/dist/http/axios-client.js +6 -3
  32. package/dist/http/index.d.ts +0 -1
  33. package/dist/http/index.d.ts.map +1 -1
  34. package/dist/http/index.js +0 -1
  35. package/dist/index.d.ts +9 -8
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/index.js +12 -10
  38. package/dist/logger/logger.d.ts +1 -17
  39. package/dist/logger/logger.d.ts.map +1 -1
  40. package/dist/logger/logger.interfaces.d.ts +2 -2
  41. package/dist/logger/logger.js +35 -27
  42. package/dist/logger/logger.test.js +66 -19
  43. package/dist/mappers/mappers.interface.d.ts +5 -5
  44. package/dist/mappers/mappers.interface.js +3 -3
  45. package/dist/mappers/mappers.test.js +8 -10
  46. package/dist/mock-server/mock-server.d.ts +47 -0
  47. package/dist/mock-server/mock-server.d.ts.map +1 -0
  48. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  49. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  50. package/dist/mock-server/mock-server.interfaces.js +4 -0
  51. package/dist/mock-server/mock-server.js +278 -0
  52. package/dist/multithreading/create-worker.test.js +36 -17
  53. package/dist/multithreading/process-task.d.ts.map +1 -1
  54. package/dist/multithreading/process-task.js +46 -44
  55. package/dist/multithreading/process-task.test.d.ts +2 -0
  56. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  57. package/dist/multithreading/process-task.test.js +146 -0
  58. package/dist/multithreading/spawn/spawn.d.ts +4 -3
  59. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  60. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  61. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  62. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  63. package/dist/multithreading/spawn/spawn.js +41 -31
  64. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  65. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  66. package/dist/multithreading/spawn/spawn.test.js +232 -0
  67. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
  68. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
  69. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  70. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
  71. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
  72. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
  73. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
  74. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
  75. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
  76. package/dist/multithreading/worker-adapter/worker-adapter.js +233 -134
  77. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
  78. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
  79. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
  80. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
  81. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
  82. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
  83. package/dist/repo/repo.d.ts.map +1 -1
  84. package/dist/repo/repo.interfaces.d.ts +4 -2
  85. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  86. package/dist/repo/repo.js +4 -3
  87. package/dist/repo/repo.test.js +60 -7
  88. package/dist/state/state.d.ts +7 -1
  89. package/dist/state/state.d.ts.map +1 -1
  90. package/dist/state/state.extract-window.test.d.ts +2 -0
  91. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  92. package/dist/state/state.extract-window.test.js +163 -0
  93. package/dist/state/state.interfaces.d.ts +25 -8
  94. package/dist/state/state.interfaces.d.ts.map +1 -1
  95. package/dist/state/state.interfaces.js +4 -2
  96. package/dist/state/state.js +111 -12
  97. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  98. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  99. package/dist/state/state.pending-boundaries.test.js +189 -0
  100. package/dist/state/state.post-state.test.d.ts +2 -0
  101. package/dist/state/state.post-state.test.d.ts.map +1 -0
  102. package/dist/state/state.post-state.test.js +77 -0
  103. package/dist/state/state.test.js +117 -73
  104. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  105. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  106. package/dist/state/state.time-value-resolution.test.js +175 -0
  107. package/dist/tsdoc-metadata.json +11 -0
  108. package/dist/types/common.d.ts +0 -25
  109. package/dist/types/common.d.ts.map +1 -1
  110. package/dist/types/common.js +1 -11
  111. package/dist/types/external-domain-metadata.d.ts +338 -0
  112. package/dist/types/external-domain-metadata.d.ts.map +1 -0
  113. package/dist/types/extraction.d.ts +89 -218
  114. package/dist/types/extraction.d.ts.map +1 -1
  115. package/dist/types/extraction.js +56 -124
  116. package/dist/types/extraction.test.js +69 -43
  117. package/dist/types/index.d.ts +4 -2
  118. package/dist/types/index.d.ts.map +1 -1
  119. package/dist/types/index.js +6 -3
  120. package/dist/types/loading.d.ts +4 -24
  121. package/dist/types/loading.d.ts.map +1 -1
  122. package/dist/types/loading.js +1 -22
  123. package/dist/types/workers.d.ts +31 -14
  124. package/dist/types/workers.d.ts.map +1 -1
  125. package/dist/types/workers.js +1 -0
  126. package/dist/uploader/uploader.d.ts +1 -0
  127. package/dist/uploader/uploader.d.ts.map +1 -1
  128. package/dist/uploader/uploader.helpers.test.js +0 -11
  129. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  130. package/dist/uploader/uploader.js +53 -20
  131. package/dist/uploader/uploader.test.js +22 -10
  132. package/package.json +12 -11
  133. package/dist/common/event-type-translation.d.ts +0 -24
  134. package/dist/common/event-type-translation.d.ts.map +0 -1
  135. package/dist/common/event-type-translation.js +0 -117
  136. package/dist/deprecated/adapter/index.d.ts +0 -62
  137. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  138. package/dist/deprecated/adapter/index.js +0 -151
  139. package/dist/deprecated/common/helpers.d.ts +0 -7
  140. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  141. package/dist/deprecated/common/helpers.js +0 -47
  142. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  143. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  144. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  145. package/dist/deprecated/demo-extractor/index.js +0 -161
  146. package/dist/deprecated/http/client.d.ts +0 -22
  147. package/dist/deprecated/http/client.d.ts.map +0 -1
  148. package/dist/deprecated/http/client.js +0 -161
  149. package/dist/deprecated/uploader/index.d.ts +0 -35
  150. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  151. package/dist/deprecated/uploader/index.js +0 -161
  152. package/dist/http/types.d.ts +0 -17
  153. package/dist/http/types.d.ts.map +0 -1
  154. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  155. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  156. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
  157. /package/dist/{http/types.js → types/external-domain-metadata.js} +0 -0
@@ -0,0 +1,232 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const events_1 = require("events");
4
+ const constants_1 = require("../../common/constants");
5
+ const extraction_1 = require("../../types/extraction");
6
+ const workers_1 = require("../../types/workers");
7
+ const test_utils_1 = require("../../common/test-utils");
8
+ // ---------------------------------------------------------------------------
9
+ // Mocks
10
+ // ---------------------------------------------------------------------------
11
+ jest.mock('../create-worker', () => ({
12
+ createWorker: jest.fn(),
13
+ }));
14
+ jest.mock('../../common/control-protocol', () => ({
15
+ emit: jest.fn().mockResolvedValue({}),
16
+ }));
17
+ jest.mock('../../logger/logger', () => ({
18
+ Logger: jest.fn().mockImplementation(() => ({
19
+ log: jest.fn(),
20
+ warn: jest.fn(),
21
+ error: jest.fn(),
22
+ info: jest.fn(),
23
+ logFn: jest.fn(),
24
+ })),
25
+ serializeError: jest.fn((e) => String(e)),
26
+ }));
27
+ jest.mock('../../common/helpers', () => ({
28
+ getLibraryVersion: jest.fn().mockReturnValue('1.0.0-test'),
29
+ getMemoryUsage: jest.fn().mockReturnValue({
30
+ formattedMessage: 'Memory: RSS 100/512MB (19.53%) [...]',
31
+ rssUsedMB: '100.00',
32
+ rssUsedPercent: '19.53%',
33
+ heapUsedPercent: '30.00%',
34
+ externalMB: '10.00',
35
+ arrayBuffersMB: '5.00',
36
+ }),
37
+ sleep: jest.fn(),
38
+ truncateFilename: jest.fn((f) => f),
39
+ truncateMessage: jest.fn((m) => m),
40
+ }));
41
+ // ---------------------------------------------------------------------------
42
+ // Imports after mocks
43
+ // ---------------------------------------------------------------------------
44
+ const spawn_1 = require("./spawn");
45
+ const create_worker_1 = require("../create-worker");
46
+ const control_protocol_1 = require("../../common/control-protocol");
47
+ const helpers_1 = require("../../common/helpers");
48
+ // ---------------------------------------------------------------------------
49
+ // Factory for a fake worker (EventEmitter with postMessage + terminate)
50
+ // ---------------------------------------------------------------------------
51
+ function makeWorker() {
52
+ const w = new events_1.EventEmitter();
53
+ w.postMessage = jest.fn();
54
+ w.terminate = jest.fn().mockResolvedValue(0);
55
+ return w;
56
+ }
57
+ // ---------------------------------------------------------------------------
58
+ // Helper: instantiate Spawn directly, injecting a mock logger via console swap
59
+ // ---------------------------------------------------------------------------
60
+ function buildSpawn(overrides) {
61
+ var _a;
62
+ const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
63
+ payload: { event_type: extraction_1.EventType.StartExtractingData },
64
+ });
65
+ const mockLogger = {
66
+ log: jest.fn(),
67
+ warn: jest.fn(),
68
+ error: jest.fn(),
69
+ info: jest.fn(),
70
+ logFn: jest.fn(),
71
+ };
72
+ const originalConsole = console;
73
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
74
+ global.console = mockLogger;
75
+ const s = new spawn_1.Spawn({
76
+ event,
77
+ worker: overrides.worker,
78
+ options: overrides.options,
79
+ resolve: (_a = overrides.resolve) !== null && _a !== void 0 ? _a : jest.fn(),
80
+ originalConsole: originalConsole,
81
+ });
82
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
83
+ global.console = originalConsole;
84
+ return s;
85
+ }
86
+ // ---------------------------------------------------------------------------
87
+ // spawn() factory tests
88
+ // ---------------------------------------------------------------------------
89
+ describe('spawn() factory', () => {
90
+ beforeEach(() => {
91
+ jest.clearAllMocks();
92
+ jest.useFakeTimers({ legacyFakeTimers: true });
93
+ });
94
+ afterEach(() => {
95
+ jest.useRealTimers();
96
+ });
97
+ it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => {
98
+ // Arrange
99
+ const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
100
+ payload: { event_type: extraction_1.EventType.UnknownEventType },
101
+ });
102
+ // Act
103
+ await (0, spawn_1.spawn)({ event, initialState: {} });
104
+ // Assert: no worker process should be started, but the platform should
105
+ // still receive a terminal event (so the run doesn't hang).
106
+ expect(create_worker_1.createWorker).not.toHaveBeenCalled();
107
+ expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({ event, eventType: expect.any(String) }));
108
+ });
109
+ it('should reject the returned promise when createWorker throws', async () => {
110
+ // Arrange
111
+ create_worker_1.createWorker.mockRejectedValue(new Error('worker boom'));
112
+ const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
113
+ payload: { event_type: extraction_1.EventType.StartExtractingData },
114
+ });
115
+ // Act & Assert
116
+ await expect((0, spawn_1.spawn)({
117
+ event,
118
+ initialState: {},
119
+ baseWorkerPath: '/fake',
120
+ options: {
121
+ workerPathOverrides: {
122
+ [extraction_1.EventType.StartExtractingData]: '/path.js',
123
+ },
124
+ },
125
+ })).rejects.toThrow('worker boom');
126
+ });
127
+ });
128
+ // ---------------------------------------------------------------------------
129
+ // Spawn class — lifecycle tests
130
+ // ---------------------------------------------------------------------------
131
+ describe('Spawn class', () => {
132
+ let worker;
133
+ let resolveMock;
134
+ beforeEach(() => {
135
+ jest.clearAllMocks();
136
+ jest.useFakeTimers({ legacyFakeTimers: true });
137
+ worker = makeWorker();
138
+ resolveMock = jest.fn();
139
+ });
140
+ afterEach(() => {
141
+ jest.useRealTimers();
142
+ });
143
+ // -------------------------------------------------------------------------
144
+ // WorkerMessageFailed captured and propagated in error emit
145
+ // -------------------------------------------------------------------------
146
+ it('should include the WorkerMessageFailed reason in the error event emitted to the platform', async () => {
147
+ // Arrange
148
+ buildSpawn({ worker, resolve: resolveMock });
149
+ // Act
150
+ worker.emit(workers_1.WorkerEvent.WorkerMessage, {
151
+ subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
152
+ payload: { message: 'connector exploded' },
153
+ });
154
+ worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
155
+ await Promise.resolve();
156
+ await Promise.resolve();
157
+ // Assert: the platform receives an error event whose message contains
158
+ // the reason sent by the worker — this is what operators see in the run log.
159
+ expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({
160
+ data: expect.objectContaining({
161
+ error: expect.objectContaining({
162
+ message: expect.stringContaining('connector exploded'),
163
+ }),
164
+ }),
165
+ }));
166
+ expect(resolveMock).toHaveBeenCalled();
167
+ });
168
+ it('should emit a data extraction error when the worker exits without ever emitting', async () => {
169
+ // Arrange
170
+ buildSpawn({ worker, resolve: resolveMock });
171
+ // Act
172
+ worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
173
+ await Promise.resolve();
174
+ await Promise.resolve();
175
+ // Assert: a silent worker exit must surface a DataExtractionError to the
176
+ // platform with a non-empty message. The event type is the upstream
177
+ // contract; the exact message wording is implementation detail.
178
+ expect(control_protocol_1.emit).toHaveBeenCalledTimes(1);
179
+ const [emitted] = control_protocol_1.emit.mock.calls[0];
180
+ expect(emitted.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
181
+ expect(emitted.data.error.message).toEqual(expect.any(String));
182
+ expect(emitted.data.error.message.length).toBeGreaterThan(0);
183
+ expect(resolveMock).toHaveBeenCalled();
184
+ });
185
+ // Soft-timeout and hard-timeout timer behavior is covered end-to-end by
186
+ // src/tests/timeout-handling/ (real workers, real timers). Unit tests with
187
+ // fake timers only re-asserted the mocked setTimeout call and gave no signal.
188
+ // -------------------------------------------------------------------------
189
+ // Memory monitoring — error clears the interval
190
+ // -------------------------------------------------------------------------
191
+ it('should clear the memory monitoring interval when getMemoryUsage throws to prevent repeated crashes', async () => {
192
+ // Arrange
193
+ helpers_1.getMemoryUsage.mockImplementation(() => {
194
+ throw new Error('OOM');
195
+ });
196
+ const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
197
+ buildSpawn({ worker, resolve: resolveMock });
198
+ // Act
199
+ jest.advanceTimersByTime(30001); // MEMORY_LOG_INTERVAL = 30 s
200
+ await Promise.resolve();
201
+ // Assert
202
+ expect(clearIntervalSpy).toHaveBeenCalled();
203
+ });
204
+ // -------------------------------------------------------------------------
205
+ // Soft-timeout race: worker emits AFTER softTimeoutSent — no double-emit
206
+ // -------------------------------------------------------------------------
207
+ it('should NOT emit an error when the worker successfully emits just after receiving the soft-timeout signal', async () => {
208
+ // Arrange
209
+ buildSpawn({ worker, resolve: resolveMock });
210
+ // Act: trigger soft timeout — sends WorkerMessageExit to the worker
211
+ jest.advanceTimersByTime(constants_1.DEFAULT_LAMBDA_TIMEOUT + 1);
212
+ await Promise.resolve();
213
+ // Confirm the soft-timeout path actually fired — guards against a silent
214
+ // pass if the timeout constant changes and the timer never runs.
215
+ expect(worker.postMessage).toHaveBeenCalledWith(expect.objectContaining({
216
+ subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
217
+ }));
218
+ // Worker responds: emits its event successfully, then exits normally
219
+ worker.emit(workers_1.WorkerEvent.WorkerMessage, {
220
+ subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
221
+ });
222
+ worker.emit(workers_1.WorkerEvent.WorkerExit, 0);
223
+ // The exit handler defers via setImmediate when softTimeoutSent=true
224
+ jest.runAllImmediates();
225
+ await Promise.resolve();
226
+ await Promise.resolve();
227
+ // Assert: no error should reach the platform — the worker completed its job
228
+ const errorEmits = control_protocol_1.emit.mock.calls.filter((call) => { var _a, _b; return (_b = (_a = call[0]) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.error; });
229
+ expect(errorEmits).toHaveLength(0);
230
+ expect(resolveMock).toHaveBeenCalled();
231
+ });
232
+ });
@@ -1,7 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
+ const constants_1 = require("../../common/constants");
3
4
  const state_1 = require("../../state/state");
5
+ const jest_setup_1 = require("../../tests/jest.setup");
4
6
  const test_helpers_1 = require("../../tests/test-helpers");
7
+ const test_utils_1 = require("../../common/test-utils");
5
8
  const types_1 = require("../../types");
6
9
  const worker_adapter_1 = require("./worker-adapter");
7
10
  // 1. Create a mock function for the method you want to override.
@@ -49,7 +52,9 @@ describe('Artifact ordering when artifacts overflow batch sizes in repositories'
49
52
  let testAdapter;
50
53
  beforeEach(() => {
51
54
  // Create a fresh adapter instance for this test to avoid mocking conflicts
52
- const mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
55
+ const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
56
+ payload: { event_type: types_1.EventType.StartExtractingData },
57
+ });
53
58
  const mockAdapterState = new state_1.State({
54
59
  event: mockEvent,
55
60
  initialState: { attachments: { completed: false } },
@@ -125,3 +130,46 @@ describe('Artifact ordering when artifacts overflow batch sizes in repositories'
125
130
  expect(checkArtifactOrder(artifacts, repos)).toBe(true);
126
131
  });
127
132
  });
133
+ describe('External sync units splitting into artifacts', () => {
134
+ let testAdapter;
135
+ beforeEach(() => {
136
+ const mockEvent = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
137
+ payload: { event_type: types_1.EventType.StartExtractingExternalSyncUnits },
138
+ });
139
+ const mockAdapterState = new state_1.State({
140
+ event: mockEvent,
141
+ initialState: {},
142
+ });
143
+ testAdapter = new worker_adapter_1.WorkerAdapter({
144
+ event: mockEvent,
145
+ adapterState: mockAdapterState,
146
+ });
147
+ });
148
+ it('should split 125k external sync units into 5 artifacts', async () => {
149
+ const BATCH_SIZE = 25000;
150
+ const TOTAL_UNITS = 125000;
151
+ const externalSyncUnits = Array.from({ length: TOTAL_UNITS }, (_, i) => ({
152
+ id: String(i),
153
+ name: `Unit ${i}`,
154
+ description: `Description ${i}`,
155
+ }));
156
+ testAdapter.initializeRepos([
157
+ {
158
+ itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
159
+ overridenOptions: {
160
+ batchSize: BATCH_SIZE,
161
+ skipConfirmation: true,
162
+ },
163
+ },
164
+ ]);
165
+ const repo = testAdapter.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS);
166
+ const chunkSize = 10000;
167
+ for (let i = 0; i < TOTAL_UNITS; i += chunkSize) {
168
+ await (repo === null || repo === void 0 ? void 0 : repo.push(externalSyncUnits.slice(i, i + chunkSize)));
169
+ }
170
+ await testAdapter.uploadAllRepos();
171
+ expect(testAdapter.artifacts.length).toBe(TOTAL_UNITS / BATCH_SIZE // 5
172
+ );
173
+ expect(testAdapter.artifacts.every((a) => a.item_type === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)).toBe(true);
174
+ });
175
+ });
@@ -2,13 +2,13 @@ import { Mappers } from '../../mappers/mappers';
2
2
  import { Repo } from '../../repo/repo';
3
3
  import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
4
4
  import { AdapterState } from '../../state/state.interfaces';
5
- import { AirdropEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
5
+ import { AirSyncEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
6
6
  import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
7
7
  import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
8
8
  import { Artifact } from '../../uploader/uploader.interfaces';
9
9
  export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
10
10
  /**
11
- * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
11
+ * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
12
12
  * as parameter in processTask and onTimeout functions. The class provides
13
13
  * utilities to emit control events to the platform, update the state of the connector,
14
14
  * and upload artifacts to the platform.
@@ -23,13 +23,14 @@ export declare function createWorkerAdapter<ConnectorState>({ event, adapterStat
23
23
  * @public
24
24
  */
25
25
  export declare class WorkerAdapter<ConnectorState> {
26
- readonly event: AirdropEvent;
26
+ readonly event: AirSyncEvent;
27
27
  readonly options?: WorkerAdapterOptions;
28
28
  isTimeout: boolean;
29
+ hasWorkerEmitted: boolean;
29
30
  private adapterState;
30
31
  private _artifacts;
31
- private hasWorkerEmitted;
32
32
  private repos;
33
+ private currentEventDataLength;
33
34
  private loaderReports;
34
35
  private _processedFiles;
35
36
  private _mappers;
@@ -40,6 +41,12 @@ export declare class WorkerAdapter<ConnectorState> {
40
41
  get reports(): LoaderReport[];
41
42
  get processedFiles(): string[];
42
43
  get mappers(): Mappers;
44
+ get extractionScope(): import("../../types/workers").ExtractionScope;
45
+ /**
46
+ * Returns whether the given item type should be extracted.
47
+ * Defaults to true if the scope is empty or the item type is not listed.
48
+ */
49
+ shouldExtract(itemType: string): boolean;
43
50
  initializeRepos(repos: RepoInterface[]): void;
44
51
  getRepo(itemType: string): Repo | undefined;
45
52
  postState(): Promise<void>;
@@ -53,7 +60,6 @@ export declare class WorkerAdapter<ConnectorState> {
53
60
  */
54
61
  emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
55
62
  uploadAllRepos(): Promise<void>;
56
- handleTimeout(): void;
57
63
  loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
58
64
  getLoaderBatches({ supportedItemTypes, }: {
59
65
  supportedItemTypes: string[];
@@ -1 +1 @@
1
- {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAqFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAqHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6E5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoGvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAyHzC"}
1
+ {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmHV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA+GvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker-adapter.emit.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.emit.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.emit.test.ts"],"names":[],"mappings":""}