@devrev/ts-adaas 1.13.3 → 1.13.4-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +1 -1
  2. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  3. package/dist/common/helpers.d.ts +28 -21
  4. package/dist/common/helpers.d.ts.map +1 -1
  5. package/dist/common/helpers.js +18 -156
  6. package/dist/index.d.ts +3 -3
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +3 -3
  9. package/dist/multithreading/create-worker.d.ts.map +1 -0
  10. package/dist/multithreading/create-worker.test.d.ts.map +1 -0
  11. package/dist/multithreading/process-task.d.ts.map +1 -0
  12. package/dist/{workers → multithreading}/process-task.js +1 -1
  13. package/dist/{workers → multithreading/spawn}/spawn.d.ts +1 -1
  14. package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
  15. package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
  16. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
  17. package/dist/multithreading/spawn/spawn.helpers.js +114 -0
  18. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
  19. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
  20. package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
  21. package/dist/{workers → multithreading/spawn}/spawn.js +11 -10
  22. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
  23. package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.js +4 -4
  24. package/dist/{workers → multithreading/worker-adapter}/worker-adapter.d.ts +8 -8
  25. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
  26. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
  27. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
  28. package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
  29. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
  30. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
  31. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
  32. package/dist/{workers → multithreading/worker-adapter}/worker-adapter.js +16 -16
  33. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
  34. package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.js +10 -10
  35. package/dist/{workers → multithreading}/worker.d.ts.map +1 -1
  36. package/dist/state/state.d.ts.map +1 -1
  37. package/dist/state/state.js +1 -2
  38. package/dist/types/extraction.d.ts +1 -1
  39. package/dist/types/extraction.d.ts.map +1 -1
  40. package/dist/types/workers.d.ts +1 -1
  41. package/dist/types/workers.d.ts.map +1 -1
  42. package/dist/uploader/uploader.d.ts +1 -26
  43. package/dist/uploader/uploader.d.ts.map +1 -1
  44. package/dist/uploader/uploader.helpers.d.ts +32 -0
  45. package/dist/uploader/uploader.helpers.d.ts.map +1 -0
  46. package/dist/uploader/uploader.helpers.js +140 -0
  47. package/dist/uploader/uploader.helpers.test.d.ts +2 -0
  48. package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
  49. package/dist/uploader/uploader.helpers.test.js +265 -0
  50. package/dist/uploader/uploader.js +7 -117
  51. package/dist/uploader/uploader.test.js +561 -72
  52. package/package.json +1 -1
  53. package/dist/common/helpers.test.d.ts +0 -2
  54. package/dist/common/helpers.test.d.ts.map +0 -1
  55. package/dist/common/helpers.test.js +0 -158
  56. package/dist/workers/create-worker.d.ts.map +0 -1
  57. package/dist/workers/create-worker.test.d.ts.map +0 -1
  58. package/dist/workers/process-task.d.ts.map +0 -1
  59. package/dist/workers/spawn.d.ts.map +0 -1
  60. package/dist/workers/worker-adapter.artifacts.test.d.ts.map +0 -1
  61. package/dist/workers/worker-adapter.d.ts.map +0 -1
  62. package/dist/workers/worker-adapter.test.d.ts.map +0 -1
  63. /package/dist/{workers → multithreading}/create-worker.d.ts +0 -0
  64. /package/dist/{workers → multithreading}/create-worker.js +0 -0
  65. /package/dist/{workers → multithreading}/create-worker.test.d.ts +0 -0
  66. /package/dist/{workers → multithreading}/create-worker.test.js +0 -0
  67. /package/dist/{workers → multithreading}/process-task.d.ts +0 -0
  68. /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.d.ts +0 -0
  69. /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.d.ts +0 -0
  70. /package/dist/{workers → multithreading}/worker.d.ts +0 -0
  71. /package/dist/{workers → multithreading}/worker.js +0 -0
@@ -0,0 +1,293 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const extraction_1 = require("../../types/extraction");
4
+ const loading_1 = require("../../types/loading");
5
+ const spawn_helpers_1 = require("./spawn.helpers");
6
+ describe(spawn_helpers_1.getTimeoutErrorEventType.name, () => {
7
+ describe('metadata extraction', () => {
8
+ it('should return MetadataExtractionError for StartExtractingMetadata', () => {
9
+ // Arrange
10
+ const eventType = extraction_1.EventType.StartExtractingMetadata;
11
+ // Act
12
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
13
+ // Assert
14
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
15
+ });
16
+ it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => {
17
+ // Arrange
18
+ const eventType = extraction_1.EventType.ExtractionMetadataStart;
19
+ // Act
20
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
21
+ // Assert
22
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
23
+ });
24
+ });
25
+ describe('data extraction', () => {
26
+ it('should return DataExtractionError for StartExtractingData', () => {
27
+ // Arrange
28
+ const eventType = extraction_1.EventType.StartExtractingData;
29
+ // Act
30
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
31
+ // Assert
32
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
33
+ });
34
+ it('should return DataExtractionError for ContinueExtractingData', () => {
35
+ // Arrange
36
+ const eventType = extraction_1.EventType.ContinueExtractingData;
37
+ // Act
38
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
39
+ // Assert
40
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
41
+ });
42
+ it('should return DataExtractionError for deprecated ExtractionDataStart', () => {
43
+ // Arrange
44
+ const eventType = extraction_1.EventType.ExtractionDataStart;
45
+ // Act
46
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
47
+ // Assert
48
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
49
+ });
50
+ it('should return DataExtractionError for deprecated ExtractionDataContinue', () => {
51
+ // Arrange
52
+ const eventType = extraction_1.EventType.ExtractionDataContinue;
53
+ // Act
54
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
55
+ // Assert
56
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
57
+ });
58
+ });
59
+ describe('extractor state deletion', () => {
60
+ it('should return ExtractorStateDeletionError for StartDeletingExtractorState', () => {
61
+ // Arrange
62
+ const eventType = extraction_1.EventType.StartDeletingExtractorState;
63
+ // Act
64
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
65
+ // Assert
66
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorStateDeletionError);
67
+ });
68
+ it('should return ExtractorStateDeletionError for deprecated ExtractionDataDelete', () => {
69
+ // Arrange
70
+ const eventType = extraction_1.EventType.ExtractionDataDelete;
71
+ // Act
72
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
73
+ // Assert
74
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorStateDeletionError);
75
+ });
76
+ });
77
+ describe('attachments extraction', () => {
78
+ it('should return AttachmentExtractionError for StartExtractingAttachments', () => {
79
+ // Arrange
80
+ const eventType = extraction_1.EventType.StartExtractingAttachments;
81
+ // Act
82
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
83
+ // Assert
84
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.AttachmentExtractionError);
85
+ });
86
+ it('should return AttachmentExtractionError for ContinueExtractingAttachments', () => {
87
+ // Arrange
88
+ const eventType = extraction_1.EventType.ContinueExtractingAttachments;
89
+ // Act
90
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
91
+ // Assert
92
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.AttachmentExtractionError);
93
+ });
94
+ it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsStart', () => {
95
+ // Arrange
96
+ const eventType = extraction_1.EventType.ExtractionAttachmentsStart;
97
+ // Act
98
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
99
+ // Assert
100
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.AttachmentExtractionError);
101
+ });
102
+ it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsContinue', () => {
103
+ // Arrange
104
+ const eventType = extraction_1.EventType.ExtractionAttachmentsContinue;
105
+ // Act
106
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
107
+ // Assert
108
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.AttachmentExtractionError);
109
+ });
110
+ });
111
+ describe('extractor attachments state deletion', () => {
112
+ it('should return ExtractorAttachmentsStateDeletionError for StartDeletingExtractorAttachmentsState', () => {
113
+ // Arrange
114
+ const eventType = extraction_1.EventType.StartDeletingExtractorAttachmentsState;
115
+ // Act
116
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
117
+ // Assert
118
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError);
119
+ });
120
+ it('should return ExtractorAttachmentsStateDeletionError for deprecated ExtractionAttachmentsDelete', () => {
121
+ // Arrange
122
+ const eventType = extraction_1.EventType.ExtractionAttachmentsDelete;
123
+ // Act
124
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
125
+ // Assert
126
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError);
127
+ });
128
+ });
129
+ describe('external sync units extraction', () => {
130
+ it('should return ExternalSyncUnitExtractionError for StartExtractingExternalSyncUnits', () => {
131
+ // Arrange
132
+ const eventType = extraction_1.EventType.StartExtractingExternalSyncUnits;
133
+ // Act
134
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
135
+ // Assert
136
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError);
137
+ });
138
+ it('should return ExternalSyncUnitExtractionError for deprecated ExtractionExternalSyncUnitsStart', () => {
139
+ // Arrange
140
+ const eventType = extraction_1.EventType.ExtractionExternalSyncUnitsStart;
141
+ // Act
142
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
143
+ // Assert
144
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError);
145
+ });
146
+ });
147
+ describe('data loading', () => {
148
+ it('should return DataLoadingError for StartLoadingData', () => {
149
+ // Arrange
150
+ const eventType = extraction_1.EventType.StartLoadingData;
151
+ // Act
152
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
153
+ // Assert
154
+ expect(result.eventType).toBe(loading_1.LoaderEventType.DataLoadingError);
155
+ });
156
+ it('should return DataLoadingError for ContinueLoadingData', () => {
157
+ // Arrange
158
+ const eventType = extraction_1.EventType.ContinueLoadingData;
159
+ // Act
160
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
161
+ // Assert
162
+ expect(result.eventType).toBe(loading_1.LoaderEventType.DataLoadingError);
163
+ });
164
+ });
165
+ describe('loader state deletion', () => {
166
+ it('should return LoaderStateDeletionError for StartDeletingLoaderState', () => {
167
+ // Arrange
168
+ const eventType = extraction_1.EventType.StartDeletingLoaderState;
169
+ // Act
170
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
171
+ // Assert
172
+ expect(result.eventType).toBe(loading_1.LoaderEventType.LoaderStateDeletionError);
173
+ });
174
+ });
175
+ describe('attachment loading', () => {
176
+ it('should return AttachmentLoadingError for StartLoadingAttachments', () => {
177
+ // Arrange
178
+ const eventType = extraction_1.EventType.StartLoadingAttachments;
179
+ // Act
180
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
181
+ // Assert
182
+ expect(result.eventType).toBe(loading_1.LoaderEventType.AttachmentLoadingError);
183
+ });
184
+ it('should return AttachmentLoadingError for ContinueLoadingAttachments', () => {
185
+ // Arrange
186
+ const eventType = extraction_1.EventType.ContinueLoadingAttachments;
187
+ // Act
188
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
189
+ // Assert
190
+ expect(result.eventType).toBe(loading_1.LoaderEventType.AttachmentLoadingError);
191
+ });
192
+ });
193
+ describe('loader attachment state deletion', () => {
194
+ it('should return LoaderAttachmentStateDeletionError for StartDeletingLoaderAttachmentState', () => {
195
+ // Arrange
196
+ const eventType = extraction_1.EventType.StartDeletingLoaderAttachmentState;
197
+ // Act
198
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
199
+ // Assert
200
+ expect(result.eventType).toBe(loading_1.LoaderEventType.LoaderAttachmentStateDeletionError);
201
+ });
202
+ });
203
+ describe('unknown event types', () => {
204
+ it('[edge] should return UnknownEventType and log error for unrecognized event type', () => {
205
+ // Arrange
206
+ const eventType = extraction_1.EventType.UnknownEventType;
207
+ const consoleErrorSpy = jest
208
+ .spyOn(console, 'error')
209
+ .mockImplementation(() => { });
210
+ // Act
211
+ const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
212
+ // Assert
213
+ expect(result.eventType).toBe(loading_1.LoaderEventType.UnknownEventType);
214
+ expect(consoleErrorSpy).toHaveBeenCalledWith('Event type not recognized in getTimeoutErrorEventType function: ' +
215
+ eventType);
216
+ // Cleanup
217
+ consoleErrorSpy.mockRestore();
218
+ });
219
+ });
220
+ });
221
+ describe(spawn_helpers_1.getNoScriptEventType.name, () => {
222
+ describe('extractor state deletion', () => {
223
+ it('should return ExtractorStateDeletionDone for StartDeletingExtractorState', () => {
224
+ // Arrange
225
+ const eventType = extraction_1.EventType.StartDeletingExtractorState;
226
+ // Act
227
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
228
+ // Assert
229
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorStateDeletionDone);
230
+ });
231
+ });
232
+ describe('extractor attachments state deletion', () => {
233
+ it('should return ExtractorAttachmentsStateDeletionDone for StartDeletingExtractorAttachmentsState', () => {
234
+ // Arrange
235
+ const eventType = extraction_1.EventType.StartDeletingExtractorAttachmentsState;
236
+ // Act
237
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
238
+ // Assert
239
+ expect(result.eventType).toBe(extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionDone);
240
+ });
241
+ });
242
+ describe('loader state deletion', () => {
243
+ it('should return LoaderStateDeletionDone for StartDeletingLoaderState', () => {
244
+ // Arrange
245
+ const eventType = extraction_1.EventType.StartDeletingLoaderState;
246
+ // Act
247
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
248
+ // Assert
249
+ expect(result.eventType).toBe(loading_1.LoaderEventType.LoaderStateDeletionDone);
250
+ });
251
+ });
252
+ describe('loader attachment state deletion', () => {
253
+ it('should return LoaderAttachmentStateDeletionDone for StartDeletingLoaderAttachmentState', () => {
254
+ // Arrange
255
+ const eventType = extraction_1.EventType.StartDeletingLoaderAttachmentState;
256
+ // Act
257
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
258
+ // Assert
259
+ expect(result.eventType).toBe(loading_1.LoaderEventType.LoaderAttachmentStateDeletionDone);
260
+ });
261
+ });
262
+ describe('unknown event types', () => {
263
+ it('[edge] should return UnknownEventType and log error for unrecognized event type', () => {
264
+ // Arrange
265
+ const eventType = extraction_1.EventType.StartExtractingData;
266
+ const consoleErrorSpy = jest
267
+ .spyOn(console, 'error')
268
+ .mockImplementation(() => { });
269
+ // Act
270
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
271
+ // Assert
272
+ expect(result.eventType).toBe(loading_1.LoaderEventType.UnknownEventType);
273
+ expect(consoleErrorSpy).toHaveBeenCalledWith('Event type not recognized in getNoScriptEventType function: ' +
274
+ eventType);
275
+ // Cleanup
276
+ consoleErrorSpy.mockRestore();
277
+ });
278
+ it('[edge] should return UnknownEventType for StartLoadingData', () => {
279
+ // Arrange
280
+ const eventType = extraction_1.EventType.StartLoadingData;
281
+ const consoleErrorSpy = jest
282
+ .spyOn(console, 'error')
283
+ .mockImplementation(() => { });
284
+ // Act
285
+ const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
286
+ // Assert
287
+ expect(result.eventType).toBe(loading_1.LoaderEventType.UnknownEventType);
288
+ expect(consoleErrorSpy).toHaveBeenCalled();
289
+ // Cleanup
290
+ consoleErrorSpy.mockRestore();
291
+ });
292
+ });
293
+ });
@@ -7,14 +7,15 @@ exports.Spawn = void 0;
7
7
  exports.spawn = spawn;
8
8
  const yargs_1 = __importDefault(require("yargs"));
9
9
  const helpers_1 = require("yargs/helpers");
10
- const control_protocol_1 = require("../common/control-protocol");
11
- const event_type_translation_1 = require("../common/event-type-translation");
12
- const helpers_2 = require("../common/helpers");
13
- const logger_1 = require("../logger/logger");
14
- const extraction_1 = require("../types/extraction");
15
- const workers_1 = require("../types/workers");
16
- const constants_1 = require("../common/constants");
17
- const create_worker_1 = require("./create-worker");
10
+ const control_protocol_1 = require("../../common/control-protocol");
11
+ const event_type_translation_1 = require("../../common/event-type-translation");
12
+ const helpers_2 = require("../../common/helpers");
13
+ const logger_1 = require("../../logger/logger");
14
+ const extraction_1 = require("../../types/extraction");
15
+ const workers_1 = require("../../types/workers");
16
+ const constants_1 = require("../../common/constants");
17
+ const create_worker_1 = require("../create-worker");
18
+ const spawn_helpers_1 = require("./spawn.helpers");
18
19
  function getWorkerPath({ event, workerBasePath, }) {
19
20
  let path = null;
20
21
  switch (event.payload.event_type) {
@@ -120,7 +121,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
120
121
  }
121
122
  }
122
123
  else {
123
- const { eventType } = (0, helpers_2.getNoScriptEventType)(event.payload.event_type);
124
+ const { eventType } = (0, spawn_helpers_1.getNoScriptEventType)(event.payload.event_type);
124
125
  await (0, control_protocol_1.emit)({
125
126
  event,
126
127
  eventType,
@@ -226,7 +227,7 @@ class Spawn {
226
227
  return;
227
228
  }
228
229
  this.alreadyEmitted = true;
229
- const { eventType } = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
230
+ const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
230
231
  try {
231
232
  await (0, control_protocol_1.emit)({
232
233
  eventType,
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts"],"names":[],"mappings":""}
@@ -1,8 +1,8 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- const state_1 = require("../state/state");
4
- const test_helpers_1 = require("../tests/test-helpers");
5
- const types_1 = require("../types");
3
+ const state_1 = require("../../state/state");
4
+ const test_helpers_1 = require("../../tests/test-helpers");
5
+ const types_1 = require("../../types");
6
6
  const worker_adapter_1 = require("./worker-adapter");
7
7
  // 1. Create a mock function for the method you want to override.
8
8
  const mockUpload = (itemType, objects) => {
@@ -17,7 +17,7 @@ const mockUpload = (itemType, objects) => {
17
17
  };
18
18
  // 2. Mock the entire 'uploader' module.
19
19
  // The factory function () => { ... } returns the mock implementation.
20
- jest.mock('../uploader/uploader', () => {
20
+ jest.mock('../../uploader/uploader', () => {
21
21
  return {
22
22
  // The mocked Uploader class
23
23
  Uploader: jest.fn().mockImplementation(() => {
@@ -1,11 +1,11 @@
1
- import { Mappers } from '../mappers/mappers';
2
- import { Repo } from '../repo/repo';
3
- import { NormalizedAttachment, RepoInterface } from '../repo/repo.interfaces';
4
- import { AdapterState } from '../state/state.interfaces';
5
- import { AirdropEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../types/extraction';
6
- import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../types/loading';
7
- import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
8
- import { Artifact } from '../uploader/uploader.interfaces';
1
+ import { Mappers } from '../../mappers/mappers';
2
+ import { Repo } from '../../repo/repo';
3
+ import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
4
+ import { AdapterState } from '../../state/state.interfaces';
5
+ import { AirdropEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
6
+ import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
7
+ import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
8
+ import { Artifact } from '../../uploader/uploader.interfaces';
9
9
  export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
10
10
  /**
11
11
  * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAgHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IAuBK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwE5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6LvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4FvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAmD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAsHzC"}
@@ -0,0 +1,22 @@
1
+ import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
2
+ /**
3
+ * Gets the files to load for the loader.
4
+ * @param {string[]} supportedItemTypes - The supported item types
5
+ * @param {StatsFileObject[]} statsFile - The stats file
6
+ * @returns {FileToLoad[]} The files to load
7
+ */
8
+ export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
9
+ supportedItemTypes: string[];
10
+ statsFile: StatsFileObject[];
11
+ }): FileToLoad[];
12
+ /**
13
+ * Adds a report to the loader report.
14
+ * @param {LoaderReport[]} loaderReports - The loader reports
15
+ * @param {LoaderReport} report - The report to add
16
+ * @returns {LoaderReport[]} The updated loader reports
17
+ */
18
+ export declare function addReportToLoaderReport({ loaderReports, report, }: {
19
+ loaderReports: LoaderReport[];
20
+ report: LoaderReport;
21
+ }): LoaderReport[];
22
+ //# sourceMappingURL=worker-adapter.helpers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.getFilesToLoad = getFilesToLoad;
4
+ exports.addReportToLoaderReport = addReportToLoaderReport;
5
+ const loading_1 = require("../../types/loading");
6
+ /**
7
+ * Gets the files to load for the loader.
8
+ * @param {string[]} supportedItemTypes - The supported item types
9
+ * @param {StatsFileObject[]} statsFile - The stats file
10
+ * @returns {FileToLoad[]} The files to load
11
+ */
12
+ function getFilesToLoad({ supportedItemTypes, statsFile, }) {
13
+ const filesToLoad = [];
14
+ if (supportedItemTypes.length === 0 || statsFile.length === 0) {
15
+ return [];
16
+ }
17
+ const filteredStatsFile = statsFile.filter((file) => supportedItemTypes.includes(file.item_type));
18
+ const orderedFiles = filteredStatsFile.sort((a, b) => {
19
+ const aIndex = supportedItemTypes.indexOf(a.item_type);
20
+ const bIndex = supportedItemTypes.indexOf(b.item_type);
21
+ return aIndex - bIndex;
22
+ });
23
+ for (const file of orderedFiles) {
24
+ filesToLoad.push({
25
+ id: file.id,
26
+ file_name: file.file_name,
27
+ itemType: file.item_type,
28
+ count: parseInt(file.count),
29
+ completed: false,
30
+ lineToProcess: 0,
31
+ });
32
+ }
33
+ return filesToLoad;
34
+ }
35
+ /**
36
+ * Adds a report to the loader report.
37
+ * @param {LoaderReport[]} loaderReports - The loader reports
38
+ * @param {LoaderReport} report - The report to add
39
+ * @returns {LoaderReport[]} The updated loader reports
40
+ */
41
+ function addReportToLoaderReport({ loaderReports, report, }) {
42
+ const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
43
+ if (existingReport) {
44
+ existingReport[loading_1.ActionType.CREATED] = existingReport[loading_1.ActionType.CREATED]
45
+ ? report[loading_1.ActionType.CREATED]
46
+ ? existingReport[loading_1.ActionType.CREATED] + report[loading_1.ActionType.CREATED]
47
+ : existingReport[loading_1.ActionType.CREATED]
48
+ : report[loading_1.ActionType.CREATED];
49
+ existingReport[loading_1.ActionType.UPDATED] = existingReport[loading_1.ActionType.UPDATED]
50
+ ? report[loading_1.ActionType.UPDATED]
51
+ ? existingReport[loading_1.ActionType.UPDATED] + report[loading_1.ActionType.UPDATED]
52
+ : existingReport[loading_1.ActionType.UPDATED]
53
+ : report[loading_1.ActionType.UPDATED];
54
+ existingReport[loading_1.ActionType.FAILED] = existingReport[loading_1.ActionType.FAILED]
55
+ ? report[loading_1.ActionType.FAILED]
56
+ ? existingReport[loading_1.ActionType.FAILED] + report[loading_1.ActionType.FAILED]
57
+ : existingReport[loading_1.ActionType.FAILED]
58
+ : report[loading_1.ActionType.FAILED];
59
+ }
60
+ else {
61
+ loaderReports.push(report);
62
+ }
63
+ return loaderReports;
64
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker-adapter.helpers.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.helpers.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.test.ts"],"names":[],"mappings":""}