@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +1802 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.js +13 -2
  5. package/dist/attachments-streaming/attachments-streaming-pool.test.js +201 -6
  6. package/dist/common/constants.d.ts +16 -5
  7. package/dist/common/constants.d.ts.map +1 -1
  8. package/dist/common/constants.js +20 -6
  9. package/dist/common/control-protocol.d.ts +2 -2
  10. package/dist/common/control-protocol.d.ts.map +1 -1
  11. package/dist/common/control-protocol.js +1 -5
  12. package/dist/common/helpers.d.ts +8 -0
  13. package/dist/common/helpers.d.ts.map +1 -1
  14. package/dist/common/helpers.js +15 -0
  15. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  16. package/dist/common/install-initial-domain-mapping.test.js +5 -2
  17. package/dist/common/test-utils.d.ts +19 -0
  18. package/dist/common/test-utils.d.ts.map +1 -0
  19. package/dist/common/test-utils.js +101 -0
  20. package/dist/common/time-value-resolver.d.ts +47 -0
  21. package/dist/common/time-value-resolver.d.ts.map +1 -0
  22. package/dist/common/time-value-resolver.js +141 -0
  23. package/dist/common/time-value-resolver.test.d.ts +2 -0
  24. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  25. package/dist/common/time-value-resolver.test.js +268 -0
  26. package/dist/http/axios-client-internal.d.ts.map +1 -1
  27. package/dist/http/axios-client-internal.js +7 -0
  28. package/dist/http/axios-client-internal.test.js +21 -0
  29. package/dist/http/axios-client.d.ts.map +1 -1
  30. package/dist/http/axios-client.js +6 -3
  31. package/dist/http/index.d.ts +0 -1
  32. package/dist/http/index.d.ts.map +1 -1
  33. package/dist/http/index.js +0 -1
  34. package/dist/index.d.ts +9 -8
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +12 -10
  37. package/dist/logger/logger.d.ts +1 -17
  38. package/dist/logger/logger.d.ts.map +1 -1
  39. package/dist/logger/logger.interfaces.d.ts +2 -2
  40. package/dist/logger/logger.js +35 -27
  41. package/dist/logger/logger.test.js +66 -19
  42. package/dist/mappers/mappers.interface.d.ts +5 -5
  43. package/dist/mappers/mappers.interface.js +3 -3
  44. package/dist/mappers/mappers.test.js +8 -10
  45. package/dist/mock-server/mock-server.d.ts +47 -0
  46. package/dist/mock-server/mock-server.d.ts.map +1 -0
  47. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  48. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  49. package/dist/mock-server/mock-server.interfaces.js +4 -0
  50. package/dist/mock-server/mock-server.js +278 -0
  51. package/dist/multithreading/create-worker.test.js +36 -17
  52. package/dist/multithreading/process-task.d.ts.map +1 -1
  53. package/dist/multithreading/process-task.js +15 -48
  54. package/dist/multithreading/process-task.test.d.ts +2 -0
  55. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  56. package/dist/multithreading/process-task.test.js +146 -0
  57. package/dist/multithreading/spawn/spawn.d.ts +3 -3
  58. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  59. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  60. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  61. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  62. package/dist/multithreading/spawn/spawn.js +20 -25
  63. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  64. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  65. package/dist/multithreading/spawn/spawn.test.js +232 -0
  66. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
  67. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
  68. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  69. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
  70. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
  71. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
  72. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
  73. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
  74. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
  75. package/dist/multithreading/worker-adapter/worker-adapter.js +232 -131
  76. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
  77. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
  78. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
  79. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
  80. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
  81. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
  82. package/dist/repo/repo.d.ts.map +1 -1
  83. package/dist/repo/repo.interfaces.d.ts +4 -2
  84. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  85. package/dist/repo/repo.js +4 -3
  86. package/dist/repo/repo.test.js +60 -7
  87. package/dist/state/state.d.ts +7 -1
  88. package/dist/state/state.d.ts.map +1 -1
  89. package/dist/state/state.extract-window.test.d.ts +2 -0
  90. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  91. package/dist/state/state.extract-window.test.js +163 -0
  92. package/dist/state/state.interfaces.d.ts +17 -7
  93. package/dist/state/state.interfaces.d.ts.map +1 -1
  94. package/dist/state/state.interfaces.js +4 -2
  95. package/dist/state/state.js +111 -12
  96. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  97. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  98. package/dist/state/state.pending-boundaries.test.js +189 -0
  99. package/dist/state/state.post-state.test.d.ts +2 -0
  100. package/dist/state/state.post-state.test.d.ts.map +1 -0
  101. package/dist/state/state.post-state.test.js +77 -0
  102. package/dist/state/state.test.js +117 -73
  103. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  104. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  105. package/dist/state/state.time-value-resolution.test.js +175 -0
  106. package/dist/tsdoc-metadata.json +11 -0
  107. package/dist/types/common.d.ts +0 -25
  108. package/dist/types/common.d.ts.map +1 -1
  109. package/dist/types/common.js +1 -11
  110. package/dist/types/external-domain-metadata.d.ts +20 -1
  111. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  112. package/dist/types/extraction.d.ts +89 -218
  113. package/dist/types/extraction.d.ts.map +1 -1
  114. package/dist/types/extraction.js +56 -124
  115. package/dist/types/extraction.test.js +69 -43
  116. package/dist/types/index.d.ts +3 -2
  117. package/dist/types/index.d.ts.map +1 -1
  118. package/dist/types/index.js +6 -3
  119. package/dist/types/loading.d.ts +4 -24
  120. package/dist/types/loading.d.ts.map +1 -1
  121. package/dist/types/loading.js +1 -22
  122. package/dist/types/workers.d.ts +31 -14
  123. package/dist/types/workers.d.ts.map +1 -1
  124. package/dist/types/workers.js +1 -0
  125. package/dist/uploader/uploader.d.ts +1 -0
  126. package/dist/uploader/uploader.d.ts.map +1 -1
  127. package/dist/uploader/uploader.helpers.test.js +0 -11
  128. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  129. package/dist/uploader/uploader.js +53 -20
  130. package/dist/uploader/uploader.test.js +22 -10
  131. package/package.json +10 -13
  132. package/dist/common/event-type-translation.d.ts +0 -24
  133. package/dist/common/event-type-translation.d.ts.map +0 -1
  134. package/dist/common/event-type-translation.js +0 -117
  135. package/dist/deprecated/adapter/index.d.ts +0 -62
  136. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  137. package/dist/deprecated/adapter/index.js +0 -151
  138. package/dist/deprecated/common/helpers.d.ts +0 -7
  139. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  140. package/dist/deprecated/common/helpers.js +0 -47
  141. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  142. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  143. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  144. package/dist/deprecated/demo-extractor/index.js +0 -161
  145. package/dist/deprecated/http/client.d.ts +0 -22
  146. package/dist/deprecated/http/client.d.ts.map +0 -1
  147. package/dist/deprecated/http/client.js +0 -161
  148. package/dist/deprecated/uploader/index.d.ts +0 -35
  149. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  150. package/dist/deprecated/uploader/index.js +0 -161
  151. package/dist/http/types.d.ts +0 -17
  152. package/dist/http/types.d.ts.map +0 -1
  153. package/dist/http/types.js +0 -2
  154. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  155. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  156. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
@@ -20,7 +20,7 @@ const extraction_1 = require("../../types/extraction");
20
20
  const loading_1 = require("../../types/loading");
21
21
  const workers_1 = require("../../types/workers");
22
22
  const uploader_1 = require("../../uploader/uploader");
23
- const event_type_translation_1 = require("../../common/event-type-translation");
23
+ const helpers_1 = require("../../common/helpers");
24
24
  function createWorkerAdapter({ event, adapterState, options, }) {
25
25
  return new WorkerAdapter({
26
26
  event,
@@ -29,7 +29,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
29
29
  });
30
30
  }
31
31
  /**
32
- * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
32
+ * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
33
33
  * as parameter in processTask and onTimeout functions. The class provides
34
34
  * utilities to emit control events to the platform, update the state of the connector,
35
35
  * and upload artifacts to the platform.
@@ -46,6 +46,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
46
46
  class WorkerAdapter {
47
47
  constructor({ event, adapterState, options, }) {
48
48
  this.repos = [];
49
+ this.currentEventDataLength = 0;
49
50
  this.event = event;
50
51
  this.options = options;
51
52
  this.adapterState = adapterState;
@@ -79,17 +80,38 @@ class WorkerAdapter {
79
80
  get mappers() {
80
81
  return this._mappers;
81
82
  }
83
+ get extractionScope() {
84
+ return this.adapterState.extractionScope;
85
+ }
86
+ /**
87
+ * Returns whether the given item type should be extracted.
88
+ * Defaults to true if the scope is empty or the item type is not listed.
89
+ */
90
+ shouldExtract(itemType) {
91
+ const scope = this.extractionScope;
92
+ if (Object.keys(scope).length === 0)
93
+ return true;
94
+ if (!(itemType in scope))
95
+ return true;
96
+ return scope[itemType].extract;
97
+ }
82
98
  initializeRepos(repos) {
83
99
  this.repos = repos.map((repo) => {
84
- const shouldNormalize = repo.itemType !== constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
85
- repo.itemType !== constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT;
100
+ const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
101
+ repo.itemType !== constants_1.SSOR_ATTACHMENT;
86
102
  return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
87
103
  var _a;
88
104
  // We need to store artifacts ids in state for later use when streaming attachments
89
- if (repo.itemType === constants_1.AIRDROP_DEFAULT_ITEM_TYPES.ATTACHMENTS) {
105
+ if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
90
106
  (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
91
107
  }
92
- }, options: this.options }));
108
+ // Calculate size of the entire artifact object that goes in the SQS message
109
+ this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
110
+ if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
111
+ !this.isTimeout) {
112
+ this.isTimeout = true;
113
+ }
114
+ }, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
93
115
  });
94
116
  }
95
117
  getRepo(itemType) {
@@ -123,29 +145,42 @@ class WorkerAdapter {
123
145
  */
124
146
  async emit(newEventType, data) {
125
147
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
126
- newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
148
+ var _a;
127
149
  if (this.hasWorkerEmitted) {
128
150
  console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
129
151
  return;
130
152
  }
131
- // We want to upload all the repos before emitting the event, except for the external sync units done event
132
- if (newEventType !== extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone) {
133
- console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
134
- try {
135
- await this.uploadAllRepos();
136
- }
137
- catch (error) {
138
- console.error('Error while uploading repos', error);
139
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
140
- this.hasWorkerEmitted = true;
141
- return;
142
- }
153
+ // Upload all repos before emitting the event
154
+ console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
155
+ try {
156
+ await this.uploadAllRepos();
157
+ }
158
+ catch (error) {
159
+ console.error('Error while uploading repos', error);
160
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
161
+ this.hasWorkerEmitted = true;
162
+ return;
143
163
  }
144
- // If the extraction is done, we want to save the timestamp of the last successful sync
145
164
  if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
146
- console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
147
- this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
148
- this.state.lastSyncStarted = '';
165
+ // Clear pending extraction boundaries now that the cycle is complete
166
+ this.state.pendingWorkersOldest = '';
167
+ this.state.pendingWorkersNewest = '';
168
+ // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
169
+ // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
170
+ const extractionStart = this.event.payload.event_context.extract_from;
171
+ const extractionEnd = this.event.payload.event_context.extract_to;
172
+ if (extractionStart &&
173
+ (!this.state.workersOldest ||
174
+ extractionStart < this.state.workersOldest)) {
175
+ console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
176
+ this.state.workersOldest = extractionStart;
177
+ }
178
+ if (extractionEnd &&
179
+ (!this.state.workersNewest ||
180
+ extractionEnd > this.state.workersNewest)) {
181
+ console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
182
+ this.state.workersNewest = extractionEnd;
183
+ }
149
184
  }
150
185
  // We want to save the state every time we emit an event, except for the start and delete events
151
186
  if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
@@ -161,11 +196,17 @@ class WorkerAdapter {
161
196
  }
162
197
  }
163
198
  try {
199
+ // Always prune error messages to make them shorter before emit
200
+ if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
201
+ data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
202
+ }
203
+ const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
204
+ const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
164
205
  await (0, control_protocol_1.emit)({
165
206
  eventType: newEventType,
166
207
  event: this.event,
167
- data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
168
- ? { artifacts: this.artifacts }
208
+ data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
209
+ ? { reports: this.reports, processed_files: this.processedFiles }
169
210
  : {})),
170
211
  });
171
212
  const message = {
@@ -192,9 +233,6 @@ class WorkerAdapter {
192
233
  }
193
234
  }
194
235
  }
195
- handleTimeout() {
196
- this.isTimeout = true;
197
- }
198
236
  async loadItemTypes({ itemTypesToLoad, }) {
199
237
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
200
238
  var _a;
@@ -223,57 +261,74 @@ class WorkerAdapter {
223
261
  };
224
262
  }
225
263
  console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
226
- outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
227
- .filesToLoad) {
228
- const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
229
- if (!itemTypeToLoad) {
230
- console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
231
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
232
- error: {
233
- message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
234
- },
235
- });
236
- break;
237
- }
238
- if (!fileToLoad.completed) {
239
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
240
- artifactId: fileToLoad.id,
241
- isGzipped: true,
242
- });
243
- if (transformerFileError) {
244
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
264
+ try {
265
+ outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
266
+ .filesToLoad) {
267
+ const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
268
+ if (!itemTypeToLoad) {
269
+ console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
245
270
  await this.emit(loading_1.LoaderEventType.DataLoadingError, {
246
271
  error: {
247
- message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
272
+ message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
248
273
  },
249
274
  });
275
+ break;
250
276
  }
251
- const transformerFile = response;
252
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
253
- const { report, rateLimit } = await this.loadItem({
254
- item: transformerFile[i],
255
- itemTypeToLoad,
277
+ if (!fileToLoad.completed) {
278
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
279
+ artifactId: fileToLoad.id,
280
+ isGzipped: true,
256
281
  });
257
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
258
- await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
259
- delay: rateLimit.delay,
260
- reports: this.reports,
261
- processed_files: this.processedFiles,
282
+ if (transformerFileError) {
283
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
284
+ await this.emit(loading_1.LoaderEventType.DataLoadingError, {
285
+ error: {
286
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
287
+ },
262
288
  });
263
289
  break outerloop;
264
290
  }
265
- if (report) {
266
- (0, worker_adapter_helpers_1.addReportToLoaderReport)({
267
- loaderReports: this.loaderReports,
268
- report,
291
+ const transformerFile = response;
292
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
293
+ if (this.isTimeout) {
294
+ console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
295
+ await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
296
+ process.exit(0);
297
+ }
298
+ const { report, rateLimit } = await this.loadItem({
299
+ item: transformerFile[i],
300
+ itemTypeToLoad,
269
301
  });
270
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
302
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
303
+ await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
304
+ delay: rateLimit.delay,
305
+ reports: this.reports,
306
+ processed_files: this.processedFiles,
307
+ });
308
+ break outerloop;
309
+ }
310
+ if (report) {
311
+ (0, worker_adapter_helpers_1.addReportToLoaderReport)({
312
+ loaderReports: this.loaderReports,
313
+ report,
314
+ });
315
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
316
+ }
271
317
  }
318
+ fileToLoad.completed = true;
319
+ this._processedFiles.push(fileToLoad.id);
272
320
  }
273
- fileToLoad.completed = true;
274
- this._processedFiles.push(fileToLoad.id);
275
321
  }
276
322
  }
323
+ catch (error) {
324
+ console.error('Error during data loading.', (0, logger_1.serializeError)(error));
325
+ await this.emit(loading_1.LoaderEventType.DataLoadingError, {
326
+ error: {
327
+ message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
328
+ },
329
+ });
330
+ process.exit(1);
331
+ }
277
332
  return {
278
333
  reports: this.reports,
279
334
  processed_files: this.processedFiles,
@@ -320,42 +375,63 @@ class WorkerAdapter {
320
375
  };
321
376
  }
322
377
  const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
323
- outerloop: for (const fileToLoad of filesToLoad) {
324
- if (!fileToLoad.completed) {
325
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
326
- artifactId: fileToLoad.id,
327
- isGzipped: true,
328
- });
329
- const transformerFile = response;
330
- if (transformerFileError) {
331
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
332
- break outerloop;
333
- }
334
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
335
- const { report, rateLimit } = await this.loadAttachment({
336
- item: transformerFile[i],
337
- create,
378
+ try {
379
+ outerloop: for (const fileToLoad of filesToLoad) {
380
+ if (!fileToLoad.completed) {
381
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
382
+ artifactId: fileToLoad.id,
383
+ isGzipped: true,
338
384
  });
339
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
340
- await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
341
- delay: rateLimit.delay,
342
- reports: this.reports,
343
- processed_files: this.processedFiles,
385
+ const transformerFile = response;
386
+ if (transformerFileError) {
387
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
388
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
389
+ error: {
390
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
391
+ },
344
392
  });
345
393
  break outerloop;
346
394
  }
347
- if (report) {
348
- (0, worker_adapter_helpers_1.addReportToLoaderReport)({
349
- loaderReports: this.loaderReports,
350
- report,
395
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
396
+ if (this.isTimeout) {
397
+ console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
398
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
399
+ process.exit(0);
400
+ }
401
+ const { report, rateLimit } = await this.loadAttachment({
402
+ item: transformerFile[i],
403
+ create,
351
404
  });
352
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
405
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
406
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingDelayed, {
407
+ delay: rateLimit.delay,
408
+ reports: this.reports,
409
+ processed_files: this.processedFiles,
410
+ });
411
+ break outerloop;
412
+ }
413
+ if (report) {
414
+ (0, worker_adapter_helpers_1.addReportToLoaderReport)({
415
+ loaderReports: this.loaderReports,
416
+ report,
417
+ });
418
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
419
+ }
353
420
  }
421
+ fileToLoad.completed = true;
422
+ this._processedFiles.push(fileToLoad.id);
354
423
  }
355
- fileToLoad.completed = true;
356
- this._processedFiles.push(fileToLoad.id);
357
424
  }
358
425
  }
426
+ catch (error) {
427
+ console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
428
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
429
+ error: {
430
+ message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
431
+ },
432
+ });
433
+ process.exit(1);
434
+ }
359
435
  return {
360
436
  reports: this.reports,
361
437
  processed_files: this.processedFiles,
@@ -381,10 +457,12 @@ class WorkerAdapter {
381
457
  };
382
458
  }
383
459
  // Update item in external system
384
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
385
- item,
386
- mappers: this._mappers,
387
- event: this.event,
460
+ const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
461
+ return await itemTypeToLoad.update({
462
+ item,
463
+ mappers: this._mappers,
464
+ event: this.event,
465
+ });
388
466
  });
389
467
  if (id) {
390
468
  try {
@@ -442,10 +520,12 @@ class WorkerAdapter {
442
520
  if (axios_1.default.isAxiosError(error)) {
443
521
  if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
444
522
  // Create item in external system if mapper record not found
445
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
446
- item,
447
- mappers: this._mappers,
448
- event: this.event,
523
+ const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
524
+ return await itemTypeToLoad.create({
525
+ item,
526
+ mappers: this._mappers,
527
+ event: this.event,
528
+ });
449
529
  });
450
530
  if (id) {
451
531
  // Create mapper
@@ -513,11 +593,11 @@ class WorkerAdapter {
513
593
  }
514
594
  async processAttachment(attachment, stream) {
515
595
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
516
- var _a;
517
- const { httpStream, delay, error } = await stream({
596
+ var _a, _b, _c;
597
+ const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
518
598
  item: attachment,
519
599
  event: this.event,
520
- });
600
+ }));
521
601
  if (error) {
522
602
  return { error };
523
603
  }
@@ -525,37 +605,44 @@ class WorkerAdapter {
525
605
  return { delay };
526
606
  }
527
607
  if (httpStream) {
528
- const fileType = httpStream.headers['content-type'] || 'application/octet-stream';
529
- const fileSize = httpStream.headers['content-length']
530
- ? parseInt(httpStream.headers['content-length'])
531
- : undefined;
608
+ const fileType = attachment.content_type ||
609
+ ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
610
+ 'application/octet-stream';
611
+ const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
612
+ const fileSize = contentLength ? parseInt(contentLength) : undefined;
532
613
  // Get upload URL
533
614
  const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
534
615
  if (artifactUrlError) {
535
- console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
536
- (0, logger_1.serializeError)(artifactUrlError));
537
616
  this.destroyHttpStream(httpStream);
538
- return;
539
- }
540
- if (this.isTimeout) {
541
- this.destroyHttpStream(httpStream);
542
- return;
617
+ return {
618
+ error: {
619
+ message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
620
+ fileSize: fileSize,
621
+ },
622
+ };
543
623
  }
544
624
  // Stream attachment
545
625
  const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
546
626
  if (uploadedArtifactError) {
547
- console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
548
- (0, logger_1.serializeError)(uploadedArtifactError));
549
627
  this.destroyHttpStream(httpStream);
550
- return;
628
+ return {
629
+ error: {
630
+ message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
631
+ (0, logger_1.serializeError)(uploadedArtifactError),
632
+ fileSize: fileSize,
633
+ },
634
+ };
551
635
  }
552
636
  // Confirm attachment upload
553
637
  const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
554
638
  if (confirmArtifactUploadError) {
555
- console.warn('Error while confirming upload for attachment ID ' +
556
- attachment.id +
557
- '.', confirmArtifactUploadError);
558
- return;
639
+ return {
640
+ error: {
641
+ message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
642
+ (0, logger_1.serializeError)(confirmArtifactUploadError),
643
+ fileSize: fileSize,
644
+ },
645
+ };
559
646
  }
560
647
  const ssorAttachment = {
561
648
  id: {
@@ -579,9 +666,14 @@ class WorkerAdapter {
579
666
  else if (attachment.inline === false) {
580
667
  ssorAttachment.inline = false;
581
668
  }
582
- await ((_a = this.getRepo('ssor_attachment')) === null || _a === void 0 ? void 0 : _a.push([ssorAttachment]));
669
+ await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
670
+ return;
583
671
  }
584
- return;
672
+ return {
673
+ error: {
674
+ message: `Error while opening attachment stream. Skipping attachment.`,
675
+ },
676
+ };
585
677
  });
586
678
  }
587
679
  /**
@@ -606,11 +698,11 @@ class WorkerAdapter {
606
698
  async loadAttachment({ item, create, }) {
607
699
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
608
700
  // Create item
609
- const { id, delay, error } = await create({
701
+ const { id, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => create({
610
702
  item,
611
703
  mappers: this._mappers,
612
704
  event: this.event,
613
- });
705
+ }));
614
706
  if (delay) {
615
707
  return {
616
708
  rateLimit: {
@@ -707,15 +799,17 @@ class WorkerAdapter {
707
799
  console.log(`Using custom processors for attachments.`);
708
800
  const reducer = processors.reducer;
709
801
  const iterator = processors.iterator;
710
- const reducedAttachments = reducer({
802
+ const reducedAttachments = (0, logger_context_1.runWithUserLogContext)(() => reducer({
711
803
  attachments,
712
804
  adapter: this,
713
805
  batchSize,
714
- });
715
- response = await iterator({
716
- reducedAttachments,
717
- adapter: this,
718
- stream,
806
+ }));
807
+ response = await (0, logger_context_1.runWithUserLogContext)(async () => {
808
+ return await iterator({
809
+ reducedAttachments,
810
+ adapter: this,
811
+ stream,
812
+ });
719
813
  });
720
814
  }
721
815
  else {
@@ -731,6 +825,13 @@ class WorkerAdapter {
731
825
  if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
732
826
  return response;
733
827
  }
828
+ // On timeout, emit progress and exit to allow continuation.
829
+ if (this.isTimeout) {
830
+ console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.`);
831
+ await this.emit(extraction_1.ExtractorEventType.AttachmentExtractionProgress);
832
+ process.exit(0);
833
+ return;
834
+ }
734
835
  console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
735
836
  attachmentsMetadata.artifactIds.shift();
736
837
  attachmentsMetadata.lastProcessed = 0;
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker-adapter.loading.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.loading.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.loading.test.ts"],"names":[],"mappings":""}