@devrev/ts-adaas 1.12.3-beta.2 → 1.12.3-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/common/constants.js +11 -11
  2. package/dist/common/control-protocol.d.ts.map +1 -1
  3. package/dist/common/control-protocol.js +5 -1
  4. package/dist/common/event-type-translation.d.ts +24 -0
  5. package/dist/common/event-type-translation.d.ts.map +1 -0
  6. package/dist/common/event-type-translation.js +117 -0
  7. package/dist/common/helpers.d.ts.map +1 -1
  8. package/dist/common/helpers.js +25 -7
  9. package/dist/deprecated/adapter/index.d.ts.map +1 -1
  10. package/dist/deprecated/adapter/index.js +2 -0
  11. package/dist/logger/logger.d.ts +1 -16
  12. package/dist/logger/logger.d.ts.map +1 -1
  13. package/dist/logger/logger.interfaces.d.ts +3 -4
  14. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  15. package/dist/logger/logger.js +6 -33
  16. package/dist/logger/logger.test.js +10 -71
  17. package/dist/state/state.js +2 -2
  18. package/dist/state/state.test.js +3 -3
  19. package/dist/tests/backwards-compatibility/backwards-compatibility.test.js +2 -17
  20. package/dist/tests/timeout-handling/timeout-1.js +2 -2
  21. package/dist/tests/timeout-handling/timeout-1.test.js +1 -1
  22. package/dist/tests/timeout-handling/timeout-2.js +3 -3
  23. package/dist/tests/timeout-handling/timeout-2.test.js +1 -1
  24. package/dist/tests/timeout-handling/timeout-3a.js +2 -2
  25. package/dist/tests/timeout-handling/timeout-3a.test.js +1 -1
  26. package/dist/tests/timeout-handling/timeout-3b.js +2 -2
  27. package/dist/tests/timeout-handling/timeout-3b.test.js +1 -1
  28. package/dist/types/extraction.d.ts +99 -2
  29. package/dist/types/extraction.d.ts.map +1 -1
  30. package/dist/types/extraction.js +102 -2
  31. package/dist/types/loading.d.ts +21 -1
  32. package/dist/types/loading.d.ts.map +1 -1
  33. package/dist/types/loading.js +23 -0
  34. package/dist/types/workers.d.ts +7 -15
  35. package/dist/types/workers.d.ts.map +1 -1
  36. package/dist/workers/process-task.d.ts.map +1 -1
  37. package/dist/workers/process-task.js +35 -37
  38. package/dist/workers/spawn.d.ts +3 -3
  39. package/dist/workers/spawn.d.ts.map +1 -1
  40. package/dist/workers/spawn.js +43 -49
  41. package/dist/workers/worker-adapter.d.ts.map +1 -1
  42. package/dist/workers/worker-adapter.js +475 -496
  43. package/dist/workers/worker-adapter.test.js +0 -7
  44. package/dist/workers/worker.js +1 -4
  45. package/package.json +1 -1
  46. package/dist/logger/logger.context.d.ts +0 -6
  47. package/dist/logger/logger.context.d.ts.map +0 -1
  48. package/dist/logger/logger.context.js +0 -31
  49. package/dist/logger/logger.worker-fixture.d.ts +0 -2
  50. package/dist/logger/logger.worker-fixture.d.ts.map +0 -1
  51. package/dist/logger/logger.worker-fixture.js +0 -10
  52. package/dist/workers/default-workers/attachments-deletion.d.ts +0 -2
  53. package/dist/workers/default-workers/attachments-deletion.d.ts.map +0 -1
  54. package/dist/workers/default-workers/attachments-deletion.js +0 -13
  55. package/dist/workers/default-workers/attachments-extraction.d.ts +0 -2
  56. package/dist/workers/default-workers/attachments-extraction.d.ts.map +0 -1
  57. package/dist/workers/default-workers/attachments-extraction.js +0 -95
  58. package/dist/workers/default-workers/data-deletion.d.ts +0 -2
  59. package/dist/workers/default-workers/data-deletion.d.ts.map +0 -1
  60. package/dist/workers/default-workers/data-deletion.js +0 -15
  61. package/dist/workers/default-workers/data-extraction.d.ts +0 -2
  62. package/dist/workers/default-workers/data-extraction.d.ts.map +0 -1
  63. package/dist/workers/default-workers/data-extraction.js +0 -100
  64. package/dist/workers/default-workers/delete-loader-attachment-state.d.ts +0 -2
  65. package/dist/workers/default-workers/delete-loader-attachment-state.d.ts.map +0 -1
  66. package/dist/workers/default-workers/delete-loader-attachment-state.js +0 -15
  67. package/dist/workers/default-workers/delete-loader-state.d.ts +0 -2
  68. package/dist/workers/default-workers/delete-loader-state.d.ts.map +0 -1
  69. package/dist/workers/default-workers/delete-loader-state.js +0 -15
  70. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +0 -2
  71. package/dist/workers/default-workers/external-sync-units-extraction.d.ts.map +0 -1
  72. package/dist/workers/default-workers/external-sync-units-extraction.js +0 -27
  73. package/dist/workers/default-workers/load-attachments.d.ts +0 -2
  74. package/dist/workers/default-workers/load-attachments.d.ts.map +0 -1
  75. package/dist/workers/default-workers/load-attachments.js +0 -19
  76. package/dist/workers/default-workers/load-data.d.ts +0 -2
  77. package/dist/workers/default-workers/load-data.d.ts.map +0 -1
  78. package/dist/workers/default-workers/load-data.js +0 -18
  79. package/dist/workers/default-workers/metadata-extraction.d.ts +0 -2
  80. package/dist/workers/default-workers/metadata-extraction.d.ts.map +0 -1
  81. package/dist/workers/default-workers/metadata-extraction.js +0 -26
  82. package/dist/workers/dummy-extractor/data-normalization.d.ts +0 -5
  83. package/dist/workers/dummy-extractor/data-normalization.d.ts.map +0 -1
  84. package/dist/workers/dummy-extractor/data-normalization.js +0 -41
  85. package/dist/workers/dummy-extractor/external_domain_metadata.json +0 -58
@@ -12,7 +12,6 @@ const constants_1 = require("../common/constants");
12
12
  const control_protocol_1 = require("../common/control-protocol");
13
13
  const helpers_1 = require("../common/helpers");
14
14
  const logger_1 = require("../logger/logger");
15
- const logger_context_1 = require("../logger/logger.context");
16
15
  const mappers_1 = require("../mappers/mappers");
17
16
  const mappers_interface_1 = require("../mappers/mappers.interface");
18
17
  const repo_1 = require("../repo/repo");
@@ -94,19 +93,15 @@ class WorkerAdapter {
94
93
  });
95
94
  }
96
95
  getRepo(itemType) {
97
- return (0, logger_context_1.runWithSdkLogContext)(() => {
98
- const repo = this.repos.find((repo) => repo.itemType === itemType);
99
- if (!repo) {
100
- console.error(`Repo for item type ${itemType} not found.`);
101
- return;
102
- }
103
- return repo;
104
- });
96
+ const repo = this.repos.find((repo) => repo.itemType === itemType);
97
+ if (!repo) {
98
+ console.error(`Repo for item type ${itemType} not found.`);
99
+ return;
100
+ }
101
+ return repo;
105
102
  }
106
103
  async postState() {
107
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
108
- await this.adapterState.postState();
109
- });
104
+ await this.adapterState.postState();
110
105
  }
111
106
  get artifacts() {
112
107
  return this._artifacts;
@@ -123,65 +118,63 @@ class WorkerAdapter {
123
118
  * @param data - The data to be sent with the event
124
119
  */
125
120
  async emit(newEventType, data) {
126
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
127
- if (this.hasWorkerEmitted) {
128
- console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
129
- return;
130
- }
131
- // We want to upload all the repos before emitting the event, except for the external sync units done event
132
- if (newEventType !== extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone) {
133
- console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
134
- try {
135
- await this.uploadAllRepos();
136
- }
137
- catch (error) {
138
- console.error('Error while uploading repos', error);
139
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
140
- this.hasWorkerEmitted = true;
141
- return;
142
- }
143
- }
144
- // If the extraction is done, we want to save the timestamp of the last successful sync
145
- if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
146
- console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
147
- this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
148
- this.state.lastSyncStarted = '';
121
+ if (this.hasWorkerEmitted) {
122
+ console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
123
+ return;
124
+ }
125
+ // We want to upload all the repos before emitting the event, except for the external sync units done event
126
+ if (newEventType !== extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone) {
127
+ console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
128
+ try {
129
+ await this.uploadAllRepos();
149
130
  }
150
- // We want to save the state every time we emit an event, except for the start and delete events
151
- if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
152
- console.log(`Saving state before emitting event with event type: ${newEventType}.`);
153
- try {
154
- await this.adapterState.postState(this.state);
155
- }
156
- catch (error) {
157
- console.error('Error while posting state', error);
158
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
159
- this.hasWorkerEmitted = true;
160
- return;
161
- }
131
+ catch (error) {
132
+ console.error('Error while uploading repos', error);
133
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
134
+ this.hasWorkerEmitted = true;
135
+ return;
162
136
  }
137
+ }
138
+ // If the extraction is done, we want to save the timestamp of the last successful sync
139
+ if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
140
+ console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
141
+ this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
142
+ this.state.lastSyncStarted = '';
143
+ }
144
+ // We want to save the state every time we emit an event, except for the start and delete events
145
+ if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
146
+ console.log(`Saving state before emitting event with event type: ${newEventType}.`);
163
147
  try {
164
- await (0, control_protocol_1.emit)({
165
- eventType: newEventType,
166
- event: this.event,
167
- data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
168
- ? { artifacts: this.artifacts }
169
- : {})),
170
- });
171
- const message = {
172
- subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
173
- payload: { eventType: newEventType },
174
- };
175
- this.artifacts = [];
176
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
177
- this.hasWorkerEmitted = true;
148
+ await this.adapterState.postState(this.state);
178
149
  }
179
150
  catch (error) {
180
- console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
151
+ console.error('Error while posting state', error);
181
152
  node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
182
153
  this.hasWorkerEmitted = true;
154
+ return;
183
155
  }
184
- });
156
+ }
157
+ try {
158
+ await (0, control_protocol_1.emit)({
159
+ eventType: newEventType,
160
+ event: this.event,
161
+ data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
162
+ ? { artifacts: this.artifacts }
163
+ : {})),
164
+ });
165
+ const message = {
166
+ subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
167
+ payload: { eventType: newEventType },
168
+ };
169
+ this.artifacts = [];
170
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
171
+ this.hasWorkerEmitted = true;
172
+ }
173
+ catch (error) {
174
+ console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
175
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
176
+ this.hasWorkerEmitted = true;
177
+ }
185
178
  }
186
179
  async uploadAllRepos() {
187
180
  for (const repo of this.repos) {
@@ -196,383 +189,373 @@ class WorkerAdapter {
196
189
  this.isTimeout = true;
197
190
  }
198
191
  async loadItemTypes({ itemTypesToLoad, }) {
199
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
200
- var _a;
201
- if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
202
- const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
203
- if (!itemTypes.length) {
204
- console.warn('No item types to load, returning.');
205
- return {
206
- reports: this.reports,
207
- processed_files: this.processedFiles,
208
- };
209
- }
210
- const filesToLoad = await this.getLoaderBatches({
211
- supportedItemTypes: itemTypes,
212
- });
213
- this.adapterState.state.fromDevRev = {
214
- filesToLoad,
215
- };
216
- }
217
- if (!this.adapterState.state.fromDevRev ||
218
- !this.adapterState.state.fromDevRev.filesToLoad.length) {
219
- console.warn('No files to load, returning.');
192
+ var _a;
193
+ if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
194
+ const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
195
+ if (!itemTypes.length) {
196
+ console.warn('No item types to load, returning.');
220
197
  return {
221
198
  reports: this.reports,
222
199
  processed_files: this.processedFiles,
223
200
  };
224
201
  }
225
- console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
226
- outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
227
- .filesToLoad) {
228
- const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
229
- if (!itemTypeToLoad) {
230
- console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
202
+ const filesToLoad = await this.getLoaderBatches({
203
+ supportedItemTypes: itemTypes,
204
+ });
205
+ this.adapterState.state.fromDevRev = {
206
+ filesToLoad,
207
+ };
208
+ }
209
+ if (!this.adapterState.state.fromDevRev ||
210
+ !this.adapterState.state.fromDevRev.filesToLoad.length) {
211
+ console.warn('No files to load, returning.');
212
+ return {
213
+ reports: this.reports,
214
+ processed_files: this.processedFiles,
215
+ };
216
+ }
217
+ console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
218
+ outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
219
+ .filesToLoad) {
220
+ const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
221
+ if (!itemTypeToLoad) {
222
+ console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
223
+ await this.emit(loading_1.LoaderEventType.DataLoadingError, {
224
+ error: {
225
+ message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
226
+ },
227
+ });
228
+ break;
229
+ }
230
+ if (!fileToLoad.completed) {
231
+ const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
232
+ artifactId: fileToLoad.id,
233
+ isGzipped: true,
234
+ }));
235
+ if (!transformerFile) {
236
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
231
237
  await this.emit(loading_1.LoaderEventType.DataLoadingError, {
232
238
  error: {
233
- message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
239
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
234
240
  },
235
241
  });
236
- break;
237
242
  }
238
- if (!fileToLoad.completed) {
239
- const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
240
- artifactId: fileToLoad.id,
241
- isGzipped: true,
242
- }));
243
- if (!transformerFile) {
244
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
245
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
246
- error: {
247
- message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
248
- },
243
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
244
+ const { report, rateLimit } = await this.loadItem({
245
+ item: transformerFile[i],
246
+ itemTypeToLoad,
247
+ });
248
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
249
+ await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
250
+ delay: rateLimit.delay,
251
+ reports: this.reports,
252
+ processed_files: this.processedFiles,
249
253
  });
254
+ break outerloop;
250
255
  }
251
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
252
- const { report, rateLimit } = await this.loadItem({
253
- item: transformerFile[i],
254
- itemTypeToLoad,
256
+ if (report) {
257
+ (0, helpers_1.addReportToLoaderReport)({
258
+ loaderReports: this.loaderReports,
259
+ report,
255
260
  });
256
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
257
- await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
258
- delay: rateLimit.delay,
259
- reports: this.reports,
260
- processed_files: this.processedFiles,
261
- });
262
- break outerloop;
263
- }
264
- if (report) {
265
- (0, helpers_1.addReportToLoaderReport)({
266
- loaderReports: this.loaderReports,
267
- report,
268
- });
269
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
270
- }
261
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
271
262
  }
272
- fileToLoad.completed = true;
273
- this._processedFiles.push(fileToLoad.id);
274
263
  }
264
+ fileToLoad.completed = true;
265
+ this._processedFiles.push(fileToLoad.id);
266
+ }
267
+ }
268
+ return {
269
+ reports: this.reports,
270
+ processed_files: this.processedFiles,
271
+ };
272
+ }
273
+ async getLoaderBatches({ supportedItemTypes, }) {
274
+ var _a;
275
+ const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
276
+ if (statsFileArtifactId) {
277
+ const statsFile = (await this.uploader.getJsonObjectByArtifactId({
278
+ artifactId: statsFileArtifactId,
279
+ }));
280
+ if (!statsFile || statsFile.length === 0) {
281
+ return [];
275
282
  }
283
+ const filesToLoad = (0, helpers_1.getFilesToLoad)({
284
+ supportedItemTypes,
285
+ statsFile,
286
+ });
287
+ return filesToLoad;
288
+ }
289
+ return [];
290
+ }
291
+ async loadAttachments({ create, }) {
292
+ var _a, _b;
293
+ if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
294
+ this.adapterState.state.fromDevRev = {
295
+ filesToLoad: await this.getLoaderBatches({
296
+ supportedItemTypes: ['attachment'],
297
+ }),
298
+ };
299
+ }
300
+ if (!this.adapterState.state.fromDevRev ||
301
+ ((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
302
+ console.log('No files to load, returning.');
276
303
  return {
277
304
  reports: this.reports,
278
305
  processed_files: this.processedFiles,
279
306
  };
280
- });
281
- }
282
- async getLoaderBatches({ supportedItemTypes, }) {
283
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
284
- var _a;
285
- const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
286
- if (statsFileArtifactId) {
287
- const statsFile = (await this.uploader.getJsonObjectByArtifactId({
288
- artifactId: statsFileArtifactId,
307
+ }
308
+ const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
309
+ outerloop: for (const fileToLoad of filesToLoad) {
310
+ if (!fileToLoad.completed) {
311
+ const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
312
+ artifactId: fileToLoad.id,
313
+ isGzipped: true,
289
314
  }));
290
- if (!statsFile || statsFile.length === 0) {
291
- return [];
315
+ if (!transformerFile) {
316
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
317
+ break outerloop;
292
318
  }
293
- const filesToLoad = (0, helpers_1.getFilesToLoad)({
294
- supportedItemTypes,
295
- statsFile,
296
- });
297
- return filesToLoad;
298
- }
299
- return [];
300
- });
301
- }
302
- async loadAttachments({ create, }) {
303
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
304
- var _a, _b;
305
- if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
306
- this.adapterState.state.fromDevRev = {
307
- filesToLoad: await this.getLoaderBatches({
308
- supportedItemTypes: ['attachment'],
309
- }),
310
- };
311
- }
312
- if (!this.adapterState.state.fromDevRev ||
313
- ((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
314
- console.log('No files to load, returning.');
315
- return {
316
- reports: this.reports,
317
- processed_files: this.processedFiles,
318
- };
319
- }
320
- const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
321
- outerloop: for (const fileToLoad of filesToLoad) {
322
- if (!fileToLoad.completed) {
323
- const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
324
- artifactId: fileToLoad.id,
325
- isGzipped: true,
326
- }));
327
- if (!transformerFile) {
328
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
319
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
320
+ const { report, rateLimit } = await this.loadAttachment({
321
+ item: transformerFile[i],
322
+ create,
323
+ });
324
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
325
+ await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
326
+ delay: rateLimit.delay,
327
+ reports: this.reports,
328
+ processed_files: this.processedFiles,
329
+ });
329
330
  break outerloop;
330
331
  }
331
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
332
- const { report, rateLimit } = await this.loadAttachment({
333
- item: transformerFile[i],
334
- create,
332
+ if (report) {
333
+ (0, helpers_1.addReportToLoaderReport)({
334
+ loaderReports: this.loaderReports,
335
+ report,
335
336
  });
336
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
337
- await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
338
- delay: rateLimit.delay,
339
- reports: this.reports,
340
- processed_files: this.processedFiles,
341
- });
342
- break outerloop;
343
- }
344
- if (report) {
345
- (0, helpers_1.addReportToLoaderReport)({
346
- loaderReports: this.loaderReports,
347
- report,
348
- });
349
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
350
- }
337
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
351
338
  }
352
- fileToLoad.completed = true;
353
- this._processedFiles.push(fileToLoad.id);
354
339
  }
340
+ fileToLoad.completed = true;
341
+ this._processedFiles.push(fileToLoad.id);
355
342
  }
356
- return {
357
- reports: this.reports,
358
- processed_files: this.processedFiles,
359
- };
360
- });
343
+ }
344
+ return {
345
+ reports: this.reports,
346
+ processed_files: this.processedFiles,
347
+ };
361
348
  }
362
349
  async loadItem({ item, itemTypeToLoad, }) {
363
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
364
- var _a;
365
- const devrevId = item.id.devrev;
366
- try {
367
- const syncMapperRecordResponse = await this._mappers.getByTargetId({
368
- sync_unit: this.event.payload.event_context.sync_unit,
369
- target: devrevId,
370
- });
371
- const syncMapperRecord = syncMapperRecordResponse.data;
372
- if (!syncMapperRecord) {
373
- console.warn('Failed to get sync mapper record from response.');
350
+ var _a;
351
+ const devrevId = item.id.devrev;
352
+ try {
353
+ const syncMapperRecordResponse = await this._mappers.getByTargetId({
354
+ sync_unit: this.event.payload.event_context.sync_unit,
355
+ target: devrevId,
356
+ });
357
+ const syncMapperRecord = syncMapperRecordResponse.data;
358
+ if (!syncMapperRecord) {
359
+ console.warn('Failed to get sync mapper record from response.');
360
+ return {
361
+ error: {
362
+ message: 'Failed to get sync mapper record from response.',
363
+ },
364
+ };
365
+ }
366
+ // Update item in external system
367
+ const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
368
+ item,
369
+ mappers: this._mappers,
370
+ event: this.event,
371
+ });
372
+ if (id) {
373
+ try {
374
+ const syncMapperRecordUpdateResponse = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL }, (modifiedDate && {
375
+ external_versions: {
376
+ add: [
377
+ {
378
+ modified_date: modifiedDate,
379
+ recipe_version: 0,
380
+ },
381
+ ],
382
+ },
383
+ })), { external_ids: {
384
+ add: [id],
385
+ }, targets: {
386
+ add: [devrevId],
387
+ } }));
388
+ console.log('Successfully updated sync mapper record.', syncMapperRecordUpdateResponse.data);
389
+ }
390
+ catch (error) {
391
+ console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
374
392
  return {
375
393
  error: {
376
- message: 'Failed to get sync mapper record from response.',
394
+ message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
377
395
  },
378
396
  };
379
397
  }
380
- // Update item in external system
381
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
382
- item,
383
- mappers: this._mappers,
384
- event: this.event,
385
- });
386
- if (id) {
387
- try {
388
- const syncMapperRecordUpdateResponse = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL }, (modifiedDate && {
389
- external_versions: {
390
- add: [
398
+ return {
399
+ report: {
400
+ item_type: itemTypeToLoad.itemType,
401
+ [loading_1.ActionType.UPDATED]: 1,
402
+ },
403
+ };
404
+ }
405
+ else if (delay) {
406
+ console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
407
+ return {
408
+ rateLimit: {
409
+ delay,
410
+ },
411
+ };
412
+ }
413
+ else {
414
+ console.warn('Failed to update item in external system', error);
415
+ return {
416
+ report: {
417
+ item_type: itemTypeToLoad.itemType,
418
+ [loading_1.ActionType.FAILED]: 1,
419
+ },
420
+ };
421
+ }
422
+ // TODO: Update mapper (optional)
423
+ }
424
+ catch (error) {
425
+ if (axios_1.default.isAxiosError(error)) {
426
+ if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
427
+ // Create item in external system if mapper record not found
428
+ const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
429
+ item,
430
+ mappers: this._mappers,
431
+ event: this.event,
432
+ });
433
+ if (id) {
434
+ // Create mapper
435
+ try {
436
+ const syncMapperRecordCreateResponse = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
437
+ external_versions: [
391
438
  {
392
439
  modified_date: modifiedDate,
393
440
  recipe_version: 0,
394
441
  },
395
442
  ],
396
- },
397
- })), { external_ids: {
398
- add: [id],
399
- }, targets: {
400
- add: [devrevId],
401
- } }));
402
- console.log('Successfully updated sync mapper record.', syncMapperRecordUpdateResponse.data);
403
- }
404
- catch (error) {
405
- console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
406
- return {
407
- error: {
408
- message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
409
- },
410
- };
411
- }
412
- return {
413
- report: {
414
- item_type: itemTypeToLoad.itemType,
415
- [loading_1.ActionType.UPDATED]: 1,
416
- },
417
- };
418
- }
419
- else if (delay) {
420
- console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
421
- return {
422
- rateLimit: {
423
- delay,
424
- },
425
- };
426
- }
427
- else {
428
- console.warn('Failed to update item in external system', error);
429
- return {
430
- report: {
431
- item_type: itemTypeToLoad.itemType,
432
- [loading_1.ActionType.FAILED]: 1,
433
- },
434
- };
435
- }
436
- // TODO: Update mapper (optional)
437
- }
438
- catch (error) {
439
- if (axios_1.default.isAxiosError(error)) {
440
- if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
441
- // Create item in external system if mapper record not found
442
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
443
- item,
444
- mappers: this._mappers,
445
- event: this.event,
446
- });
447
- if (id) {
448
- // Create mapper
449
- try {
450
- const syncMapperRecordCreateResponse = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
451
- external_versions: [
452
- {
453
- modified_date: modifiedDate,
454
- recipe_version: 0,
455
- },
456
- ],
457
- })));
458
- console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
459
- return {
460
- report: {
461
- item_type: itemTypeToLoad.itemType,
462
- [loading_1.ActionType.CREATED]: 1,
463
- },
464
- };
465
- }
466
- catch (error) {
467
- console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
468
- return {
469
- error: {
470
- message: 'Failed to create sync mapper record. ' +
471
- (0, logger_1.serializeError)(error),
472
- },
473
- };
474
- }
475
- }
476
- else if (delay) {
443
+ })));
444
+ console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
477
445
  return {
478
- rateLimit: {
479
- delay,
446
+ report: {
447
+ item_type: itemTypeToLoad.itemType,
448
+ [loading_1.ActionType.CREATED]: 1,
480
449
  },
481
450
  };
482
451
  }
483
- else {
484
- console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
452
+ catch (error) {
453
+ console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
485
454
  return {
486
- report: {
487
- item_type: itemTypeToLoad.itemType,
488
- [loading_1.ActionType.FAILED]: 1,
455
+ error: {
456
+ message: 'Failed to create sync mapper record. ' +
457
+ (0, logger_1.serializeError)(error),
489
458
  },
490
459
  };
491
460
  }
492
461
  }
462
+ else if (delay) {
463
+ return {
464
+ rateLimit: {
465
+ delay,
466
+ },
467
+ };
468
+ }
493
469
  else {
494
- console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
470
+ console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
495
471
  return {
496
- error: {
497
- message: error.message,
472
+ report: {
473
+ item_type: itemTypeToLoad.itemType,
474
+ [loading_1.ActionType.FAILED]: 1,
498
475
  },
499
476
  };
500
477
  }
501
478
  }
502
- console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
503
- return {
504
- error: {
505
- message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
506
- },
507
- };
479
+ else {
480
+ console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
481
+ return {
482
+ error: {
483
+ message: error.message,
484
+ },
485
+ };
486
+ }
508
487
  }
509
- });
488
+ console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
489
+ return {
490
+ error: {
491
+ message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
492
+ },
493
+ };
494
+ }
510
495
  }
511
496
  async processAttachment(attachment, stream) {
512
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
513
- var _a, _b;
514
- const { httpStream, delay, error } = await stream({
515
- item: attachment,
516
- event: this.event,
517
- });
518
- if (error) {
519
- return { error };
497
+ var _a, _b;
498
+ const { httpStream, delay, error } = await stream({
499
+ item: attachment,
500
+ event: this.event,
501
+ });
502
+ if (error) {
503
+ return { error };
504
+ }
505
+ else if (delay) {
506
+ return { delay };
507
+ }
508
+ if (httpStream) {
509
+ const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
510
+ // Get upload URL
511
+ const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType);
512
+ if (!preparedArtifact) {
513
+ console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment.`);
514
+ this.destroyHttpStream(httpStream);
515
+ return;
520
516
  }
521
- else if (delay) {
522
- return { delay };
517
+ if (this.isTimeout) {
518
+ this.destroyHttpStream(httpStream);
519
+ return;
523
520
  }
524
- if (httpStream) {
525
- const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
526
- // Get upload URL
527
- const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType);
528
- if (!preparedArtifact) {
529
- console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment.`);
530
- this.destroyHttpStream(httpStream);
531
- return;
532
- }
533
- if (this.isTimeout) {
534
- this.destroyHttpStream(httpStream);
535
- return;
536
- }
537
- // Stream attachment
538
- const uploadedArtifact = await this.uploader.streamArtifact(preparedArtifact, httpStream);
539
- if (!uploadedArtifact) {
540
- console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment.`);
541
- this.destroyHttpStream(httpStream);
542
- return;
543
- }
544
- // Confirm attachment upload
545
- const confirmArtifactUploadResponse = await this.uploader.confirmArtifactUpload(preparedArtifact.artifact_id);
546
- if (!confirmArtifactUploadResponse) {
547
- console.warn('Error while confirming upload for attachment ID ' + attachment.id);
548
- return;
549
- }
550
- const ssorAttachment = {
551
- id: {
552
- devrev: preparedArtifact.artifact_id,
553
- external: attachment.id,
554
- },
555
- parent_id: {
556
- external: attachment.parent_id,
557
- },
521
+ // Stream attachment
522
+ const uploadedArtifact = await this.uploader.streamArtifact(preparedArtifact, httpStream);
523
+ if (!uploadedArtifact) {
524
+ console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment.`);
525
+ this.destroyHttpStream(httpStream);
526
+ return;
527
+ }
528
+ // Confirm attachment upload
529
+ const confirmArtifactUploadResponse = await this.uploader.confirmArtifactUpload(preparedArtifact.artifact_id);
530
+ if (!confirmArtifactUploadResponse) {
531
+ console.warn('Error while confirming upload for attachment ID ' + attachment.id);
532
+ return;
533
+ }
534
+ const ssorAttachment = {
535
+ id: {
536
+ devrev: preparedArtifact.artifact_id,
537
+ external: attachment.id,
538
+ },
539
+ parent_id: {
540
+ external: attachment.parent_id,
541
+ },
542
+ };
543
+ if (attachment.author_id) {
544
+ ssorAttachment.actor_id = {
545
+ external: attachment.author_id,
558
546
  };
559
- if (attachment.author_id) {
560
- ssorAttachment.actor_id = {
561
- external: attachment.author_id,
562
- };
563
- }
564
- // This will set inline flag in ssor_attachment only if it is explicity
565
- // set in the attachment object.
566
- if (attachment.inline === true) {
567
- ssorAttachment.inline = true;
568
- }
569
- else if (attachment.inline === false) {
570
- ssorAttachment.inline = false;
571
- }
572
- await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
573
547
  }
574
- return;
575
- });
548
+ // This will set inline flag in ssor_attachment only if it is explicity
549
+ // set in the attachment object.
550
+ if (attachment.inline === true) {
551
+ ssorAttachment.inline = true;
552
+ }
553
+ else if (attachment.inline === false) {
554
+ ssorAttachment.inline = false;
555
+ }
556
+ await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
557
+ }
558
+ return;
576
559
  }
577
560
  /**
578
561
  * Destroys a stream to prevent memory leaks.
@@ -594,50 +577,48 @@ class WorkerAdapter {
594
577
  }
595
578
  }
596
579
  async loadAttachment({ item, create, }) {
597
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
598
- // Create item
599
- const { id, delay, error } = await create({
600
- item,
601
- mappers: this._mappers,
602
- event: this.event,
603
- });
604
- if (delay) {
605
- return {
606
- rateLimit: {
607
- delay,
608
- },
609
- };
610
- }
611
- else if (id) {
612
- try {
613
- const syncMapperRecordCreateResponse = await this._mappers.create({
614
- sync_unit: this.event.payload.event_context.sync_unit,
615
- external_ids: [id],
616
- targets: [item.reference_id],
617
- status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
618
- });
619
- console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
620
- }
621
- catch (error) {
622
- console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
623
- }
624
- return {
625
- report: {
626
- item_type: 'attachment',
627
- [loading_1.ActionType.CREATED]: 1,
628
- },
629
- };
580
+ // Create item
581
+ const { id, delay, error } = await create({
582
+ item,
583
+ mappers: this._mappers,
584
+ event: this.event,
585
+ });
586
+ if (delay) {
587
+ return {
588
+ rateLimit: {
589
+ delay,
590
+ },
591
+ };
592
+ }
593
+ else if (id) {
594
+ try {
595
+ const syncMapperRecordCreateResponse = await this._mappers.create({
596
+ sync_unit: this.event.payload.event_context.sync_unit,
597
+ external_ids: [id],
598
+ targets: [item.reference_id],
599
+ status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
600
+ });
601
+ console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
630
602
  }
631
- else {
632
- console.warn('Failed to create attachment in external system', error);
633
- return {
634
- report: {
635
- item_type: 'attachment',
636
- [loading_1.ActionType.FAILED]: 1,
637
- },
638
- };
603
+ catch (error) {
604
+ console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
639
605
  }
640
- });
606
+ return {
607
+ report: {
608
+ item_type: 'attachment',
609
+ [loading_1.ActionType.CREATED]: 1,
610
+ },
611
+ };
612
+ }
613
+ else {
614
+ console.warn('Failed to create attachment in external system', error);
615
+ return {
616
+ report: {
617
+ item_type: 'attachment',
618
+ [loading_1.ActionType.FAILED]: 1,
619
+ },
620
+ };
621
+ }
641
622
  }
642
623
  /**
643
624
  * Streams the attachments to the DevRev platform.
@@ -648,88 +629,86 @@ class WorkerAdapter {
648
629
  */
649
630
  async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
650
631
  }) {
651
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
652
- var _a, _b;
653
- if (batchSize <= 0) {
654
- console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
655
- batchSize = 1;
632
+ var _a, _b;
633
+ if (batchSize <= 0) {
634
+ console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
635
+ batchSize = 1;
636
+ }
637
+ if (batchSize > 50) {
638
+ console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
639
+ batchSize = 50;
640
+ }
641
+ const repos = [
642
+ {
643
+ itemType: 'ssor_attachment',
644
+ },
645
+ ];
646
+ this.initializeRepos(repos);
647
+ const attachmentsMetadata = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
648
+ // If there are no attachments metadata artifact IDs in state, finish here
649
+ if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
650
+ console.log(`No attachments metadata artifact IDs found in state.`);
651
+ return;
652
+ }
653
+ else {
654
+ console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
655
+ }
656
+ // Loop through the attachments metadata artifact IDs
657
+ while (attachmentsMetadata.artifactIds.length > 0) {
658
+ const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
659
+ console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
660
+ const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
661
+ artifact: attachmentsMetadataArtifactId,
662
+ });
663
+ if (error) {
664
+ console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
665
+ return { error };
656
666
  }
657
- if (batchSize > 50) {
658
- console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
659
- batchSize = 50;
667
+ if (!attachments || attachments.length === 0) {
668
+ console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
669
+ // Remove empty artifact and reset lastProcessed
670
+ attachmentsMetadata.artifactIds.shift();
671
+ attachmentsMetadata.lastProcessed = 0;
672
+ continue;
660
673
  }
661
- const repos = [
662
- {
663
- itemType: 'ssor_attachment',
664
- },
665
- ];
666
- this.initializeRepos(repos);
667
- const attachmentsMetadata = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
668
- // If there are no attachments metadata artifact IDs in state, finish here
669
- if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
670
- console.log(`No attachments metadata artifact IDs found in state.`);
671
- return;
674
+ console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
675
+ let response;
676
+ if (processors) {
677
+ console.log(`Using custom processors for attachments.`);
678
+ const reducer = processors.reducer;
679
+ const iterator = processors.iterator;
680
+ const reducedAttachments = reducer({
681
+ attachments,
682
+ adapter: this,
683
+ batchSize,
684
+ });
685
+ response = await iterator({
686
+ reducedAttachments,
687
+ adapter: this,
688
+ stream,
689
+ });
672
690
  }
673
691
  else {
674
- console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
675
- }
676
- // Loop through the attachments metadata artifact IDs
677
- while (attachmentsMetadata.artifactIds.length > 0) {
678
- const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
679
- console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
680
- const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
681
- artifact: attachmentsMetadataArtifactId,
692
+ console.log(`Using attachments streaming pool for attachments streaming.`);
693
+ const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
694
+ adapter: this,
695
+ attachments,
696
+ batchSize,
697
+ stream,
682
698
  });
683
- if (error) {
684
- console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
685
- return { error };
686
- }
687
- if (!attachments || attachments.length === 0) {
688
- console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
689
- // Remove empty artifact and reset lastProcessed
690
- attachmentsMetadata.artifactIds.shift();
691
- attachmentsMetadata.lastProcessed = 0;
692
- continue;
693
- }
694
- console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
695
- let response;
696
- if (processors) {
697
- console.log(`Using custom processors for attachments.`);
698
- const reducer = processors.reducer;
699
- const iterator = processors.iterator;
700
- const reducedAttachments = reducer({
701
- attachments,
702
- adapter: this,
703
- batchSize,
704
- });
705
- response = await iterator({
706
- reducedAttachments,
707
- adapter: this,
708
- stream,
709
- });
710
- }
711
- else {
712
- console.log(`Using attachments streaming pool for attachments streaming.`);
713
- const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
714
- adapter: this,
715
- attachments,
716
- batchSize,
717
- stream,
718
- });
719
- response = await attachmentsPool.streamAll();
720
- }
721
- if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
722
- return response;
723
- }
724
- console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
725
- attachmentsMetadata.artifactIds.shift();
726
- attachmentsMetadata.lastProcessed = 0;
727
- if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
728
- attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
729
- }
699
+ response = await attachmentsPool.streamAll();
730
700
  }
731
- return;
732
- });
701
+ if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
702
+ return response;
703
+ }
704
+ console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
705
+ attachmentsMetadata.artifactIds.shift();
706
+ attachmentsMetadata.lastProcessed = 0;
707
+ if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
708
+ attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
709
+ }
710
+ }
711
+ return;
733
712
  }
734
713
  }
735
714
  exports.WorkerAdapter = WorkerAdapter;