@devrev/airsync-sdk 2.0.0-beta.0 → 2.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +1802 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -0
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.js +44 -4
  6. package/dist/attachments-streaming/attachments-streaming-pool.test.js +259 -10
  7. package/dist/common/constants.d.ts +16 -5
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +20 -6
  10. package/dist/common/control-protocol.d.ts +2 -2
  11. package/dist/common/control-protocol.d.ts.map +1 -1
  12. package/dist/common/control-protocol.js +1 -5
  13. package/dist/common/helpers.d.ts +8 -0
  14. package/dist/common/helpers.d.ts.map +1 -1
  15. package/dist/common/helpers.js +15 -0
  16. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  17. package/dist/common/install-initial-domain-mapping.test.js +7 -3
  18. package/dist/common/test-utils.d.ts +19 -0
  19. package/dist/common/test-utils.d.ts.map +1 -0
  20. package/dist/common/test-utils.js +101 -0
  21. package/dist/common/time-value-resolver.d.ts +47 -0
  22. package/dist/common/time-value-resolver.d.ts.map +1 -0
  23. package/dist/common/time-value-resolver.js +141 -0
  24. package/dist/common/time-value-resolver.test.d.ts +2 -0
  25. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  26. package/dist/common/time-value-resolver.test.js +268 -0
  27. package/dist/http/axios-client-internal.d.ts.map +1 -1
  28. package/dist/http/axios-client-internal.js +7 -0
  29. package/dist/http/axios-client-internal.test.js +27 -6
  30. package/dist/http/axios-client.d.ts.map +1 -1
  31. package/dist/http/axios-client.js +6 -3
  32. package/dist/http/index.d.ts +0 -1
  33. package/dist/http/index.d.ts.map +1 -1
  34. package/dist/http/index.js +0 -1
  35. package/dist/index.d.ts +9 -8
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/index.js +12 -10
  38. package/dist/logger/logger.d.ts +1 -17
  39. package/dist/logger/logger.d.ts.map +1 -1
  40. package/dist/logger/logger.interfaces.d.ts +2 -2
  41. package/dist/logger/logger.js +35 -27
  42. package/dist/logger/logger.test.js +66 -19
  43. package/dist/mappers/mappers.interface.d.ts +5 -5
  44. package/dist/mappers/mappers.interface.js +3 -3
  45. package/dist/mappers/mappers.test.js +8 -10
  46. package/dist/mock-server/mock-server.d.ts +47 -0
  47. package/dist/mock-server/mock-server.d.ts.map +1 -0
  48. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  49. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  50. package/dist/mock-server/mock-server.interfaces.js +4 -0
  51. package/dist/mock-server/mock-server.js +278 -0
  52. package/dist/multithreading/create-worker.test.js +36 -17
  53. package/dist/multithreading/process-task.d.ts.map +1 -1
  54. package/dist/multithreading/process-task.js +46 -44
  55. package/dist/multithreading/process-task.test.d.ts +2 -0
  56. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  57. package/dist/multithreading/process-task.test.js +146 -0
  58. package/dist/multithreading/spawn/spawn.d.ts +4 -3
  59. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  60. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  61. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  62. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  63. package/dist/multithreading/spawn/spawn.js +41 -31
  64. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  65. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  66. package/dist/multithreading/spawn/spawn.test.js +232 -0
  67. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
  68. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
  69. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  70. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
  71. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
  72. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
  73. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
  74. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
  75. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
  76. package/dist/multithreading/worker-adapter/worker-adapter.js +233 -134
  77. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
  78. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
  79. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
  80. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
  81. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
  82. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
  83. package/dist/repo/repo.d.ts.map +1 -1
  84. package/dist/repo/repo.interfaces.d.ts +4 -2
  85. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  86. package/dist/repo/repo.js +4 -3
  87. package/dist/repo/repo.test.js +60 -7
  88. package/dist/state/state.d.ts +7 -1
  89. package/dist/state/state.d.ts.map +1 -1
  90. package/dist/state/state.extract-window.test.d.ts +2 -0
  91. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  92. package/dist/state/state.extract-window.test.js +163 -0
  93. package/dist/state/state.interfaces.d.ts +25 -8
  94. package/dist/state/state.interfaces.d.ts.map +1 -1
  95. package/dist/state/state.interfaces.js +4 -2
  96. package/dist/state/state.js +111 -12
  97. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  98. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  99. package/dist/state/state.pending-boundaries.test.js +189 -0
  100. package/dist/state/state.post-state.test.d.ts +2 -0
  101. package/dist/state/state.post-state.test.d.ts.map +1 -0
  102. package/dist/state/state.post-state.test.js +77 -0
  103. package/dist/state/state.test.js +117 -73
  104. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  105. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  106. package/dist/state/state.time-value-resolution.test.js +175 -0
  107. package/dist/tsdoc-metadata.json +11 -0
  108. package/dist/types/common.d.ts +0 -25
  109. package/dist/types/common.d.ts.map +1 -1
  110. package/dist/types/common.js +1 -11
  111. package/dist/types/external-domain-metadata.d.ts +338 -0
  112. package/dist/types/external-domain-metadata.d.ts.map +1 -0
  113. package/dist/types/extraction.d.ts +89 -218
  114. package/dist/types/extraction.d.ts.map +1 -1
  115. package/dist/types/extraction.js +56 -124
  116. package/dist/types/extraction.test.js +69 -43
  117. package/dist/types/index.d.ts +4 -2
  118. package/dist/types/index.d.ts.map +1 -1
  119. package/dist/types/index.js +6 -3
  120. package/dist/types/loading.d.ts +4 -24
  121. package/dist/types/loading.d.ts.map +1 -1
  122. package/dist/types/loading.js +1 -22
  123. package/dist/types/workers.d.ts +31 -14
  124. package/dist/types/workers.d.ts.map +1 -1
  125. package/dist/types/workers.js +1 -0
  126. package/dist/uploader/uploader.d.ts +1 -0
  127. package/dist/uploader/uploader.d.ts.map +1 -1
  128. package/dist/uploader/uploader.helpers.test.js +0 -11
  129. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  130. package/dist/uploader/uploader.js +53 -20
  131. package/dist/uploader/uploader.test.js +22 -10
  132. package/package.json +12 -11
  133. package/dist/common/event-type-translation.d.ts +0 -24
  134. package/dist/common/event-type-translation.d.ts.map +0 -1
  135. package/dist/common/event-type-translation.js +0 -117
  136. package/dist/deprecated/adapter/index.d.ts +0 -62
  137. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  138. package/dist/deprecated/adapter/index.js +0 -151
  139. package/dist/deprecated/common/helpers.d.ts +0 -7
  140. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  141. package/dist/deprecated/common/helpers.js +0 -47
  142. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  143. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  144. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  145. package/dist/deprecated/demo-extractor/index.js +0 -161
  146. package/dist/deprecated/http/client.d.ts +0 -22
  147. package/dist/deprecated/http/client.d.ts.map +0 -1
  148. package/dist/deprecated/http/client.js +0 -161
  149. package/dist/deprecated/uploader/index.d.ts +0 -35
  150. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  151. package/dist/deprecated/uploader/index.js +0 -161
  152. package/dist/http/types.d.ts +0 -17
  153. package/dist/http/types.d.ts.map +0 -1
  154. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  155. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  156. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
  157. /package/dist/{http/types.js → types/external-domain-metadata.js} +0 -0
@@ -20,7 +20,7 @@ const extraction_1 = require("../../types/extraction");
20
20
  const loading_1 = require("../../types/loading");
21
21
  const workers_1 = require("../../types/workers");
22
22
  const uploader_1 = require("../../uploader/uploader");
23
- const event_type_translation_1 = require("../../common/event-type-translation");
23
+ const helpers_1 = require("../../common/helpers");
24
24
  function createWorkerAdapter({ event, adapterState, options, }) {
25
25
  return new WorkerAdapter({
26
26
  event,
@@ -29,7 +29,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
29
29
  });
30
30
  }
31
31
  /**
32
- * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
32
+ * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
33
33
  * as parameter in processTask and onTimeout functions. The class provides
34
34
  * utilities to emit control events to the platform, update the state of the connector,
35
35
  * and upload artifacts to the platform.
@@ -46,6 +46,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
46
46
  class WorkerAdapter {
47
47
  constructor({ event, adapterState, options, }) {
48
48
  this.repos = [];
49
+ this.currentEventDataLength = 0;
49
50
  this.event = event;
50
51
  this.options = options;
51
52
  this.adapterState = adapterState;
@@ -68,9 +69,7 @@ class WorkerAdapter {
68
69
  return this.adapterState.state;
69
70
  }
70
71
  set state(value) {
71
- if (!this.isTimeout) {
72
- this.adapterState.state = value;
73
- }
72
+ this.adapterState.state = value;
74
73
  }
75
74
  get reports() {
76
75
  return this.loaderReports;
@@ -81,17 +80,38 @@ class WorkerAdapter {
81
80
  get mappers() {
82
81
  return this._mappers;
83
82
  }
83
+ get extractionScope() {
84
+ return this.adapterState.extractionScope;
85
+ }
86
+ /**
87
+ * Returns whether the given item type should be extracted.
88
+ * Defaults to true if the scope is empty or the item type is not listed.
89
+ */
90
+ shouldExtract(itemType) {
91
+ const scope = this.extractionScope;
92
+ if (Object.keys(scope).length === 0)
93
+ return true;
94
+ if (!(itemType in scope))
95
+ return true;
96
+ return scope[itemType].extract;
97
+ }
84
98
  initializeRepos(repos) {
85
99
  this.repos = repos.map((repo) => {
86
- const shouldNormalize = repo.itemType !== constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
87
- repo.itemType !== constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT;
100
+ const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
101
+ repo.itemType !== constants_1.SSOR_ATTACHMENT;
88
102
  return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
89
103
  var _a;
90
104
  // We need to store artifacts ids in state for later use when streaming attachments
91
- if (repo.itemType === constants_1.AIRDROP_DEFAULT_ITEM_TYPES.ATTACHMENTS) {
105
+ if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
92
106
  (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
93
107
  }
94
- }, options: this.options }));
108
+ // Calculate size of the entire artifact object that goes in the SQS message
109
+ this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
110
+ if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
111
+ !this.isTimeout) {
112
+ this.isTimeout = true;
113
+ }
114
+ }, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
95
115
  });
96
116
  }
97
117
  getRepo(itemType) {
@@ -125,29 +145,42 @@ class WorkerAdapter {
125
145
  */
126
146
  async emit(newEventType, data) {
127
147
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
128
- newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
148
+ var _a;
129
149
  if (this.hasWorkerEmitted) {
130
150
  console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
131
151
  return;
132
152
  }
133
- // We want to upload all the repos before emitting the event, except for the external sync units done event
134
- if (newEventType !== extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone) {
135
- console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
136
- try {
137
- await this.uploadAllRepos();
138
- }
139
- catch (error) {
140
- console.error('Error while uploading repos', error);
141
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
142
- this.hasWorkerEmitted = true;
143
- return;
144
- }
153
+ // Upload all repos before emitting the event
154
+ console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
155
+ try {
156
+ await this.uploadAllRepos();
157
+ }
158
+ catch (error) {
159
+ console.error('Error while uploading repos', error);
160
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
161
+ this.hasWorkerEmitted = true;
162
+ return;
145
163
  }
146
- // If the extraction is done, we want to save the timestamp of the last successful sync
147
164
  if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
148
- console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
149
- this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
150
- this.state.lastSyncStarted = '';
165
+ // Clear pending extraction boundaries now that the cycle is complete
166
+ this.state.pendingWorkersOldest = '';
167
+ this.state.pendingWorkersNewest = '';
168
+ // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
169
+ // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
170
+ const extractionStart = this.event.payload.event_context.extract_from;
171
+ const extractionEnd = this.event.payload.event_context.extract_to;
172
+ if (extractionStart &&
173
+ (!this.state.workersOldest ||
174
+ extractionStart < this.state.workersOldest)) {
175
+ console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
176
+ this.state.workersOldest = extractionStart;
177
+ }
178
+ if (extractionEnd &&
179
+ (!this.state.workersNewest ||
180
+ extractionEnd > this.state.workersNewest)) {
181
+ console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
182
+ this.state.workersNewest = extractionEnd;
183
+ }
151
184
  }
152
185
  // We want to save the state every time we emit an event, except for the start and delete events
153
186
  if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
@@ -163,11 +196,17 @@ class WorkerAdapter {
163
196
  }
164
197
  }
165
198
  try {
199
+ // Always prune error messages to make them shorter before emit
200
+ if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
201
+ data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
202
+ }
203
+ const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
204
+ const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
166
205
  await (0, control_protocol_1.emit)({
167
206
  eventType: newEventType,
168
207
  event: this.event,
169
- data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
170
- ? { artifacts: this.artifacts }
208
+ data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
209
+ ? { reports: this.reports, processed_files: this.processedFiles }
171
210
  : {})),
172
211
  });
173
212
  const message = {
@@ -194,9 +233,6 @@ class WorkerAdapter {
194
233
  }
195
234
  }
196
235
  }
197
- handleTimeout() {
198
- this.isTimeout = true;
199
- }
200
236
  async loadItemTypes({ itemTypesToLoad, }) {
201
237
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
202
238
  var _a;
@@ -225,57 +261,74 @@ class WorkerAdapter {
225
261
  };
226
262
  }
227
263
  console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
228
- outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
229
- .filesToLoad) {
230
- const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
231
- if (!itemTypeToLoad) {
232
- console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
233
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
234
- error: {
235
- message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
236
- },
237
- });
238
- break;
239
- }
240
- if (!fileToLoad.completed) {
241
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
242
- artifactId: fileToLoad.id,
243
- isGzipped: true,
244
- });
245
- if (transformerFileError) {
246
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
264
+ try {
265
+ outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
266
+ .filesToLoad) {
267
+ const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
268
+ if (!itemTypeToLoad) {
269
+ console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
247
270
  await this.emit(loading_1.LoaderEventType.DataLoadingError, {
248
271
  error: {
249
- message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
272
+ message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
250
273
  },
251
274
  });
275
+ break;
252
276
  }
253
- const transformerFile = response;
254
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
255
- const { report, rateLimit } = await this.loadItem({
256
- item: transformerFile[i],
257
- itemTypeToLoad,
277
+ if (!fileToLoad.completed) {
278
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
279
+ artifactId: fileToLoad.id,
280
+ isGzipped: true,
258
281
  });
259
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
260
- await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
261
- delay: rateLimit.delay,
262
- reports: this.reports,
263
- processed_files: this.processedFiles,
282
+ if (transformerFileError) {
283
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
284
+ await this.emit(loading_1.LoaderEventType.DataLoadingError, {
285
+ error: {
286
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
287
+ },
264
288
  });
265
289
  break outerloop;
266
290
  }
267
- if (report) {
268
- (0, worker_adapter_helpers_1.addReportToLoaderReport)({
269
- loaderReports: this.loaderReports,
270
- report,
291
+ const transformerFile = response;
292
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
293
+ if (this.isTimeout) {
294
+ console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
295
+ await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
296
+ process.exit(0);
297
+ }
298
+ const { report, rateLimit } = await this.loadItem({
299
+ item: transformerFile[i],
300
+ itemTypeToLoad,
271
301
  });
272
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
302
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
303
+ await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
304
+ delay: rateLimit.delay,
305
+ reports: this.reports,
306
+ processed_files: this.processedFiles,
307
+ });
308
+ break outerloop;
309
+ }
310
+ if (report) {
311
+ (0, worker_adapter_helpers_1.addReportToLoaderReport)({
312
+ loaderReports: this.loaderReports,
313
+ report,
314
+ });
315
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
316
+ }
273
317
  }
318
+ fileToLoad.completed = true;
319
+ this._processedFiles.push(fileToLoad.id);
274
320
  }
275
- fileToLoad.completed = true;
276
- this._processedFiles.push(fileToLoad.id);
277
321
  }
278
322
  }
323
+ catch (error) {
324
+ console.error('Error during data loading.', (0, logger_1.serializeError)(error));
325
+ await this.emit(loading_1.LoaderEventType.DataLoadingError, {
326
+ error: {
327
+ message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
328
+ },
329
+ });
330
+ process.exit(1);
331
+ }
279
332
  return {
280
333
  reports: this.reports,
281
334
  processed_files: this.processedFiles,
@@ -322,42 +375,63 @@ class WorkerAdapter {
322
375
  };
323
376
  }
324
377
  const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
325
- outerloop: for (const fileToLoad of filesToLoad) {
326
- if (!fileToLoad.completed) {
327
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
328
- artifactId: fileToLoad.id,
329
- isGzipped: true,
330
- });
331
- const transformerFile = response;
332
- if (transformerFileError) {
333
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
334
- break outerloop;
335
- }
336
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
337
- const { report, rateLimit } = await this.loadAttachment({
338
- item: transformerFile[i],
339
- create,
378
+ try {
379
+ outerloop: for (const fileToLoad of filesToLoad) {
380
+ if (!fileToLoad.completed) {
381
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
382
+ artifactId: fileToLoad.id,
383
+ isGzipped: true,
340
384
  });
341
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
342
- await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
343
- delay: rateLimit.delay,
344
- reports: this.reports,
345
- processed_files: this.processedFiles,
385
+ const transformerFile = response;
386
+ if (transformerFileError) {
387
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
388
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
389
+ error: {
390
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
391
+ },
346
392
  });
347
393
  break outerloop;
348
394
  }
349
- if (report) {
350
- (0, worker_adapter_helpers_1.addReportToLoaderReport)({
351
- loaderReports: this.loaderReports,
352
- report,
395
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
396
+ if (this.isTimeout) {
397
+ console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
398
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
399
+ process.exit(0);
400
+ }
401
+ const { report, rateLimit } = await this.loadAttachment({
402
+ item: transformerFile[i],
403
+ create,
353
404
  });
354
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
405
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
406
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingDelayed, {
407
+ delay: rateLimit.delay,
408
+ reports: this.reports,
409
+ processed_files: this.processedFiles,
410
+ });
411
+ break outerloop;
412
+ }
413
+ if (report) {
414
+ (0, worker_adapter_helpers_1.addReportToLoaderReport)({
415
+ loaderReports: this.loaderReports,
416
+ report,
417
+ });
418
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
419
+ }
355
420
  }
421
+ fileToLoad.completed = true;
422
+ this._processedFiles.push(fileToLoad.id);
356
423
  }
357
- fileToLoad.completed = true;
358
- this._processedFiles.push(fileToLoad.id);
359
424
  }
360
425
  }
426
+ catch (error) {
427
+ console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
428
+ await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
429
+ error: {
430
+ message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
431
+ },
432
+ });
433
+ process.exit(1);
434
+ }
361
435
  return {
362
436
  reports: this.reports,
363
437
  processed_files: this.processedFiles,
@@ -383,10 +457,12 @@ class WorkerAdapter {
383
457
  };
384
458
  }
385
459
  // Update item in external system
386
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
387
- item,
388
- mappers: this._mappers,
389
- event: this.event,
460
+ const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
461
+ return await itemTypeToLoad.update({
462
+ item,
463
+ mappers: this._mappers,
464
+ event: this.event,
465
+ });
390
466
  });
391
467
  if (id) {
392
468
  try {
@@ -444,10 +520,12 @@ class WorkerAdapter {
444
520
  if (axios_1.default.isAxiosError(error)) {
445
521
  if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
446
522
  // Create item in external system if mapper record not found
447
- const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
448
- item,
449
- mappers: this._mappers,
450
- event: this.event,
523
+ const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
524
+ return await itemTypeToLoad.create({
525
+ item,
526
+ mappers: this._mappers,
527
+ event: this.event,
528
+ });
451
529
  });
452
530
  if (id) {
453
531
  // Create mapper
@@ -515,11 +593,11 @@ class WorkerAdapter {
515
593
  }
516
594
  async processAttachment(attachment, stream) {
517
595
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
518
- var _a;
519
- const { httpStream, delay, error } = await stream({
596
+ var _a, _b, _c;
597
+ const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
520
598
  item: attachment,
521
599
  event: this.event,
522
- });
600
+ }));
523
601
  if (error) {
524
602
  return { error };
525
603
  }
@@ -527,37 +605,44 @@ class WorkerAdapter {
527
605
  return { delay };
528
606
  }
529
607
  if (httpStream) {
530
- const fileType = httpStream.headers['content-type'] || 'application/octet-stream';
531
- const fileSize = httpStream.headers['content-length']
532
- ? parseInt(httpStream.headers['content-length'])
533
- : undefined;
608
+ const fileType = attachment.content_type ||
609
+ ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
610
+ 'application/octet-stream';
611
+ const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
612
+ const fileSize = contentLength ? parseInt(contentLength) : undefined;
534
613
  // Get upload URL
535
614
  const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
536
615
  if (artifactUrlError) {
537
- console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
538
- (0, logger_1.serializeError)(artifactUrlError));
539
- this.destroyHttpStream(httpStream);
540
- return;
541
- }
542
- if (this.isTimeout) {
543
616
  this.destroyHttpStream(httpStream);
544
- return;
617
+ return {
618
+ error: {
619
+ message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
620
+ fileSize: fileSize,
621
+ },
622
+ };
545
623
  }
546
624
  // Stream attachment
547
625
  const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
548
626
  if (uploadedArtifactError) {
549
- console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
550
- (0, logger_1.serializeError)(uploadedArtifactError));
551
627
  this.destroyHttpStream(httpStream);
552
- return;
628
+ return {
629
+ error: {
630
+ message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
631
+ (0, logger_1.serializeError)(uploadedArtifactError),
632
+ fileSize: fileSize,
633
+ },
634
+ };
553
635
  }
554
636
  // Confirm attachment upload
555
637
  const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
556
638
  if (confirmArtifactUploadError) {
557
- console.warn('Error while confirming upload for attachment ID ' +
558
- attachment.id +
559
- '.', confirmArtifactUploadError);
560
- return;
639
+ return {
640
+ error: {
641
+ message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
642
+ (0, logger_1.serializeError)(confirmArtifactUploadError),
643
+ fileSize: fileSize,
644
+ },
645
+ };
561
646
  }
562
647
  const ssorAttachment = {
563
648
  id: {
@@ -581,9 +666,14 @@ class WorkerAdapter {
581
666
  else if (attachment.inline === false) {
582
667
  ssorAttachment.inline = false;
583
668
  }
584
- await ((_a = this.getRepo('ssor_attachment')) === null || _a === void 0 ? void 0 : _a.push([ssorAttachment]));
669
+ await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
670
+ return;
585
671
  }
586
- return;
672
+ return {
673
+ error: {
674
+ message: `Error while opening attachment stream. Skipping attachment.`,
675
+ },
676
+ };
587
677
  });
588
678
  }
589
679
  /**
@@ -608,11 +698,11 @@ class WorkerAdapter {
608
698
  async loadAttachment({ item, create, }) {
609
699
  return (0, logger_context_1.runWithSdkLogContext)(async () => {
610
700
  // Create item
611
- const { id, delay, error } = await create({
701
+ const { id, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => create({
612
702
  item,
613
703
  mappers: this._mappers,
614
704
  event: this.event,
615
- });
705
+ }));
616
706
  if (delay) {
617
707
  return {
618
708
  rateLimit: {
@@ -709,15 +799,17 @@ class WorkerAdapter {
709
799
  console.log(`Using custom processors for attachments.`);
710
800
  const reducer = processors.reducer;
711
801
  const iterator = processors.iterator;
712
- const reducedAttachments = reducer({
802
+ const reducedAttachments = (0, logger_context_1.runWithUserLogContext)(() => reducer({
713
803
  attachments,
714
804
  adapter: this,
715
805
  batchSize,
716
- });
717
- response = await iterator({
718
- reducedAttachments,
719
- adapter: this,
720
- stream,
806
+ }));
807
+ response = await (0, logger_context_1.runWithUserLogContext)(async () => {
808
+ return await iterator({
809
+ reducedAttachments,
810
+ adapter: this,
811
+ stream,
812
+ });
721
813
  });
722
814
  }
723
815
  else {
@@ -733,6 +825,13 @@ class WorkerAdapter {
733
825
  if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
734
826
  return response;
735
827
  }
828
+ // On timeout, emit progress and exit to allow continuation.
829
+ if (this.isTimeout) {
830
+ console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.`);
831
+ await this.emit(extraction_1.ExtractorEventType.AttachmentExtractionProgress);
832
+ process.exit(0);
833
+ return;
834
+ }
736
835
  console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
737
836
  attachmentsMetadata.artifactIds.shift();
738
837
  attachmentsMetadata.lastProcessed = 0;
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker-adapter.loading.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-adapter.loading.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.loading.test.ts"],"names":[],"mappings":""}