@devrev/ts-adaas 1.12.3-beta.2 → 1.12.3-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.js +11 -11
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +5 -1
- package/dist/common/event-type-translation.d.ts +24 -0
- package/dist/common/event-type-translation.d.ts.map +1 -0
- package/dist/common/event-type-translation.js +117 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +25 -7
- package/dist/deprecated/adapter/index.d.ts.map +1 -1
- package/dist/deprecated/adapter/index.js +2 -0
- package/dist/logger/logger.d.ts +1 -16
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +3 -4
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +6 -33
- package/dist/logger/logger.test.js +10 -71
- package/dist/state/state.js +2 -2
- package/dist/state/state.test.js +3 -3
- package/dist/tests/backwards-compatibility/backwards-compatibility.test.js +2 -17
- package/dist/tests/timeout-handling/timeout-1.js +2 -2
- package/dist/tests/timeout-handling/timeout-1.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-2.js +3 -3
- package/dist/tests/timeout-handling/timeout-2.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3a.js +2 -2
- package/dist/tests/timeout-handling/timeout-3a.test.js +1 -1
- package/dist/tests/timeout-handling/timeout-3b.js +2 -2
- package/dist/tests/timeout-handling/timeout-3b.test.js +1 -1
- package/dist/types/extraction.d.ts +99 -2
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +102 -2
- package/dist/types/loading.d.ts +21 -1
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +23 -0
- package/dist/types/workers.d.ts +7 -15
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/workers/process-task.d.ts.map +1 -1
- package/dist/workers/process-task.js +35 -37
- package/dist/workers/spawn.d.ts +3 -3
- package/dist/workers/spawn.d.ts.map +1 -1
- package/dist/workers/spawn.js +43 -49
- package/dist/workers/worker-adapter.d.ts.map +1 -1
- package/dist/workers/worker-adapter.js +475 -496
- package/dist/workers/worker-adapter.test.js +0 -7
- package/dist/workers/worker.js +1 -4
- package/package.json +1 -1
- package/dist/logger/logger.context.d.ts +0 -6
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -31
- package/dist/logger/logger.worker-fixture.d.ts +0 -2
- package/dist/logger/logger.worker-fixture.d.ts.map +0 -1
- package/dist/logger/logger.worker-fixture.js +0 -10
- package/dist/workers/default-workers/attachments-deletion.d.ts +0 -2
- package/dist/workers/default-workers/attachments-deletion.d.ts.map +0 -1
- package/dist/workers/default-workers/attachments-deletion.js +0 -13
- package/dist/workers/default-workers/attachments-extraction.d.ts +0 -2
- package/dist/workers/default-workers/attachments-extraction.d.ts.map +0 -1
- package/dist/workers/default-workers/attachments-extraction.js +0 -95
- package/dist/workers/default-workers/data-deletion.d.ts +0 -2
- package/dist/workers/default-workers/data-deletion.d.ts.map +0 -1
- package/dist/workers/default-workers/data-deletion.js +0 -15
- package/dist/workers/default-workers/data-extraction.d.ts +0 -2
- package/dist/workers/default-workers/data-extraction.d.ts.map +0 -1
- package/dist/workers/default-workers/data-extraction.js +0 -100
- package/dist/workers/default-workers/delete-loader-attachment-state.d.ts +0 -2
- package/dist/workers/default-workers/delete-loader-attachment-state.d.ts.map +0 -1
- package/dist/workers/default-workers/delete-loader-attachment-state.js +0 -15
- package/dist/workers/default-workers/delete-loader-state.d.ts +0 -2
- package/dist/workers/default-workers/delete-loader-state.d.ts.map +0 -1
- package/dist/workers/default-workers/delete-loader-state.js +0 -15
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +0 -2
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts.map +0 -1
- package/dist/workers/default-workers/external-sync-units-extraction.js +0 -27
- package/dist/workers/default-workers/load-attachments.d.ts +0 -2
- package/dist/workers/default-workers/load-attachments.d.ts.map +0 -1
- package/dist/workers/default-workers/load-attachments.js +0 -19
- package/dist/workers/default-workers/load-data.d.ts +0 -2
- package/dist/workers/default-workers/load-data.d.ts.map +0 -1
- package/dist/workers/default-workers/load-data.js +0 -18
- package/dist/workers/default-workers/metadata-extraction.d.ts +0 -2
- package/dist/workers/default-workers/metadata-extraction.d.ts.map +0 -1
- package/dist/workers/default-workers/metadata-extraction.js +0 -26
- package/dist/workers/dummy-extractor/data-normalization.d.ts +0 -5
- package/dist/workers/dummy-extractor/data-normalization.d.ts.map +0 -1
- package/dist/workers/dummy-extractor/data-normalization.js +0 -41
- package/dist/workers/dummy-extractor/external_domain_metadata.json +0 -58
|
@@ -12,7 +12,6 @@ const constants_1 = require("../common/constants");
|
|
|
12
12
|
const control_protocol_1 = require("../common/control-protocol");
|
|
13
13
|
const helpers_1 = require("../common/helpers");
|
|
14
14
|
const logger_1 = require("../logger/logger");
|
|
15
|
-
const logger_context_1 = require("../logger/logger.context");
|
|
16
15
|
const mappers_1 = require("../mappers/mappers");
|
|
17
16
|
const mappers_interface_1 = require("../mappers/mappers.interface");
|
|
18
17
|
const repo_1 = require("../repo/repo");
|
|
@@ -94,19 +93,15 @@ class WorkerAdapter {
|
|
|
94
93
|
});
|
|
95
94
|
}
|
|
96
95
|
getRepo(itemType) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
return repo;
|
|
104
|
-
});
|
|
96
|
+
const repo = this.repos.find((repo) => repo.itemType === itemType);
|
|
97
|
+
if (!repo) {
|
|
98
|
+
console.error(`Repo for item type ${itemType} not found.`);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
return repo;
|
|
105
102
|
}
|
|
106
103
|
async postState() {
|
|
107
|
-
|
|
108
|
-
await this.adapterState.postState();
|
|
109
|
-
});
|
|
104
|
+
await this.adapterState.postState();
|
|
110
105
|
}
|
|
111
106
|
get artifacts() {
|
|
112
107
|
return this._artifacts;
|
|
@@ -123,65 +118,63 @@ class WorkerAdapter {
|
|
|
123
118
|
* @param data - The data to be sent with the event
|
|
124
119
|
*/
|
|
125
120
|
async emit(newEventType, data) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
await this.uploadAllRepos();
|
|
136
|
-
}
|
|
137
|
-
catch (error) {
|
|
138
|
-
console.error('Error while uploading repos', error);
|
|
139
|
-
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
140
|
-
this.hasWorkerEmitted = true;
|
|
141
|
-
return;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
145
|
-
if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
|
|
146
|
-
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
|
|
147
|
-
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
148
|
-
this.state.lastSyncStarted = '';
|
|
121
|
+
if (this.hasWorkerEmitted) {
|
|
122
|
+
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
// We want to upload all the repos before emitting the event, except for the external sync units done event
|
|
126
|
+
if (newEventType !== extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone) {
|
|
127
|
+
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
128
|
+
try {
|
|
129
|
+
await this.uploadAllRepos();
|
|
149
130
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
catch (error) {
|
|
157
|
-
console.error('Error while posting state', error);
|
|
158
|
-
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
159
|
-
this.hasWorkerEmitted = true;
|
|
160
|
-
return;
|
|
161
|
-
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
console.error('Error while uploading repos', error);
|
|
133
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
134
|
+
this.hasWorkerEmitted = true;
|
|
135
|
+
return;
|
|
162
136
|
}
|
|
137
|
+
}
|
|
138
|
+
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
139
|
+
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
140
|
+
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
|
|
141
|
+
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
142
|
+
this.state.lastSyncStarted = '';
|
|
143
|
+
}
|
|
144
|
+
// We want to save the state every time we emit an event, except for the start and delete events
|
|
145
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
146
|
+
console.log(`Saving state before emitting event with event type: ${newEventType}.`);
|
|
163
147
|
try {
|
|
164
|
-
await (
|
|
165
|
-
eventType: newEventType,
|
|
166
|
-
event: this.event,
|
|
167
|
-
data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
|
|
168
|
-
? { artifacts: this.artifacts }
|
|
169
|
-
: {})),
|
|
170
|
-
});
|
|
171
|
-
const message = {
|
|
172
|
-
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
|
173
|
-
payload: { eventType: newEventType },
|
|
174
|
-
};
|
|
175
|
-
this.artifacts = [];
|
|
176
|
-
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
|
|
177
|
-
this.hasWorkerEmitted = true;
|
|
148
|
+
await this.adapterState.postState(this.state);
|
|
178
149
|
}
|
|
179
150
|
catch (error) {
|
|
180
|
-
console.error(
|
|
151
|
+
console.error('Error while posting state', error);
|
|
181
152
|
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
182
153
|
this.hasWorkerEmitted = true;
|
|
154
|
+
return;
|
|
183
155
|
}
|
|
184
|
-
}
|
|
156
|
+
}
|
|
157
|
+
try {
|
|
158
|
+
await (0, control_protocol_1.emit)({
|
|
159
|
+
eventType: newEventType,
|
|
160
|
+
event: this.event,
|
|
161
|
+
data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
|
|
162
|
+
? { artifacts: this.artifacts }
|
|
163
|
+
: {})),
|
|
164
|
+
});
|
|
165
|
+
const message = {
|
|
166
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
|
167
|
+
payload: { eventType: newEventType },
|
|
168
|
+
};
|
|
169
|
+
this.artifacts = [];
|
|
170
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
|
|
171
|
+
this.hasWorkerEmitted = true;
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
|
|
175
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
176
|
+
this.hasWorkerEmitted = true;
|
|
177
|
+
}
|
|
185
178
|
}
|
|
186
179
|
async uploadAllRepos() {
|
|
187
180
|
for (const repo of this.repos) {
|
|
@@ -196,383 +189,373 @@ class WorkerAdapter {
|
|
|
196
189
|
this.isTimeout = true;
|
|
197
190
|
}
|
|
198
191
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
console.warn('No item types to load, returning.');
|
|
205
|
-
return {
|
|
206
|
-
reports: this.reports,
|
|
207
|
-
processed_files: this.processedFiles,
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
const filesToLoad = await this.getLoaderBatches({
|
|
211
|
-
supportedItemTypes: itemTypes,
|
|
212
|
-
});
|
|
213
|
-
this.adapterState.state.fromDevRev = {
|
|
214
|
-
filesToLoad,
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
if (!this.adapterState.state.fromDevRev ||
|
|
218
|
-
!this.adapterState.state.fromDevRev.filesToLoad.length) {
|
|
219
|
-
console.warn('No files to load, returning.');
|
|
192
|
+
var _a;
|
|
193
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
|
|
194
|
+
const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
|
|
195
|
+
if (!itemTypes.length) {
|
|
196
|
+
console.warn('No item types to load, returning.');
|
|
220
197
|
return {
|
|
221
198
|
reports: this.reports,
|
|
222
199
|
processed_files: this.processedFiles,
|
|
223
200
|
};
|
|
224
201
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
202
|
+
const filesToLoad = await this.getLoaderBatches({
|
|
203
|
+
supportedItemTypes: itemTypes,
|
|
204
|
+
});
|
|
205
|
+
this.adapterState.state.fromDevRev = {
|
|
206
|
+
filesToLoad,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
if (!this.adapterState.state.fromDevRev ||
|
|
210
|
+
!this.adapterState.state.fromDevRev.filesToLoad.length) {
|
|
211
|
+
console.warn('No files to load, returning.');
|
|
212
|
+
return {
|
|
213
|
+
reports: this.reports,
|
|
214
|
+
processed_files: this.processedFiles,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
218
|
+
outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
|
|
219
|
+
.filesToLoad) {
|
|
220
|
+
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
221
|
+
if (!itemTypeToLoad) {
|
|
222
|
+
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
223
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
224
|
+
error: {
|
|
225
|
+
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
230
|
+
if (!fileToLoad.completed) {
|
|
231
|
+
const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
|
|
232
|
+
artifactId: fileToLoad.id,
|
|
233
|
+
isGzipped: true,
|
|
234
|
+
}));
|
|
235
|
+
if (!transformerFile) {
|
|
236
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
231
237
|
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
232
238
|
error: {
|
|
233
|
-
message: `
|
|
239
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
234
240
|
},
|
|
235
241
|
});
|
|
236
|
-
break;
|
|
237
242
|
}
|
|
238
|
-
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
})
|
|
243
|
-
if (
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
},
|
|
243
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
244
|
+
const { report, rateLimit } = await this.loadItem({
|
|
245
|
+
item: transformerFile[i],
|
|
246
|
+
itemTypeToLoad,
|
|
247
|
+
});
|
|
248
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
249
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
250
|
+
delay: rateLimit.delay,
|
|
251
|
+
reports: this.reports,
|
|
252
|
+
processed_files: this.processedFiles,
|
|
249
253
|
});
|
|
254
|
+
break outerloop;
|
|
250
255
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
256
|
+
if (report) {
|
|
257
|
+
(0, helpers_1.addReportToLoaderReport)({
|
|
258
|
+
loaderReports: this.loaderReports,
|
|
259
|
+
report,
|
|
255
260
|
});
|
|
256
|
-
|
|
257
|
-
await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
|
|
258
|
-
delay: rateLimit.delay,
|
|
259
|
-
reports: this.reports,
|
|
260
|
-
processed_files: this.processedFiles,
|
|
261
|
-
});
|
|
262
|
-
break outerloop;
|
|
263
|
-
}
|
|
264
|
-
if (report) {
|
|
265
|
-
(0, helpers_1.addReportToLoaderReport)({
|
|
266
|
-
loaderReports: this.loaderReports,
|
|
267
|
-
report,
|
|
268
|
-
});
|
|
269
|
-
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
270
|
-
}
|
|
261
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
271
262
|
}
|
|
272
|
-
fileToLoad.completed = true;
|
|
273
|
-
this._processedFiles.push(fileToLoad.id);
|
|
274
263
|
}
|
|
264
|
+
fileToLoad.completed = true;
|
|
265
|
+
this._processedFiles.push(fileToLoad.id);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return {
|
|
269
|
+
reports: this.reports,
|
|
270
|
+
processed_files: this.processedFiles,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
async getLoaderBatches({ supportedItemTypes, }) {
|
|
274
|
+
var _a;
|
|
275
|
+
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
276
|
+
if (statsFileArtifactId) {
|
|
277
|
+
const statsFile = (await this.uploader.getJsonObjectByArtifactId({
|
|
278
|
+
artifactId: statsFileArtifactId,
|
|
279
|
+
}));
|
|
280
|
+
if (!statsFile || statsFile.length === 0) {
|
|
281
|
+
return [];
|
|
275
282
|
}
|
|
283
|
+
const filesToLoad = (0, helpers_1.getFilesToLoad)({
|
|
284
|
+
supportedItemTypes,
|
|
285
|
+
statsFile,
|
|
286
|
+
});
|
|
287
|
+
return filesToLoad;
|
|
288
|
+
}
|
|
289
|
+
return [];
|
|
290
|
+
}
|
|
291
|
+
async loadAttachments({ create, }) {
|
|
292
|
+
var _a, _b;
|
|
293
|
+
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
|
|
294
|
+
this.adapterState.state.fromDevRev = {
|
|
295
|
+
filesToLoad: await this.getLoaderBatches({
|
|
296
|
+
supportedItemTypes: ['attachment'],
|
|
297
|
+
}),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
if (!this.adapterState.state.fromDevRev ||
|
|
301
|
+
((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
|
|
302
|
+
console.log('No files to load, returning.');
|
|
276
303
|
return {
|
|
277
304
|
reports: this.reports,
|
|
278
305
|
processed_files: this.processedFiles,
|
|
279
306
|
};
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
const statsFile = (await this.uploader.getJsonObjectByArtifactId({
|
|
288
|
-
artifactId: statsFileArtifactId,
|
|
307
|
+
}
|
|
308
|
+
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
309
|
+
outerloop: for (const fileToLoad of filesToLoad) {
|
|
310
|
+
if (!fileToLoad.completed) {
|
|
311
|
+
const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
|
|
312
|
+
artifactId: fileToLoad.id,
|
|
313
|
+
isGzipped: true,
|
|
289
314
|
}));
|
|
290
|
-
if (!
|
|
291
|
-
|
|
315
|
+
if (!transformerFile) {
|
|
316
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
317
|
+
break outerloop;
|
|
292
318
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
var _a, _b;
|
|
305
|
-
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
|
|
306
|
-
this.adapterState.state.fromDevRev = {
|
|
307
|
-
filesToLoad: await this.getLoaderBatches({
|
|
308
|
-
supportedItemTypes: ['attachment'],
|
|
309
|
-
}),
|
|
310
|
-
};
|
|
311
|
-
}
|
|
312
|
-
if (!this.adapterState.state.fromDevRev ||
|
|
313
|
-
((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
|
|
314
|
-
console.log('No files to load, returning.');
|
|
315
|
-
return {
|
|
316
|
-
reports: this.reports,
|
|
317
|
-
processed_files: this.processedFiles,
|
|
318
|
-
};
|
|
319
|
-
}
|
|
320
|
-
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
321
|
-
outerloop: for (const fileToLoad of filesToLoad) {
|
|
322
|
-
if (!fileToLoad.completed) {
|
|
323
|
-
const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
|
|
324
|
-
artifactId: fileToLoad.id,
|
|
325
|
-
isGzipped: true,
|
|
326
|
-
}));
|
|
327
|
-
if (!transformerFile) {
|
|
328
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
319
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
320
|
+
const { report, rateLimit } = await this.loadAttachment({
|
|
321
|
+
item: transformerFile[i],
|
|
322
|
+
create,
|
|
323
|
+
});
|
|
324
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
325
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
326
|
+
delay: rateLimit.delay,
|
|
327
|
+
reports: this.reports,
|
|
328
|
+
processed_files: this.processedFiles,
|
|
329
|
+
});
|
|
329
330
|
break outerloop;
|
|
330
331
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
332
|
+
if (report) {
|
|
333
|
+
(0, helpers_1.addReportToLoaderReport)({
|
|
334
|
+
loaderReports: this.loaderReports,
|
|
335
|
+
report,
|
|
335
336
|
});
|
|
336
|
-
|
|
337
|
-
await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
|
|
338
|
-
delay: rateLimit.delay,
|
|
339
|
-
reports: this.reports,
|
|
340
|
-
processed_files: this.processedFiles,
|
|
341
|
-
});
|
|
342
|
-
break outerloop;
|
|
343
|
-
}
|
|
344
|
-
if (report) {
|
|
345
|
-
(0, helpers_1.addReportToLoaderReport)({
|
|
346
|
-
loaderReports: this.loaderReports,
|
|
347
|
-
report,
|
|
348
|
-
});
|
|
349
|
-
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
350
|
-
}
|
|
337
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
351
338
|
}
|
|
352
|
-
fileToLoad.completed = true;
|
|
353
|
-
this._processedFiles.push(fileToLoad.id);
|
|
354
339
|
}
|
|
340
|
+
fileToLoad.completed = true;
|
|
341
|
+
this._processedFiles.push(fileToLoad.id);
|
|
355
342
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
}
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
reports: this.reports,
|
|
346
|
+
processed_files: this.processedFiles,
|
|
347
|
+
};
|
|
361
348
|
}
|
|
362
349
|
async loadItem({ item, itemTypeToLoad, }) {
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
350
|
+
var _a;
|
|
351
|
+
const devrevId = item.id.devrev;
|
|
352
|
+
try {
|
|
353
|
+
const syncMapperRecordResponse = await this._mappers.getByTargetId({
|
|
354
|
+
sync_unit: this.event.payload.event_context.sync_unit,
|
|
355
|
+
target: devrevId,
|
|
356
|
+
});
|
|
357
|
+
const syncMapperRecord = syncMapperRecordResponse.data;
|
|
358
|
+
if (!syncMapperRecord) {
|
|
359
|
+
console.warn('Failed to get sync mapper record from response.');
|
|
360
|
+
return {
|
|
361
|
+
error: {
|
|
362
|
+
message: 'Failed to get sync mapper record from response.',
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
// Update item in external system
|
|
367
|
+
const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
|
|
368
|
+
item,
|
|
369
|
+
mappers: this._mappers,
|
|
370
|
+
event: this.event,
|
|
371
|
+
});
|
|
372
|
+
if (id) {
|
|
373
|
+
try {
|
|
374
|
+
const syncMapperRecordUpdateResponse = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL }, (modifiedDate && {
|
|
375
|
+
external_versions: {
|
|
376
|
+
add: [
|
|
377
|
+
{
|
|
378
|
+
modified_date: modifiedDate,
|
|
379
|
+
recipe_version: 0,
|
|
380
|
+
},
|
|
381
|
+
],
|
|
382
|
+
},
|
|
383
|
+
})), { external_ids: {
|
|
384
|
+
add: [id],
|
|
385
|
+
}, targets: {
|
|
386
|
+
add: [devrevId],
|
|
387
|
+
} }));
|
|
388
|
+
console.log('Successfully updated sync mapper record.', syncMapperRecordUpdateResponse.data);
|
|
389
|
+
}
|
|
390
|
+
catch (error) {
|
|
391
|
+
console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
|
|
374
392
|
return {
|
|
375
393
|
error: {
|
|
376
|
-
message: 'Failed to
|
|
394
|
+
message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
|
|
377
395
|
},
|
|
378
396
|
};
|
|
379
397
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
398
|
+
return {
|
|
399
|
+
report: {
|
|
400
|
+
item_type: itemTypeToLoad.itemType,
|
|
401
|
+
[loading_1.ActionType.UPDATED]: 1,
|
|
402
|
+
},
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
else if (delay) {
|
|
406
|
+
console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
|
|
407
|
+
return {
|
|
408
|
+
rateLimit: {
|
|
409
|
+
delay,
|
|
410
|
+
},
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
console.warn('Failed to update item in external system', error);
|
|
415
|
+
return {
|
|
416
|
+
report: {
|
|
417
|
+
item_type: itemTypeToLoad.itemType,
|
|
418
|
+
[loading_1.ActionType.FAILED]: 1,
|
|
419
|
+
},
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
// TODO: Update mapper (optional)
|
|
423
|
+
}
|
|
424
|
+
catch (error) {
|
|
425
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
426
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
427
|
+
// Create item in external system if mapper record not found
|
|
428
|
+
const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
|
|
429
|
+
item,
|
|
430
|
+
mappers: this._mappers,
|
|
431
|
+
event: this.event,
|
|
432
|
+
});
|
|
433
|
+
if (id) {
|
|
434
|
+
// Create mapper
|
|
435
|
+
try {
|
|
436
|
+
const syncMapperRecordCreateResponse = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
|
|
437
|
+
external_versions: [
|
|
391
438
|
{
|
|
392
439
|
modified_date: modifiedDate,
|
|
393
440
|
recipe_version: 0,
|
|
394
441
|
},
|
|
395
442
|
],
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
add: [id],
|
|
399
|
-
}, targets: {
|
|
400
|
-
add: [devrevId],
|
|
401
|
-
} }));
|
|
402
|
-
console.log('Successfully updated sync mapper record.', syncMapperRecordUpdateResponse.data);
|
|
403
|
-
}
|
|
404
|
-
catch (error) {
|
|
405
|
-
console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
|
|
406
|
-
return {
|
|
407
|
-
error: {
|
|
408
|
-
message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
|
|
409
|
-
},
|
|
410
|
-
};
|
|
411
|
-
}
|
|
412
|
-
return {
|
|
413
|
-
report: {
|
|
414
|
-
item_type: itemTypeToLoad.itemType,
|
|
415
|
-
[loading_1.ActionType.UPDATED]: 1,
|
|
416
|
-
},
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
else if (delay) {
|
|
420
|
-
console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
|
|
421
|
-
return {
|
|
422
|
-
rateLimit: {
|
|
423
|
-
delay,
|
|
424
|
-
},
|
|
425
|
-
};
|
|
426
|
-
}
|
|
427
|
-
else {
|
|
428
|
-
console.warn('Failed to update item in external system', error);
|
|
429
|
-
return {
|
|
430
|
-
report: {
|
|
431
|
-
item_type: itemTypeToLoad.itemType,
|
|
432
|
-
[loading_1.ActionType.FAILED]: 1,
|
|
433
|
-
},
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
// TODO: Update mapper (optional)
|
|
437
|
-
}
|
|
438
|
-
catch (error) {
|
|
439
|
-
if (axios_1.default.isAxiosError(error)) {
|
|
440
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
441
|
-
// Create item in external system if mapper record not found
|
|
442
|
-
const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
|
|
443
|
-
item,
|
|
444
|
-
mappers: this._mappers,
|
|
445
|
-
event: this.event,
|
|
446
|
-
});
|
|
447
|
-
if (id) {
|
|
448
|
-
// Create mapper
|
|
449
|
-
try {
|
|
450
|
-
const syncMapperRecordCreateResponse = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
|
|
451
|
-
external_versions: [
|
|
452
|
-
{
|
|
453
|
-
modified_date: modifiedDate,
|
|
454
|
-
recipe_version: 0,
|
|
455
|
-
},
|
|
456
|
-
],
|
|
457
|
-
})));
|
|
458
|
-
console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
|
|
459
|
-
return {
|
|
460
|
-
report: {
|
|
461
|
-
item_type: itemTypeToLoad.itemType,
|
|
462
|
-
[loading_1.ActionType.CREATED]: 1,
|
|
463
|
-
},
|
|
464
|
-
};
|
|
465
|
-
}
|
|
466
|
-
catch (error) {
|
|
467
|
-
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
468
|
-
return {
|
|
469
|
-
error: {
|
|
470
|
-
message: 'Failed to create sync mapper record. ' +
|
|
471
|
-
(0, logger_1.serializeError)(error),
|
|
472
|
-
},
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
else if (delay) {
|
|
443
|
+
})));
|
|
444
|
+
console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
|
|
477
445
|
return {
|
|
478
|
-
|
|
479
|
-
|
|
446
|
+
report: {
|
|
447
|
+
item_type: itemTypeToLoad.itemType,
|
|
448
|
+
[loading_1.ActionType.CREATED]: 1,
|
|
480
449
|
},
|
|
481
450
|
};
|
|
482
451
|
}
|
|
483
|
-
|
|
484
|
-
console.warn('Failed to create
|
|
452
|
+
catch (error) {
|
|
453
|
+
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
485
454
|
return {
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
455
|
+
error: {
|
|
456
|
+
message: 'Failed to create sync mapper record. ' +
|
|
457
|
+
(0, logger_1.serializeError)(error),
|
|
489
458
|
},
|
|
490
459
|
};
|
|
491
460
|
}
|
|
492
461
|
}
|
|
462
|
+
else if (delay) {
|
|
463
|
+
return {
|
|
464
|
+
rateLimit: {
|
|
465
|
+
delay,
|
|
466
|
+
},
|
|
467
|
+
};
|
|
468
|
+
}
|
|
493
469
|
else {
|
|
494
|
-
console.warn('Failed to
|
|
470
|
+
console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
|
|
495
471
|
return {
|
|
496
|
-
|
|
497
|
-
|
|
472
|
+
report: {
|
|
473
|
+
item_type: itemTypeToLoad.itemType,
|
|
474
|
+
[loading_1.ActionType.FAILED]: 1,
|
|
498
475
|
},
|
|
499
476
|
};
|
|
500
477
|
}
|
|
501
478
|
}
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
479
|
+
else {
|
|
480
|
+
console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
|
|
481
|
+
return {
|
|
482
|
+
error: {
|
|
483
|
+
message: error.message,
|
|
484
|
+
},
|
|
485
|
+
};
|
|
486
|
+
}
|
|
508
487
|
}
|
|
509
|
-
|
|
488
|
+
console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
|
|
489
|
+
return {
|
|
490
|
+
error: {
|
|
491
|
+
message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
|
|
492
|
+
},
|
|
493
|
+
};
|
|
494
|
+
}
|
|
510
495
|
}
|
|
511
496
|
async processAttachment(attachment, stream) {
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
497
|
+
var _a, _b;
|
|
498
|
+
const { httpStream, delay, error } = await stream({
|
|
499
|
+
item: attachment,
|
|
500
|
+
event: this.event,
|
|
501
|
+
});
|
|
502
|
+
if (error) {
|
|
503
|
+
return { error };
|
|
504
|
+
}
|
|
505
|
+
else if (delay) {
|
|
506
|
+
return { delay };
|
|
507
|
+
}
|
|
508
|
+
if (httpStream) {
|
|
509
|
+
const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
|
|
510
|
+
// Get upload URL
|
|
511
|
+
const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType);
|
|
512
|
+
if (!preparedArtifact) {
|
|
513
|
+
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment.`);
|
|
514
|
+
this.destroyHttpStream(httpStream);
|
|
515
|
+
return;
|
|
520
516
|
}
|
|
521
|
-
|
|
522
|
-
|
|
517
|
+
if (this.isTimeout) {
|
|
518
|
+
this.destroyHttpStream(httpStream);
|
|
519
|
+
return;
|
|
523
520
|
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
}
|
|
550
|
-
const ssorAttachment = {
|
|
551
|
-
id: {
|
|
552
|
-
devrev: preparedArtifact.artifact_id,
|
|
553
|
-
external: attachment.id,
|
|
554
|
-
},
|
|
555
|
-
parent_id: {
|
|
556
|
-
external: attachment.parent_id,
|
|
557
|
-
},
|
|
521
|
+
// Stream attachment
|
|
522
|
+
const uploadedArtifact = await this.uploader.streamArtifact(preparedArtifact, httpStream);
|
|
523
|
+
if (!uploadedArtifact) {
|
|
524
|
+
console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment.`);
|
|
525
|
+
this.destroyHttpStream(httpStream);
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
// Confirm attachment upload
|
|
529
|
+
const confirmArtifactUploadResponse = await this.uploader.confirmArtifactUpload(preparedArtifact.artifact_id);
|
|
530
|
+
if (!confirmArtifactUploadResponse) {
|
|
531
|
+
console.warn('Error while confirming upload for attachment ID ' + attachment.id);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
const ssorAttachment = {
|
|
535
|
+
id: {
|
|
536
|
+
devrev: preparedArtifact.artifact_id,
|
|
537
|
+
external: attachment.id,
|
|
538
|
+
},
|
|
539
|
+
parent_id: {
|
|
540
|
+
external: attachment.parent_id,
|
|
541
|
+
},
|
|
542
|
+
};
|
|
543
|
+
if (attachment.author_id) {
|
|
544
|
+
ssorAttachment.actor_id = {
|
|
545
|
+
external: attachment.author_id,
|
|
558
546
|
};
|
|
559
|
-
if (attachment.author_id) {
|
|
560
|
-
ssorAttachment.actor_id = {
|
|
561
|
-
external: attachment.author_id,
|
|
562
|
-
};
|
|
563
|
-
}
|
|
564
|
-
// This will set inline flag in ssor_attachment only if it is explicity
|
|
565
|
-
// set in the attachment object.
|
|
566
|
-
if (attachment.inline === true) {
|
|
567
|
-
ssorAttachment.inline = true;
|
|
568
|
-
}
|
|
569
|
-
else if (attachment.inline === false) {
|
|
570
|
-
ssorAttachment.inline = false;
|
|
571
|
-
}
|
|
572
|
-
await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
|
|
573
547
|
}
|
|
574
|
-
|
|
575
|
-
|
|
548
|
+
// This will set inline flag in ssor_attachment only if it is explicity
|
|
549
|
+
// set in the attachment object.
|
|
550
|
+
if (attachment.inline === true) {
|
|
551
|
+
ssorAttachment.inline = true;
|
|
552
|
+
}
|
|
553
|
+
else if (attachment.inline === false) {
|
|
554
|
+
ssorAttachment.inline = false;
|
|
555
|
+
}
|
|
556
|
+
await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
|
|
557
|
+
}
|
|
558
|
+
return;
|
|
576
559
|
}
|
|
577
560
|
/**
|
|
578
561
|
* Destroys a stream to prevent memory leaks.
|
|
@@ -594,50 +577,48 @@ class WorkerAdapter {
|
|
|
594
577
|
}
|
|
595
578
|
}
|
|
596
579
|
async loadAttachment({ item, create, }) {
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
|
|
620
|
-
}
|
|
621
|
-
catch (error) {
|
|
622
|
-
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
623
|
-
}
|
|
624
|
-
return {
|
|
625
|
-
report: {
|
|
626
|
-
item_type: 'attachment',
|
|
627
|
-
[loading_1.ActionType.CREATED]: 1,
|
|
628
|
-
},
|
|
629
|
-
};
|
|
580
|
+
// Create item
|
|
581
|
+
const { id, delay, error } = await create({
|
|
582
|
+
item,
|
|
583
|
+
mappers: this._mappers,
|
|
584
|
+
event: this.event,
|
|
585
|
+
});
|
|
586
|
+
if (delay) {
|
|
587
|
+
return {
|
|
588
|
+
rateLimit: {
|
|
589
|
+
delay,
|
|
590
|
+
},
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
else if (id) {
|
|
594
|
+
try {
|
|
595
|
+
const syncMapperRecordCreateResponse = await this._mappers.create({
|
|
596
|
+
sync_unit: this.event.payload.event_context.sync_unit,
|
|
597
|
+
external_ids: [id],
|
|
598
|
+
targets: [item.reference_id],
|
|
599
|
+
status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
|
|
600
|
+
});
|
|
601
|
+
console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
|
|
630
602
|
}
|
|
631
|
-
|
|
632
|
-
console.warn('Failed to create
|
|
633
|
-
return {
|
|
634
|
-
report: {
|
|
635
|
-
item_type: 'attachment',
|
|
636
|
-
[loading_1.ActionType.FAILED]: 1,
|
|
637
|
-
},
|
|
638
|
-
};
|
|
603
|
+
catch (error) {
|
|
604
|
+
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
639
605
|
}
|
|
640
|
-
|
|
606
|
+
return {
|
|
607
|
+
report: {
|
|
608
|
+
item_type: 'attachment',
|
|
609
|
+
[loading_1.ActionType.CREATED]: 1,
|
|
610
|
+
},
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
else {
|
|
614
|
+
console.warn('Failed to create attachment in external system', error);
|
|
615
|
+
return {
|
|
616
|
+
report: {
|
|
617
|
+
item_type: 'attachment',
|
|
618
|
+
[loading_1.ActionType.FAILED]: 1,
|
|
619
|
+
},
|
|
620
|
+
};
|
|
621
|
+
}
|
|
641
622
|
}
|
|
642
623
|
/**
|
|
643
624
|
* Streams the attachments to the DevRev platform.
|
|
@@ -648,88 +629,86 @@ class WorkerAdapter {
|
|
|
648
629
|
*/
|
|
649
630
|
async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
|
|
650
631
|
}) {
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
632
|
+
var _a, _b;
|
|
633
|
+
if (batchSize <= 0) {
|
|
634
|
+
console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
|
|
635
|
+
batchSize = 1;
|
|
636
|
+
}
|
|
637
|
+
if (batchSize > 50) {
|
|
638
|
+
console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
|
|
639
|
+
batchSize = 50;
|
|
640
|
+
}
|
|
641
|
+
const repos = [
|
|
642
|
+
{
|
|
643
|
+
itemType: 'ssor_attachment',
|
|
644
|
+
},
|
|
645
|
+
];
|
|
646
|
+
this.initializeRepos(repos);
|
|
647
|
+
const attachmentsMetadata = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
648
|
+
// If there are no attachments metadata artifact IDs in state, finish here
|
|
649
|
+
if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
|
|
650
|
+
console.log(`No attachments metadata artifact IDs found in state.`);
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
else {
|
|
654
|
+
console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
|
|
655
|
+
}
|
|
656
|
+
// Loop through the attachments metadata artifact IDs
|
|
657
|
+
while (attachmentsMetadata.artifactIds.length > 0) {
|
|
658
|
+
const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
|
|
659
|
+
console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
660
|
+
const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
|
|
661
|
+
artifact: attachmentsMetadataArtifactId,
|
|
662
|
+
});
|
|
663
|
+
if (error) {
|
|
664
|
+
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
665
|
+
return { error };
|
|
656
666
|
}
|
|
657
|
-
if (
|
|
658
|
-
console.warn(`
|
|
659
|
-
|
|
667
|
+
if (!attachments || attachments.length === 0) {
|
|
668
|
+
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
669
|
+
// Remove empty artifact and reset lastProcessed
|
|
670
|
+
attachmentsMetadata.artifactIds.shift();
|
|
671
|
+
attachmentsMetadata.lastProcessed = 0;
|
|
672
|
+
continue;
|
|
660
673
|
}
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
674
|
+
console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
675
|
+
let response;
|
|
676
|
+
if (processors) {
|
|
677
|
+
console.log(`Using custom processors for attachments.`);
|
|
678
|
+
const reducer = processors.reducer;
|
|
679
|
+
const iterator = processors.iterator;
|
|
680
|
+
const reducedAttachments = reducer({
|
|
681
|
+
attachments,
|
|
682
|
+
adapter: this,
|
|
683
|
+
batchSize,
|
|
684
|
+
});
|
|
685
|
+
response = await iterator({
|
|
686
|
+
reducedAttachments,
|
|
687
|
+
adapter: this,
|
|
688
|
+
stream,
|
|
689
|
+
});
|
|
672
690
|
}
|
|
673
691
|
else {
|
|
674
|
-
console.log(`
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
|
|
681
|
-
artifact: attachmentsMetadataArtifactId,
|
|
692
|
+
console.log(`Using attachments streaming pool for attachments streaming.`);
|
|
693
|
+
const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
694
|
+
adapter: this,
|
|
695
|
+
attachments,
|
|
696
|
+
batchSize,
|
|
697
|
+
stream,
|
|
682
698
|
});
|
|
683
|
-
|
|
684
|
-
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
685
|
-
return { error };
|
|
686
|
-
}
|
|
687
|
-
if (!attachments || attachments.length === 0) {
|
|
688
|
-
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
689
|
-
// Remove empty artifact and reset lastProcessed
|
|
690
|
-
attachmentsMetadata.artifactIds.shift();
|
|
691
|
-
attachmentsMetadata.lastProcessed = 0;
|
|
692
|
-
continue;
|
|
693
|
-
}
|
|
694
|
-
console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
695
|
-
let response;
|
|
696
|
-
if (processors) {
|
|
697
|
-
console.log(`Using custom processors for attachments.`);
|
|
698
|
-
const reducer = processors.reducer;
|
|
699
|
-
const iterator = processors.iterator;
|
|
700
|
-
const reducedAttachments = reducer({
|
|
701
|
-
attachments,
|
|
702
|
-
adapter: this,
|
|
703
|
-
batchSize,
|
|
704
|
-
});
|
|
705
|
-
response = await iterator({
|
|
706
|
-
reducedAttachments,
|
|
707
|
-
adapter: this,
|
|
708
|
-
stream,
|
|
709
|
-
});
|
|
710
|
-
}
|
|
711
|
-
else {
|
|
712
|
-
console.log(`Using attachments streaming pool for attachments streaming.`);
|
|
713
|
-
const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
714
|
-
adapter: this,
|
|
715
|
-
attachments,
|
|
716
|
-
batchSize,
|
|
717
|
-
stream,
|
|
718
|
-
});
|
|
719
|
-
response = await attachmentsPool.streamAll();
|
|
720
|
-
}
|
|
721
|
-
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
722
|
-
return response;
|
|
723
|
-
}
|
|
724
|
-
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
725
|
-
attachmentsMetadata.artifactIds.shift();
|
|
726
|
-
attachmentsMetadata.lastProcessed = 0;
|
|
727
|
-
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
728
|
-
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
729
|
-
}
|
|
699
|
+
response = await attachmentsPool.streamAll();
|
|
730
700
|
}
|
|
731
|
-
|
|
732
|
-
|
|
701
|
+
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
702
|
+
return response;
|
|
703
|
+
}
|
|
704
|
+
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
705
|
+
attachmentsMetadata.artifactIds.shift();
|
|
706
|
+
attachmentsMetadata.lastProcessed = 0;
|
|
707
|
+
if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
|
|
708
|
+
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
return;
|
|
733
712
|
}
|
|
734
713
|
}
|
|
735
714
|
exports.WorkerAdapter = WorkerAdapter;
|