@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/dist/airsync-sdk.d.ts +1802 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +13 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +201 -6
- package/dist/common/constants.d.ts +16 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +20 -6
- package/dist/common/control-protocol.d.ts +2 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +1 -5
- package/dist/common/helpers.d.ts +8 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +15 -0
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/common/install-initial-domain-mapping.test.js +5 -2
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +101 -0
- package/dist/common/time-value-resolver.d.ts +47 -0
- package/dist/common/time-value-resolver.d.ts.map +1 -0
- package/dist/common/time-value-resolver.js +141 -0
- package/dist/common/time-value-resolver.test.d.ts +2 -0
- package/dist/common/time-value-resolver.test.d.ts.map +1 -0
- package/dist/common/time-value-resolver.test.js +268 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +7 -0
- package/dist/http/axios-client-internal.test.js +21 -0
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +6 -3
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +9 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -10
- package/dist/logger/logger.d.ts +1 -17
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.js +35 -27
- package/dist/logger/logger.test.js +66 -19
- package/dist/mappers/mappers.interface.d.ts +5 -5
- package/dist/mappers/mappers.interface.js +3 -3
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.d.ts +47 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.js +4 -0
- package/dist/mock-server/mock-server.js +278 -0
- package/dist/multithreading/create-worker.test.js +36 -17
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +15 -48
- package/dist/multithreading/process-task.test.d.ts +2 -0
- package/dist/multithreading/process-task.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.test.js +146 -0
- package/dist/multithreading/spawn/spawn.d.ts +3 -3
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +6 -14
- package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
- package/dist/multithreading/spawn/spawn.js +20 -25
- package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.test.js +232 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +232 -131
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +4 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +4 -3
- package/dist/repo/repo.test.js +60 -7
- package/dist/state/state.d.ts +7 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.extract-window.test.d.ts +2 -0
- package/dist/state/state.extract-window.test.d.ts.map +1 -0
- package/dist/state/state.extract-window.test.js +163 -0
- package/dist/state/state.interfaces.d.ts +17 -7
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +4 -2
- package/dist/state/state.js +111 -12
- package/dist/state/state.pending-boundaries.test.d.ts +2 -0
- package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
- package/dist/state/state.pending-boundaries.test.js +189 -0
- package/dist/state/state.post-state.test.d.ts +2 -0
- package/dist/state/state.post-state.test.d.ts.map +1 -0
- package/dist/state/state.post-state.test.js +77 -0
- package/dist/state/state.test.js +117 -73
- package/dist/state/state.time-value-resolution.test.d.ts +2 -0
- package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
- package/dist/state/state.time-value-resolution.test.js +175 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/dist/types/common.d.ts +0 -25
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -11
- package/dist/types/external-domain-metadata.d.ts +20 -1
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +89 -218
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +56 -124
- package/dist/types/extraction.test.js +69 -43
- package/dist/types/index.d.ts +3 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +6 -3
- package/dist/types/loading.d.ts +4 -24
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +1 -22
- package/dist/types/workers.d.ts +31 -14
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +1 -0
- package/dist/uploader/uploader.d.ts +1 -0
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.test.js +0 -11
- package/dist/uploader/uploader.interfaces.d.ts +3 -3
- package/dist/uploader/uploader.js +53 -20
- package/dist/uploader/uploader.test.js +22 -10
- package/package.json +10 -13
- package/dist/common/event-type-translation.d.ts +0 -24
- package/dist/common/event-type-translation.d.ts.map +0 -1
- package/dist/common/event-type-translation.js +0 -117
- package/dist/deprecated/adapter/index.d.ts +0 -62
- package/dist/deprecated/adapter/index.d.ts.map +0 -1
- package/dist/deprecated/adapter/index.js +0 -151
- package/dist/deprecated/common/helpers.d.ts +0 -7
- package/dist/deprecated/common/helpers.d.ts.map +0 -1
- package/dist/deprecated/common/helpers.js +0 -47
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
- package/dist/deprecated/demo-extractor/index.d.ts +0 -18
- package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
- package/dist/deprecated/demo-extractor/index.js +0 -161
- package/dist/deprecated/http/client.d.ts +0 -22
- package/dist/deprecated/http/client.d.ts.map +0 -1
- package/dist/deprecated/http/client.js +0 -161
- package/dist/deprecated/uploader/index.d.ts +0 -35
- package/dist/deprecated/uploader/index.d.ts.map +0 -1
- package/dist/deprecated/uploader/index.js +0 -161
- package/dist/http/types.d.ts +0 -17
- package/dist/http/types.d.ts.map +0 -1
- package/dist/http/types.js +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
|
@@ -20,7 +20,7 @@ const extraction_1 = require("../../types/extraction");
|
|
|
20
20
|
const loading_1 = require("../../types/loading");
|
|
21
21
|
const workers_1 = require("../../types/workers");
|
|
22
22
|
const uploader_1 = require("../../uploader/uploader");
|
|
23
|
-
const
|
|
23
|
+
const helpers_1 = require("../../common/helpers");
|
|
24
24
|
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
25
25
|
return new WorkerAdapter({
|
|
26
26
|
event,
|
|
@@ -29,7 +29,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
|
|
|
29
29
|
});
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
|
-
* WorkerAdapter class is used to interact with
|
|
32
|
+
* WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
|
|
33
33
|
* as parameter in processTask and onTimeout functions. The class provides
|
|
34
34
|
* utilities to emit control events to the platform, update the state of the connector,
|
|
35
35
|
* and upload artifacts to the platform.
|
|
@@ -46,6 +46,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
|
|
|
46
46
|
class WorkerAdapter {
|
|
47
47
|
constructor({ event, adapterState, options, }) {
|
|
48
48
|
this.repos = [];
|
|
49
|
+
this.currentEventDataLength = 0;
|
|
49
50
|
this.event = event;
|
|
50
51
|
this.options = options;
|
|
51
52
|
this.adapterState = adapterState;
|
|
@@ -79,17 +80,38 @@ class WorkerAdapter {
|
|
|
79
80
|
get mappers() {
|
|
80
81
|
return this._mappers;
|
|
81
82
|
}
|
|
83
|
+
get extractionScope() {
|
|
84
|
+
return this.adapterState.extractionScope;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Returns whether the given item type should be extracted.
|
|
88
|
+
* Defaults to true if the scope is empty or the item type is not listed.
|
|
89
|
+
*/
|
|
90
|
+
shouldExtract(itemType) {
|
|
91
|
+
const scope = this.extractionScope;
|
|
92
|
+
if (Object.keys(scope).length === 0)
|
|
93
|
+
return true;
|
|
94
|
+
if (!(itemType in scope))
|
|
95
|
+
return true;
|
|
96
|
+
return scope[itemType].extract;
|
|
97
|
+
}
|
|
82
98
|
initializeRepos(repos) {
|
|
83
99
|
this.repos = repos.map((repo) => {
|
|
84
|
-
const shouldNormalize = repo.itemType !== constants_1.
|
|
85
|
-
repo.itemType !== constants_1.
|
|
100
|
+
const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
101
|
+
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
86
102
|
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
|
|
87
103
|
var _a;
|
|
88
104
|
// We need to store artifacts ids in state for later use when streaming attachments
|
|
89
|
-
if (repo.itemType === constants_1.
|
|
105
|
+
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
90
106
|
(_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
91
107
|
}
|
|
92
|
-
|
|
108
|
+
// Calculate size of the entire artifact object that goes in the SQS message
|
|
109
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
110
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
111
|
+
!this.isTimeout) {
|
|
112
|
+
this.isTimeout = true;
|
|
113
|
+
}
|
|
114
|
+
}, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
|
|
93
115
|
});
|
|
94
116
|
}
|
|
95
117
|
getRepo(itemType) {
|
|
@@ -123,29 +145,42 @@ class WorkerAdapter {
|
|
|
123
145
|
*/
|
|
124
146
|
async emit(newEventType, data) {
|
|
125
147
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
126
|
-
|
|
148
|
+
var _a;
|
|
127
149
|
if (this.hasWorkerEmitted) {
|
|
128
150
|
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
129
151
|
return;
|
|
130
152
|
}
|
|
131
|
-
//
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
return;
|
|
142
|
-
}
|
|
153
|
+
// Upload all repos before emitting the event
|
|
154
|
+
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
155
|
+
try {
|
|
156
|
+
await this.uploadAllRepos();
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
console.error('Error while uploading repos', error);
|
|
160
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
161
|
+
this.hasWorkerEmitted = true;
|
|
162
|
+
return;
|
|
143
163
|
}
|
|
144
|
-
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
145
164
|
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
146
|
-
|
|
147
|
-
this.state.
|
|
148
|
-
this.state.
|
|
165
|
+
// Clear pending extraction boundaries now that the cycle is complete
|
|
166
|
+
this.state.pendingWorkersOldest = '';
|
|
167
|
+
this.state.pendingWorkersNewest = '';
|
|
168
|
+
// Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
|
|
169
|
+
// Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
|
|
170
|
+
const extractionStart = this.event.payload.event_context.extract_from;
|
|
171
|
+
const extractionEnd = this.event.payload.event_context.extract_to;
|
|
172
|
+
if (extractionStart &&
|
|
173
|
+
(!this.state.workersOldest ||
|
|
174
|
+
extractionStart < this.state.workersOldest)) {
|
|
175
|
+
console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
|
|
176
|
+
this.state.workersOldest = extractionStart;
|
|
177
|
+
}
|
|
178
|
+
if (extractionEnd &&
|
|
179
|
+
(!this.state.workersNewest ||
|
|
180
|
+
extractionEnd > this.state.workersNewest)) {
|
|
181
|
+
console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
|
|
182
|
+
this.state.workersNewest = extractionEnd;
|
|
183
|
+
}
|
|
149
184
|
}
|
|
150
185
|
// We want to save the state every time we emit an event, except for the start and delete events
|
|
151
186
|
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
@@ -161,11 +196,17 @@ class WorkerAdapter {
|
|
|
161
196
|
}
|
|
162
197
|
}
|
|
163
198
|
try {
|
|
199
|
+
// Always prune error messages to make them shorter before emit
|
|
200
|
+
if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
|
|
201
|
+
data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
|
|
202
|
+
}
|
|
203
|
+
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
204
|
+
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
|
|
164
205
|
await (0, control_protocol_1.emit)({
|
|
165
206
|
eventType: newEventType,
|
|
166
207
|
event: this.event,
|
|
167
|
-
data: Object.assign(Object.assign({}, data), (
|
|
168
|
-
? {
|
|
208
|
+
data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
|
|
209
|
+
? { reports: this.reports, processed_files: this.processedFiles }
|
|
169
210
|
: {})),
|
|
170
211
|
});
|
|
171
212
|
const message = {
|
|
@@ -192,9 +233,6 @@ class WorkerAdapter {
|
|
|
192
233
|
}
|
|
193
234
|
}
|
|
194
235
|
}
|
|
195
|
-
handleTimeout() {
|
|
196
|
-
this.isTimeout = true;
|
|
197
|
-
}
|
|
198
236
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
199
237
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
200
238
|
var _a;
|
|
@@ -223,57 +261,74 @@ class WorkerAdapter {
|
|
|
223
261
|
};
|
|
224
262
|
}
|
|
225
263
|
console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
226
|
-
|
|
227
|
-
.
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
error: {
|
|
233
|
-
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
234
|
-
},
|
|
235
|
-
});
|
|
236
|
-
break;
|
|
237
|
-
}
|
|
238
|
-
if (!fileToLoad.completed) {
|
|
239
|
-
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
240
|
-
artifactId: fileToLoad.id,
|
|
241
|
-
isGzipped: true,
|
|
242
|
-
});
|
|
243
|
-
if (transformerFileError) {
|
|
244
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
264
|
+
try {
|
|
265
|
+
outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
|
|
266
|
+
.filesToLoad) {
|
|
267
|
+
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
268
|
+
if (!itemTypeToLoad) {
|
|
269
|
+
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
245
270
|
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
246
271
|
error: {
|
|
247
|
-
message: `
|
|
272
|
+
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
248
273
|
},
|
|
249
274
|
});
|
|
275
|
+
break;
|
|
250
276
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
itemTypeToLoad,
|
|
277
|
+
if (!fileToLoad.completed) {
|
|
278
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
279
|
+
artifactId: fileToLoad.id,
|
|
280
|
+
isGzipped: true,
|
|
256
281
|
});
|
|
257
|
-
if (
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
282
|
+
if (transformerFileError) {
|
|
283
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
284
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
285
|
+
error: {
|
|
286
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
287
|
+
},
|
|
262
288
|
});
|
|
263
289
|
break outerloop;
|
|
264
290
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
291
|
+
const transformerFile = response;
|
|
292
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
293
|
+
if (this.isTimeout) {
|
|
294
|
+
console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
|
|
295
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
|
|
296
|
+
process.exit(0);
|
|
297
|
+
}
|
|
298
|
+
const { report, rateLimit } = await this.loadItem({
|
|
299
|
+
item: transformerFile[i],
|
|
300
|
+
itemTypeToLoad,
|
|
269
301
|
});
|
|
270
|
-
|
|
302
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
303
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
304
|
+
delay: rateLimit.delay,
|
|
305
|
+
reports: this.reports,
|
|
306
|
+
processed_files: this.processedFiles,
|
|
307
|
+
});
|
|
308
|
+
break outerloop;
|
|
309
|
+
}
|
|
310
|
+
if (report) {
|
|
311
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
312
|
+
loaderReports: this.loaderReports,
|
|
313
|
+
report,
|
|
314
|
+
});
|
|
315
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
316
|
+
}
|
|
271
317
|
}
|
|
318
|
+
fileToLoad.completed = true;
|
|
319
|
+
this._processedFiles.push(fileToLoad.id);
|
|
272
320
|
}
|
|
273
|
-
fileToLoad.completed = true;
|
|
274
|
-
this._processedFiles.push(fileToLoad.id);
|
|
275
321
|
}
|
|
276
322
|
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
console.error('Error during data loading.', (0, logger_1.serializeError)(error));
|
|
325
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
326
|
+
error: {
|
|
327
|
+
message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
|
|
328
|
+
},
|
|
329
|
+
});
|
|
330
|
+
process.exit(1);
|
|
331
|
+
}
|
|
277
332
|
return {
|
|
278
333
|
reports: this.reports,
|
|
279
334
|
processed_files: this.processedFiles,
|
|
@@ -320,42 +375,63 @@ class WorkerAdapter {
|
|
|
320
375
|
};
|
|
321
376
|
}
|
|
322
377
|
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
const transformerFile = response;
|
|
330
|
-
if (transformerFileError) {
|
|
331
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
332
|
-
break outerloop;
|
|
333
|
-
}
|
|
334
|
-
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
335
|
-
const { report, rateLimit } = await this.loadAttachment({
|
|
336
|
-
item: transformerFile[i],
|
|
337
|
-
create,
|
|
378
|
+
try {
|
|
379
|
+
outerloop: for (const fileToLoad of filesToLoad) {
|
|
380
|
+
if (!fileToLoad.completed) {
|
|
381
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
382
|
+
artifactId: fileToLoad.id,
|
|
383
|
+
isGzipped: true,
|
|
338
384
|
});
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
385
|
+
const transformerFile = response;
|
|
386
|
+
if (transformerFileError) {
|
|
387
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
388
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
389
|
+
error: {
|
|
390
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
391
|
+
},
|
|
344
392
|
});
|
|
345
393
|
break outerloop;
|
|
346
394
|
}
|
|
347
|
-
|
|
348
|
-
(
|
|
349
|
-
|
|
350
|
-
|
|
395
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
396
|
+
if (this.isTimeout) {
|
|
397
|
+
console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
|
|
398
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
|
|
399
|
+
process.exit(0);
|
|
400
|
+
}
|
|
401
|
+
const { report, rateLimit } = await this.loadAttachment({
|
|
402
|
+
item: transformerFile[i],
|
|
403
|
+
create,
|
|
351
404
|
});
|
|
352
|
-
|
|
405
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
406
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingDelayed, {
|
|
407
|
+
delay: rateLimit.delay,
|
|
408
|
+
reports: this.reports,
|
|
409
|
+
processed_files: this.processedFiles,
|
|
410
|
+
});
|
|
411
|
+
break outerloop;
|
|
412
|
+
}
|
|
413
|
+
if (report) {
|
|
414
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
415
|
+
loaderReports: this.loaderReports,
|
|
416
|
+
report,
|
|
417
|
+
});
|
|
418
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
419
|
+
}
|
|
353
420
|
}
|
|
421
|
+
fileToLoad.completed = true;
|
|
422
|
+
this._processedFiles.push(fileToLoad.id);
|
|
354
423
|
}
|
|
355
|
-
fileToLoad.completed = true;
|
|
356
|
-
this._processedFiles.push(fileToLoad.id);
|
|
357
424
|
}
|
|
358
425
|
}
|
|
426
|
+
catch (error) {
|
|
427
|
+
console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
|
|
428
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
429
|
+
error: {
|
|
430
|
+
message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
|
|
431
|
+
},
|
|
432
|
+
});
|
|
433
|
+
process.exit(1);
|
|
434
|
+
}
|
|
359
435
|
return {
|
|
360
436
|
reports: this.reports,
|
|
361
437
|
processed_files: this.processedFiles,
|
|
@@ -381,10 +457,12 @@ class WorkerAdapter {
|
|
|
381
457
|
};
|
|
382
458
|
}
|
|
383
459
|
// Update item in external system
|
|
384
|
-
const { id, modifiedDate, delay, error } = await
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
460
|
+
const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
461
|
+
return await itemTypeToLoad.update({
|
|
462
|
+
item,
|
|
463
|
+
mappers: this._mappers,
|
|
464
|
+
event: this.event,
|
|
465
|
+
});
|
|
388
466
|
});
|
|
389
467
|
if (id) {
|
|
390
468
|
try {
|
|
@@ -442,10 +520,12 @@ class WorkerAdapter {
|
|
|
442
520
|
if (axios_1.default.isAxiosError(error)) {
|
|
443
521
|
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
444
522
|
// Create item in external system if mapper record not found
|
|
445
|
-
const { id, modifiedDate, delay, error } = await
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
523
|
+
const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
524
|
+
return await itemTypeToLoad.create({
|
|
525
|
+
item,
|
|
526
|
+
mappers: this._mappers,
|
|
527
|
+
event: this.event,
|
|
528
|
+
});
|
|
449
529
|
});
|
|
450
530
|
if (id) {
|
|
451
531
|
// Create mapper
|
|
@@ -513,11 +593,11 @@ class WorkerAdapter {
|
|
|
513
593
|
}
|
|
514
594
|
async processAttachment(attachment, stream) {
|
|
515
595
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
516
|
-
var _a;
|
|
517
|
-
const { httpStream, delay, error } = await stream({
|
|
596
|
+
var _a, _b, _c;
|
|
597
|
+
const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
|
|
518
598
|
item: attachment,
|
|
519
599
|
event: this.event,
|
|
520
|
-
});
|
|
600
|
+
}));
|
|
521
601
|
if (error) {
|
|
522
602
|
return { error };
|
|
523
603
|
}
|
|
@@ -525,37 +605,44 @@ class WorkerAdapter {
|
|
|
525
605
|
return { delay };
|
|
526
606
|
}
|
|
527
607
|
if (httpStream) {
|
|
528
|
-
const fileType =
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
608
|
+
const fileType = attachment.content_type ||
|
|
609
|
+
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
610
|
+
'application/octet-stream';
|
|
611
|
+
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
612
|
+
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
532
613
|
// Get upload URL
|
|
533
614
|
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
534
615
|
if (artifactUrlError) {
|
|
535
|
-
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
536
|
-
(0, logger_1.serializeError)(artifactUrlError));
|
|
537
616
|
this.destroyHttpStream(httpStream);
|
|
538
|
-
return
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
617
|
+
return {
|
|
618
|
+
error: {
|
|
619
|
+
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
620
|
+
fileSize: fileSize,
|
|
621
|
+
},
|
|
622
|
+
};
|
|
543
623
|
}
|
|
544
624
|
// Stream attachment
|
|
545
625
|
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
546
626
|
if (uploadedArtifactError) {
|
|
547
|
-
console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
548
|
-
(0, logger_1.serializeError)(uploadedArtifactError));
|
|
549
627
|
this.destroyHttpStream(httpStream);
|
|
550
|
-
return
|
|
628
|
+
return {
|
|
629
|
+
error: {
|
|
630
|
+
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
631
|
+
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
632
|
+
fileSize: fileSize,
|
|
633
|
+
},
|
|
634
|
+
};
|
|
551
635
|
}
|
|
552
636
|
// Confirm attachment upload
|
|
553
637
|
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
554
638
|
if (confirmArtifactUploadError) {
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
639
|
+
return {
|
|
640
|
+
error: {
|
|
641
|
+
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
642
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
643
|
+
fileSize: fileSize,
|
|
644
|
+
},
|
|
645
|
+
};
|
|
559
646
|
}
|
|
560
647
|
const ssorAttachment = {
|
|
561
648
|
id: {
|
|
@@ -579,9 +666,14 @@ class WorkerAdapter {
|
|
|
579
666
|
else if (attachment.inline === false) {
|
|
580
667
|
ssorAttachment.inline = false;
|
|
581
668
|
}
|
|
582
|
-
await ((
|
|
669
|
+
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
670
|
+
return;
|
|
583
671
|
}
|
|
584
|
-
return
|
|
672
|
+
return {
|
|
673
|
+
error: {
|
|
674
|
+
message: `Error while opening attachment stream. Skipping attachment.`,
|
|
675
|
+
},
|
|
676
|
+
};
|
|
585
677
|
});
|
|
586
678
|
}
|
|
587
679
|
/**
|
|
@@ -606,11 +698,11 @@ class WorkerAdapter {
|
|
|
606
698
|
async loadAttachment({ item, create, }) {
|
|
607
699
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
608
700
|
// Create item
|
|
609
|
-
const { id, delay, error } = await create({
|
|
701
|
+
const { id, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => create({
|
|
610
702
|
item,
|
|
611
703
|
mappers: this._mappers,
|
|
612
704
|
event: this.event,
|
|
613
|
-
});
|
|
705
|
+
}));
|
|
614
706
|
if (delay) {
|
|
615
707
|
return {
|
|
616
708
|
rateLimit: {
|
|
@@ -707,15 +799,17 @@ class WorkerAdapter {
|
|
|
707
799
|
console.log(`Using custom processors for attachments.`);
|
|
708
800
|
const reducer = processors.reducer;
|
|
709
801
|
const iterator = processors.iterator;
|
|
710
|
-
const reducedAttachments = reducer({
|
|
802
|
+
const reducedAttachments = (0, logger_context_1.runWithUserLogContext)(() => reducer({
|
|
711
803
|
attachments,
|
|
712
804
|
adapter: this,
|
|
713
805
|
batchSize,
|
|
714
|
-
});
|
|
715
|
-
response = await
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
806
|
+
}));
|
|
807
|
+
response = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
808
|
+
return await iterator({
|
|
809
|
+
reducedAttachments,
|
|
810
|
+
adapter: this,
|
|
811
|
+
stream,
|
|
812
|
+
});
|
|
719
813
|
});
|
|
720
814
|
}
|
|
721
815
|
else {
|
|
@@ -731,6 +825,13 @@ class WorkerAdapter {
|
|
|
731
825
|
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
732
826
|
return response;
|
|
733
827
|
}
|
|
828
|
+
// On timeout, emit progress and exit to allow continuation.
|
|
829
|
+
if (this.isTimeout) {
|
|
830
|
+
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.`);
|
|
831
|
+
await this.emit(extraction_1.ExtractorEventType.AttachmentExtractionProgress);
|
|
832
|
+
process.exit(0);
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
734
835
|
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
735
836
|
attachmentsMetadata.artifactIds.shift();
|
|
736
837
|
attachmentsMetadata.lastProcessed = 0;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-adapter.loading.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.loading.test.ts"],"names":[],"mappings":""}
|