@devrev/airsync-sdk 2.0.0-beta.0 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/dist/airsync-sdk.d.ts +1802 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +44 -4
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +259 -10
- package/dist/common/constants.d.ts +16 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +20 -6
- package/dist/common/control-protocol.d.ts +2 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +1 -5
- package/dist/common/helpers.d.ts +8 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +15 -0
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/common/install-initial-domain-mapping.test.js +7 -3
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +101 -0
- package/dist/common/time-value-resolver.d.ts +47 -0
- package/dist/common/time-value-resolver.d.ts.map +1 -0
- package/dist/common/time-value-resolver.js +141 -0
- package/dist/common/time-value-resolver.test.d.ts +2 -0
- package/dist/common/time-value-resolver.test.d.ts.map +1 -0
- package/dist/common/time-value-resolver.test.js +268 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +7 -0
- package/dist/http/axios-client-internal.test.js +27 -6
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +6 -3
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +9 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -10
- package/dist/logger/logger.d.ts +1 -17
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.js +35 -27
- package/dist/logger/logger.test.js +66 -19
- package/dist/mappers/mappers.interface.d.ts +5 -5
- package/dist/mappers/mappers.interface.js +3 -3
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.d.ts +47 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.js +4 -0
- package/dist/mock-server/mock-server.js +278 -0
- package/dist/multithreading/create-worker.test.js +36 -17
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -44
- package/dist/multithreading/process-task.test.d.ts +2 -0
- package/dist/multithreading/process-task.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.test.js +146 -0
- package/dist/multithreading/spawn/spawn.d.ts +4 -3
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +6 -14
- package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
- package/dist/multithreading/spawn/spawn.js +41 -31
- package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.test.js +232 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +233 -134
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +4 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +4 -3
- package/dist/repo/repo.test.js +60 -7
- package/dist/state/state.d.ts +7 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.extract-window.test.d.ts +2 -0
- package/dist/state/state.extract-window.test.d.ts.map +1 -0
- package/dist/state/state.extract-window.test.js +163 -0
- package/dist/state/state.interfaces.d.ts +25 -8
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +4 -2
- package/dist/state/state.js +111 -12
- package/dist/state/state.pending-boundaries.test.d.ts +2 -0
- package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
- package/dist/state/state.pending-boundaries.test.js +189 -0
- package/dist/state/state.post-state.test.d.ts +2 -0
- package/dist/state/state.post-state.test.d.ts.map +1 -0
- package/dist/state/state.post-state.test.js +77 -0
- package/dist/state/state.test.js +117 -73
- package/dist/state/state.time-value-resolution.test.d.ts +2 -0
- package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
- package/dist/state/state.time-value-resolution.test.js +175 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/dist/types/common.d.ts +0 -25
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -11
- package/dist/types/external-domain-metadata.d.ts +338 -0
- package/dist/types/external-domain-metadata.d.ts.map +1 -0
- package/dist/types/extraction.d.ts +89 -218
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +56 -124
- package/dist/types/extraction.test.js +69 -43
- package/dist/types/index.d.ts +4 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +6 -3
- package/dist/types/loading.d.ts +4 -24
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +1 -22
- package/dist/types/workers.d.ts +31 -14
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +1 -0
- package/dist/uploader/uploader.d.ts +1 -0
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.test.js +0 -11
- package/dist/uploader/uploader.interfaces.d.ts +3 -3
- package/dist/uploader/uploader.js +53 -20
- package/dist/uploader/uploader.test.js +22 -10
- package/package.json +12 -11
- package/dist/common/event-type-translation.d.ts +0 -24
- package/dist/common/event-type-translation.d.ts.map +0 -1
- package/dist/common/event-type-translation.js +0 -117
- package/dist/deprecated/adapter/index.d.ts +0 -62
- package/dist/deprecated/adapter/index.d.ts.map +0 -1
- package/dist/deprecated/adapter/index.js +0 -151
- package/dist/deprecated/common/helpers.d.ts +0 -7
- package/dist/deprecated/common/helpers.d.ts.map +0 -1
- package/dist/deprecated/common/helpers.js +0 -47
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
- package/dist/deprecated/demo-extractor/index.d.ts +0 -18
- package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
- package/dist/deprecated/demo-extractor/index.js +0 -161
- package/dist/deprecated/http/client.d.ts +0 -22
- package/dist/deprecated/http/client.d.ts.map +0 -1
- package/dist/deprecated/http/client.js +0 -161
- package/dist/deprecated/uploader/index.d.ts +0 -35
- package/dist/deprecated/uploader/index.d.ts.map +0 -1
- package/dist/deprecated/uploader/index.js +0 -161
- package/dist/http/types.d.ts +0 -17
- package/dist/http/types.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
- /package/dist/{http/types.js → types/external-domain-metadata.js} +0 -0
|
@@ -20,7 +20,7 @@ const extraction_1 = require("../../types/extraction");
|
|
|
20
20
|
const loading_1 = require("../../types/loading");
|
|
21
21
|
const workers_1 = require("../../types/workers");
|
|
22
22
|
const uploader_1 = require("../../uploader/uploader");
|
|
23
|
-
const
|
|
23
|
+
const helpers_1 = require("../../common/helpers");
|
|
24
24
|
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
25
25
|
return new WorkerAdapter({
|
|
26
26
|
event,
|
|
@@ -29,7 +29,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
|
|
|
29
29
|
});
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
|
-
* WorkerAdapter class is used to interact with
|
|
32
|
+
* WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in
|
|
33
33
|
* as parameter in processTask and onTimeout functions. The class provides
|
|
34
34
|
* utilities to emit control events to the platform, update the state of the connector,
|
|
35
35
|
* and upload artifacts to the platform.
|
|
@@ -46,6 +46,7 @@ function createWorkerAdapter({ event, adapterState, options, }) {
|
|
|
46
46
|
class WorkerAdapter {
|
|
47
47
|
constructor({ event, adapterState, options, }) {
|
|
48
48
|
this.repos = [];
|
|
49
|
+
this.currentEventDataLength = 0;
|
|
49
50
|
this.event = event;
|
|
50
51
|
this.options = options;
|
|
51
52
|
this.adapterState = adapterState;
|
|
@@ -68,9 +69,7 @@ class WorkerAdapter {
|
|
|
68
69
|
return this.adapterState.state;
|
|
69
70
|
}
|
|
70
71
|
set state(value) {
|
|
71
|
-
|
|
72
|
-
this.adapterState.state = value;
|
|
73
|
-
}
|
|
72
|
+
this.adapterState.state = value;
|
|
74
73
|
}
|
|
75
74
|
get reports() {
|
|
76
75
|
return this.loaderReports;
|
|
@@ -81,17 +80,38 @@ class WorkerAdapter {
|
|
|
81
80
|
get mappers() {
|
|
82
81
|
return this._mappers;
|
|
83
82
|
}
|
|
83
|
+
get extractionScope() {
|
|
84
|
+
return this.adapterState.extractionScope;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Returns whether the given item type should be extracted.
|
|
88
|
+
* Defaults to true if the scope is empty or the item type is not listed.
|
|
89
|
+
*/
|
|
90
|
+
shouldExtract(itemType) {
|
|
91
|
+
const scope = this.extractionScope;
|
|
92
|
+
if (Object.keys(scope).length === 0)
|
|
93
|
+
return true;
|
|
94
|
+
if (!(itemType in scope))
|
|
95
|
+
return true;
|
|
96
|
+
return scope[itemType].extract;
|
|
97
|
+
}
|
|
84
98
|
initializeRepos(repos) {
|
|
85
99
|
this.repos = repos.map((repo) => {
|
|
86
|
-
const shouldNormalize = repo.itemType !== constants_1.
|
|
87
|
-
repo.itemType !== constants_1.
|
|
100
|
+
const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
101
|
+
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
88
102
|
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
|
|
89
103
|
var _a;
|
|
90
104
|
// We need to store artifacts ids in state for later use when streaming attachments
|
|
91
|
-
if (repo.itemType === constants_1.
|
|
105
|
+
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
92
106
|
(_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
93
107
|
}
|
|
94
|
-
|
|
108
|
+
// Calculate size of the entire artifact object that goes in the SQS message
|
|
109
|
+
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
110
|
+
if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
|
|
111
|
+
!this.isTimeout) {
|
|
112
|
+
this.isTimeout = true;
|
|
113
|
+
}
|
|
114
|
+
}, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
|
|
95
115
|
});
|
|
96
116
|
}
|
|
97
117
|
getRepo(itemType) {
|
|
@@ -125,29 +145,42 @@ class WorkerAdapter {
|
|
|
125
145
|
*/
|
|
126
146
|
async emit(newEventType, data) {
|
|
127
147
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
128
|
-
|
|
148
|
+
var _a;
|
|
129
149
|
if (this.hasWorkerEmitted) {
|
|
130
150
|
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
131
151
|
return;
|
|
132
152
|
}
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
return;
|
|
144
|
-
}
|
|
153
|
+
// Upload all repos before emitting the event
|
|
154
|
+
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
155
|
+
try {
|
|
156
|
+
await this.uploadAllRepos();
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
console.error('Error while uploading repos', error);
|
|
160
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
161
|
+
this.hasWorkerEmitted = true;
|
|
162
|
+
return;
|
|
145
163
|
}
|
|
146
|
-
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
147
164
|
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
148
|
-
|
|
149
|
-
this.state.
|
|
150
|
-
this.state.
|
|
165
|
+
// Clear pending extraction boundaries now that the cycle is complete
|
|
166
|
+
this.state.pendingWorkersOldest = '';
|
|
167
|
+
this.state.pendingWorkersNewest = '';
|
|
168
|
+
// Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
|
|
169
|
+
// Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
|
|
170
|
+
const extractionStart = this.event.payload.event_context.extract_from;
|
|
171
|
+
const extractionEnd = this.event.payload.event_context.extract_to;
|
|
172
|
+
if (extractionStart &&
|
|
173
|
+
(!this.state.workersOldest ||
|
|
174
|
+
extractionStart < this.state.workersOldest)) {
|
|
175
|
+
console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
|
|
176
|
+
this.state.workersOldest = extractionStart;
|
|
177
|
+
}
|
|
178
|
+
if (extractionEnd &&
|
|
179
|
+
(!this.state.workersNewest ||
|
|
180
|
+
extractionEnd > this.state.workersNewest)) {
|
|
181
|
+
console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
|
|
182
|
+
this.state.workersNewest = extractionEnd;
|
|
183
|
+
}
|
|
151
184
|
}
|
|
152
185
|
// We want to save the state every time we emit an event, except for the start and delete events
|
|
153
186
|
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
@@ -163,11 +196,17 @@ class WorkerAdapter {
|
|
|
163
196
|
}
|
|
164
197
|
}
|
|
165
198
|
try {
|
|
199
|
+
// Always prune error messages to make them shorter before emit
|
|
200
|
+
if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
|
|
201
|
+
data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
|
|
202
|
+
}
|
|
203
|
+
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
204
|
+
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
|
|
166
205
|
await (0, control_protocol_1.emit)({
|
|
167
206
|
eventType: newEventType,
|
|
168
207
|
event: this.event,
|
|
169
|
-
data: Object.assign(Object.assign({}, data), (
|
|
170
|
-
? {
|
|
208
|
+
data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
|
|
209
|
+
? { reports: this.reports, processed_files: this.processedFiles }
|
|
171
210
|
: {})),
|
|
172
211
|
});
|
|
173
212
|
const message = {
|
|
@@ -194,9 +233,6 @@ class WorkerAdapter {
|
|
|
194
233
|
}
|
|
195
234
|
}
|
|
196
235
|
}
|
|
197
|
-
handleTimeout() {
|
|
198
|
-
this.isTimeout = true;
|
|
199
|
-
}
|
|
200
236
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
201
237
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
202
238
|
var _a;
|
|
@@ -225,57 +261,74 @@ class WorkerAdapter {
|
|
|
225
261
|
};
|
|
226
262
|
}
|
|
227
263
|
console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
228
|
-
|
|
229
|
-
.
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
error: {
|
|
235
|
-
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
236
|
-
},
|
|
237
|
-
});
|
|
238
|
-
break;
|
|
239
|
-
}
|
|
240
|
-
if (!fileToLoad.completed) {
|
|
241
|
-
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
242
|
-
artifactId: fileToLoad.id,
|
|
243
|
-
isGzipped: true,
|
|
244
|
-
});
|
|
245
|
-
if (transformerFileError) {
|
|
246
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
264
|
+
try {
|
|
265
|
+
outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
|
|
266
|
+
.filesToLoad) {
|
|
267
|
+
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
268
|
+
if (!itemTypeToLoad) {
|
|
269
|
+
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
247
270
|
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
248
271
|
error: {
|
|
249
|
-
message: `
|
|
272
|
+
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
250
273
|
},
|
|
251
274
|
});
|
|
275
|
+
break;
|
|
252
276
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
itemTypeToLoad,
|
|
277
|
+
if (!fileToLoad.completed) {
|
|
278
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
279
|
+
artifactId: fileToLoad.id,
|
|
280
|
+
isGzipped: true,
|
|
258
281
|
});
|
|
259
|
-
if (
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
282
|
+
if (transformerFileError) {
|
|
283
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
284
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
285
|
+
error: {
|
|
286
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
287
|
+
},
|
|
264
288
|
});
|
|
265
289
|
break outerloop;
|
|
266
290
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
291
|
+
const transformerFile = response;
|
|
292
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
293
|
+
if (this.isTimeout) {
|
|
294
|
+
console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
|
|
295
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
|
|
296
|
+
process.exit(0);
|
|
297
|
+
}
|
|
298
|
+
const { report, rateLimit } = await this.loadItem({
|
|
299
|
+
item: transformerFile[i],
|
|
300
|
+
itemTypeToLoad,
|
|
271
301
|
});
|
|
272
|
-
|
|
302
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
303
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
|
|
304
|
+
delay: rateLimit.delay,
|
|
305
|
+
reports: this.reports,
|
|
306
|
+
processed_files: this.processedFiles,
|
|
307
|
+
});
|
|
308
|
+
break outerloop;
|
|
309
|
+
}
|
|
310
|
+
if (report) {
|
|
311
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
312
|
+
loaderReports: this.loaderReports,
|
|
313
|
+
report,
|
|
314
|
+
});
|
|
315
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
316
|
+
}
|
|
273
317
|
}
|
|
318
|
+
fileToLoad.completed = true;
|
|
319
|
+
this._processedFiles.push(fileToLoad.id);
|
|
274
320
|
}
|
|
275
|
-
fileToLoad.completed = true;
|
|
276
|
-
this._processedFiles.push(fileToLoad.id);
|
|
277
321
|
}
|
|
278
322
|
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
console.error('Error during data loading.', (0, logger_1.serializeError)(error));
|
|
325
|
+
await this.emit(loading_1.LoaderEventType.DataLoadingError, {
|
|
326
|
+
error: {
|
|
327
|
+
message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
|
|
328
|
+
},
|
|
329
|
+
});
|
|
330
|
+
process.exit(1);
|
|
331
|
+
}
|
|
279
332
|
return {
|
|
280
333
|
reports: this.reports,
|
|
281
334
|
processed_files: this.processedFiles,
|
|
@@ -322,42 +375,63 @@ class WorkerAdapter {
|
|
|
322
375
|
};
|
|
323
376
|
}
|
|
324
377
|
const filesToLoad = (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const transformerFile = response;
|
|
332
|
-
if (transformerFileError) {
|
|
333
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
334
|
-
break outerloop;
|
|
335
|
-
}
|
|
336
|
-
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
337
|
-
const { report, rateLimit } = await this.loadAttachment({
|
|
338
|
-
item: transformerFile[i],
|
|
339
|
-
create,
|
|
378
|
+
try {
|
|
379
|
+
outerloop: for (const fileToLoad of filesToLoad) {
|
|
380
|
+
if (!fileToLoad.completed) {
|
|
381
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
382
|
+
artifactId: fileToLoad.id,
|
|
383
|
+
isGzipped: true,
|
|
340
384
|
});
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
385
|
+
const transformerFile = response;
|
|
386
|
+
if (transformerFileError) {
|
|
387
|
+
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
388
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
389
|
+
error: {
|
|
390
|
+
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
391
|
+
},
|
|
346
392
|
});
|
|
347
393
|
break outerloop;
|
|
348
394
|
}
|
|
349
|
-
|
|
350
|
-
(
|
|
351
|
-
|
|
352
|
-
|
|
395
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
396
|
+
if (this.isTimeout) {
|
|
397
|
+
console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
|
|
398
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
|
|
399
|
+
process.exit(0);
|
|
400
|
+
}
|
|
401
|
+
const { report, rateLimit } = await this.loadAttachment({
|
|
402
|
+
item: transformerFile[i],
|
|
403
|
+
create,
|
|
353
404
|
});
|
|
354
|
-
|
|
405
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
406
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingDelayed, {
|
|
407
|
+
delay: rateLimit.delay,
|
|
408
|
+
reports: this.reports,
|
|
409
|
+
processed_files: this.processedFiles,
|
|
410
|
+
});
|
|
411
|
+
break outerloop;
|
|
412
|
+
}
|
|
413
|
+
if (report) {
|
|
414
|
+
(0, worker_adapter_helpers_1.addReportToLoaderReport)({
|
|
415
|
+
loaderReports: this.loaderReports,
|
|
416
|
+
report,
|
|
417
|
+
});
|
|
418
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
419
|
+
}
|
|
355
420
|
}
|
|
421
|
+
fileToLoad.completed = true;
|
|
422
|
+
this._processedFiles.push(fileToLoad.id);
|
|
356
423
|
}
|
|
357
|
-
fileToLoad.completed = true;
|
|
358
|
-
this._processedFiles.push(fileToLoad.id);
|
|
359
424
|
}
|
|
360
425
|
}
|
|
426
|
+
catch (error) {
|
|
427
|
+
console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
|
|
428
|
+
await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
|
|
429
|
+
error: {
|
|
430
|
+
message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
|
|
431
|
+
},
|
|
432
|
+
});
|
|
433
|
+
process.exit(1);
|
|
434
|
+
}
|
|
361
435
|
return {
|
|
362
436
|
reports: this.reports,
|
|
363
437
|
processed_files: this.processedFiles,
|
|
@@ -383,10 +457,12 @@ class WorkerAdapter {
|
|
|
383
457
|
};
|
|
384
458
|
}
|
|
385
459
|
// Update item in external system
|
|
386
|
-
const { id, modifiedDate, delay, error } = await
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
460
|
+
const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
461
|
+
return await itemTypeToLoad.update({
|
|
462
|
+
item,
|
|
463
|
+
mappers: this._mappers,
|
|
464
|
+
event: this.event,
|
|
465
|
+
});
|
|
390
466
|
});
|
|
391
467
|
if (id) {
|
|
392
468
|
try {
|
|
@@ -444,10 +520,12 @@ class WorkerAdapter {
|
|
|
444
520
|
if (axios_1.default.isAxiosError(error)) {
|
|
445
521
|
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
446
522
|
// Create item in external system if mapper record not found
|
|
447
|
-
const { id, modifiedDate, delay, error } = await
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
523
|
+
const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
524
|
+
return await itemTypeToLoad.create({
|
|
525
|
+
item,
|
|
526
|
+
mappers: this._mappers,
|
|
527
|
+
event: this.event,
|
|
528
|
+
});
|
|
451
529
|
});
|
|
452
530
|
if (id) {
|
|
453
531
|
// Create mapper
|
|
@@ -515,11 +593,11 @@ class WorkerAdapter {
|
|
|
515
593
|
}
|
|
516
594
|
async processAttachment(attachment, stream) {
|
|
517
595
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
518
|
-
var _a;
|
|
519
|
-
const { httpStream, delay, error } = await stream({
|
|
596
|
+
var _a, _b, _c;
|
|
597
|
+
const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
|
|
520
598
|
item: attachment,
|
|
521
599
|
event: this.event,
|
|
522
|
-
});
|
|
600
|
+
}));
|
|
523
601
|
if (error) {
|
|
524
602
|
return { error };
|
|
525
603
|
}
|
|
@@ -527,37 +605,44 @@ class WorkerAdapter {
|
|
|
527
605
|
return { delay };
|
|
528
606
|
}
|
|
529
607
|
if (httpStream) {
|
|
530
|
-
const fileType =
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
608
|
+
const fileType = attachment.content_type ||
|
|
609
|
+
((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
|
|
610
|
+
'application/octet-stream';
|
|
611
|
+
const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
|
|
612
|
+
const fileSize = contentLength ? parseInt(contentLength) : undefined;
|
|
534
613
|
// Get upload URL
|
|
535
614
|
const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
|
|
536
615
|
if (artifactUrlError) {
|
|
537
|
-
console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
538
|
-
(0, logger_1.serializeError)(artifactUrlError));
|
|
539
|
-
this.destroyHttpStream(httpStream);
|
|
540
|
-
return;
|
|
541
|
-
}
|
|
542
|
-
if (this.isTimeout) {
|
|
543
616
|
this.destroyHttpStream(httpStream);
|
|
544
|
-
return
|
|
617
|
+
return {
|
|
618
|
+
error: {
|
|
619
|
+
message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
|
|
620
|
+
fileSize: fileSize,
|
|
621
|
+
},
|
|
622
|
+
};
|
|
545
623
|
}
|
|
546
624
|
// Stream attachment
|
|
547
625
|
const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
|
|
548
626
|
if (uploadedArtifactError) {
|
|
549
|
-
console.warn(`Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
550
|
-
(0, logger_1.serializeError)(uploadedArtifactError));
|
|
551
627
|
this.destroyHttpStream(httpStream);
|
|
552
|
-
return
|
|
628
|
+
return {
|
|
629
|
+
error: {
|
|
630
|
+
message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
|
|
631
|
+
(0, logger_1.serializeError)(uploadedArtifactError),
|
|
632
|
+
fileSize: fileSize,
|
|
633
|
+
},
|
|
634
|
+
};
|
|
553
635
|
}
|
|
554
636
|
// Confirm attachment upload
|
|
555
637
|
const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
|
|
556
638
|
if (confirmArtifactUploadError) {
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
639
|
+
return {
|
|
640
|
+
error: {
|
|
641
|
+
message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
|
|
642
|
+
(0, logger_1.serializeError)(confirmArtifactUploadError),
|
|
643
|
+
fileSize: fileSize,
|
|
644
|
+
},
|
|
645
|
+
};
|
|
561
646
|
}
|
|
562
647
|
const ssorAttachment = {
|
|
563
648
|
id: {
|
|
@@ -581,9 +666,14 @@ class WorkerAdapter {
|
|
|
581
666
|
else if (attachment.inline === false) {
|
|
582
667
|
ssorAttachment.inline = false;
|
|
583
668
|
}
|
|
584
|
-
await ((
|
|
669
|
+
await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
|
|
670
|
+
return;
|
|
585
671
|
}
|
|
586
|
-
return
|
|
672
|
+
return {
|
|
673
|
+
error: {
|
|
674
|
+
message: `Error while opening attachment stream. Skipping attachment.`,
|
|
675
|
+
},
|
|
676
|
+
};
|
|
587
677
|
});
|
|
588
678
|
}
|
|
589
679
|
/**
|
|
@@ -608,11 +698,11 @@ class WorkerAdapter {
|
|
|
608
698
|
async loadAttachment({ item, create, }) {
|
|
609
699
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
610
700
|
// Create item
|
|
611
|
-
const { id, delay, error } = await create({
|
|
701
|
+
const { id, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => create({
|
|
612
702
|
item,
|
|
613
703
|
mappers: this._mappers,
|
|
614
704
|
event: this.event,
|
|
615
|
-
});
|
|
705
|
+
}));
|
|
616
706
|
if (delay) {
|
|
617
707
|
return {
|
|
618
708
|
rateLimit: {
|
|
@@ -709,15 +799,17 @@ class WorkerAdapter {
|
|
|
709
799
|
console.log(`Using custom processors for attachments.`);
|
|
710
800
|
const reducer = processors.reducer;
|
|
711
801
|
const iterator = processors.iterator;
|
|
712
|
-
const reducedAttachments = reducer({
|
|
802
|
+
const reducedAttachments = (0, logger_context_1.runWithUserLogContext)(() => reducer({
|
|
713
803
|
attachments,
|
|
714
804
|
adapter: this,
|
|
715
805
|
batchSize,
|
|
716
|
-
});
|
|
717
|
-
response = await
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
806
|
+
}));
|
|
807
|
+
response = await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
808
|
+
return await iterator({
|
|
809
|
+
reducedAttachments,
|
|
810
|
+
adapter: this,
|
|
811
|
+
stream,
|
|
812
|
+
});
|
|
721
813
|
});
|
|
722
814
|
}
|
|
723
815
|
else {
|
|
@@ -733,6 +825,13 @@ class WorkerAdapter {
|
|
|
733
825
|
if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
|
|
734
826
|
return response;
|
|
735
827
|
}
|
|
828
|
+
// On timeout, emit progress and exit to allow continuation.
|
|
829
|
+
if (this.isTimeout) {
|
|
830
|
+
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.`);
|
|
831
|
+
await this.emit(extraction_1.ExtractorEventType.AttachmentExtractionProgress);
|
|
832
|
+
process.exit(0);
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
736
835
|
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
737
836
|
attachmentsMetadata.artifactIds.shift();
|
|
738
837
|
attachmentsMetadata.lastProcessed = 0;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-adapter.loading.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.loading.test.ts"],"names":[],"mappings":""}
|