@devrev/airsync-sdk 2.0.0-beta.6 → 2.0.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -12
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +8 -9
- package/dist/common/constants.d.ts +0 -7
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -28
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +1 -1
- package/dist/http/errors.d.ts +8 -0
- package/dist/http/errors.d.ts.map +1 -0
- package/dist/http/errors.js +31 -0
- package/dist/index.d.ts +18 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -11
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.d.ts +9 -3
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +0 -2
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
- package/dist/mappers/mappers.js +33 -13
- package/dist/multithreading/adapters/base-adapter.d.ts +16 -35
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +89 -23
- package/dist/multithreading/adapters/extraction-adapter.d.ts +24 -20
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +175 -122
- package/dist/multithreading/adapters/loading-adapter.d.ts +13 -23
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +176 -301
- package/dist/multithreading/emit.d.ts.map +1 -1
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +24 -6
- package/dist/multithreading/spawn/spawn.d.ts +6 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +17 -52
- package/dist/multithreading/spawn/spawn.js +30 -26
- package/dist/multithreading/worker.js +15 -1
- package/dist/repo/repo.d.ts +4 -5
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +5 -4
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +13 -15
- package/dist/state/base-state.d.ts +10 -5
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +39 -36
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +39 -4
- package/dist/state/state.interfaces.d.ts +13 -7
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +6 -8
- package/dist/state/time-value-resolver.d.ts.map +1 -1
- package/dist/state/time-value-resolver.js +3 -2
- package/dist/testing/mock-event.d.ts.map +1 -1
- package/dist/testing/mock-event.js +0 -8
- package/dist/testing/mock-server.d.ts +2 -0
- package/dist/testing/mock-server.d.ts.map +1 -1
- package/dist/testing/mock-server.interfaces.d.ts +6 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
- package/dist/testing/mock-server.js +37 -6
- package/dist/testing/test-adapters.d.ts +22 -0
- package/dist/testing/test-adapters.d.ts.map +1 -0
- package/dist/testing/test-adapters.js +32 -0
- package/dist/types/common.d.ts +4 -1
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -1
- package/dist/types/errors.d.ts +13 -7
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +24 -0
- package/dist/types/extraction.d.ts +60 -44
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +10 -1
- package/dist/types/loading.d.ts +11 -30
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/workers.d.ts +27 -23
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +1 -2
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +3 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +16 -0
- package/dist/uploader/uploader.interfaces.d.ts +4 -29
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.js +4 -23
- package/docs/MIGRATION.md +665 -0
- package/package.json +24 -10
- package/MIGRATION.md +0 -1028
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.LoadingAdapter = void 0;
|
|
7
|
-
const axios_1 = __importDefault(require("axios"));
|
|
8
4
|
const logger_1 = require("../../logger/logger");
|
|
9
5
|
const mappers_1 = require("../../mappers/mappers");
|
|
10
6
|
const mappers_interfaces_1 = require("../../mappers/mappers.interfaces");
|
|
@@ -14,14 +10,12 @@ const base_adapter_1 = require("./base-adapter");
|
|
|
14
10
|
const loading_adapter_helpers_1 = require("./loading-adapter.helpers");
|
|
15
11
|
/** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
|
|
16
12
|
class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
13
|
+
/** @internal */
|
|
17
14
|
constructor(params) {
|
|
18
15
|
super(params);
|
|
19
16
|
this.loaderReports = [];
|
|
20
17
|
this._processedFiles = [];
|
|
21
|
-
this._mappers = new mappers_1.Mappers({
|
|
22
|
-
event: params.event,
|
|
23
|
-
options: params.options,
|
|
24
|
-
});
|
|
18
|
+
this._mappers = new mappers_1.Mappers({ event: params.event });
|
|
25
19
|
}
|
|
26
20
|
get reports() {
|
|
27
21
|
return this.loaderReports;
|
|
@@ -32,9 +26,11 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
32
26
|
get mappers() {
|
|
33
27
|
return this._mappers;
|
|
34
28
|
}
|
|
29
|
+
/** @internal */
|
|
35
30
|
async beforeEmit() {
|
|
36
31
|
// Loading has no pre-emit work.
|
|
37
32
|
}
|
|
33
|
+
/** @internal */
|
|
38
34
|
buildEmitPayload(newEventType) {
|
|
39
35
|
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
|
|
40
36
|
return isLoaderEvent
|
|
@@ -44,366 +40,245 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
|
|
|
44
40
|
}
|
|
45
41
|
: {};
|
|
46
42
|
}
|
|
43
|
+
/** @internal */
|
|
47
44
|
afterEmit() {
|
|
48
45
|
// Loading keeps its accumulated reports/processed files across emits.
|
|
49
46
|
}
|
|
50
47
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
51
|
-
var _a;
|
|
52
48
|
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
|
|
53
49
|
const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
|
|
54
50
|
if (!itemTypes.length) {
|
|
55
51
|
console.warn('No item types to load, returning.');
|
|
56
52
|
return { status: 'success' };
|
|
57
53
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
});
|
|
61
|
-
this.sdkState.fromDevRev = {
|
|
62
|
-
filesToLoad,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
if (!this.sdkState.fromDevRev ||
|
|
66
|
-
!this.sdkState.fromDevRev.filesToLoad.length) {
|
|
67
|
-
console.warn('No files to load, returning.');
|
|
68
|
-
return { status: 'success' };
|
|
69
|
-
}
|
|
70
|
-
console.log('Files to load in state', (_a = this.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
|
|
71
|
-
try {
|
|
72
|
-
for (const fileToLoad of this.sdkState.fromDevRev.filesToLoad) {
|
|
73
|
-
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
74
|
-
if (!itemTypeToLoad) {
|
|
75
|
-
console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
|
|
76
|
-
return {
|
|
77
|
-
status: 'error',
|
|
78
|
-
error: {
|
|
79
|
-
message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
|
|
80
|
-
},
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
if (!fileToLoad.completed) {
|
|
84
|
-
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
85
|
-
artifactId: fileToLoad.id,
|
|
86
|
-
isGzipped: true,
|
|
87
|
-
});
|
|
88
|
-
if (transformerFileError) {
|
|
89
|
-
console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
|
|
90
|
-
return {
|
|
91
|
-
status: 'error',
|
|
92
|
-
error: {
|
|
93
|
-
message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
|
|
94
|
-
},
|
|
95
|
-
};
|
|
96
|
-
}
|
|
97
|
-
const transformerFile = response;
|
|
98
|
-
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
99
|
-
if (this.isTimeout) {
|
|
100
|
-
console.log('Timeout detected during data loading. Returning progress to allow continuation.');
|
|
101
|
-
return { status: 'progress' };
|
|
102
|
-
}
|
|
103
|
-
const { report, rateLimit } = await this.loadItem({
|
|
104
|
-
item: transformerFile[i],
|
|
105
|
-
itemTypeToLoad,
|
|
106
|
-
});
|
|
107
|
-
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
108
|
-
return { status: 'delay', delaySeconds: rateLimit.delay };
|
|
109
|
-
}
|
|
110
|
-
if (report) {
|
|
111
|
-
(0, loading_adapter_helpers_1.addReportToLoaderReport)({
|
|
112
|
-
loaderReports: this.loaderReports,
|
|
113
|
-
report,
|
|
114
|
-
});
|
|
115
|
-
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
fileToLoad.completed = true;
|
|
119
|
-
this._processedFiles.push(fileToLoad.id);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
console.error('Error during data loading.', (0, logger_1.serializeError)(error));
|
|
125
|
-
return {
|
|
126
|
-
status: 'error',
|
|
127
|
-
error: {
|
|
128
|
-
message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
|
|
129
|
-
},
|
|
54
|
+
this.adapterState.sdkState.fromDevRev = {
|
|
55
|
+
filesToLoad: await this.getLoaderBatches(itemTypes),
|
|
130
56
|
};
|
|
131
57
|
}
|
|
132
|
-
return
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
137
|
-
if (statsFileArtifactId) {
|
|
138
|
-
const { response, error: statsFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
139
|
-
artifactId: statsFileArtifactId,
|
|
140
|
-
});
|
|
141
|
-
const statsFile = response;
|
|
142
|
-
if (statsFileError || statsFile.length === 0) {
|
|
143
|
-
return [];
|
|
58
|
+
return this.loadFiles('data', (fileToLoad) => {
|
|
59
|
+
const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
|
|
60
|
+
if (!itemTypeToLoad) {
|
|
61
|
+
return;
|
|
144
62
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
statsFile,
|
|
148
|
-
});
|
|
149
|
-
return filesToLoad;
|
|
150
|
-
}
|
|
151
|
-
return [];
|
|
63
|
+
return async (item) => this.loadItem({ item, itemTypeToLoad });
|
|
64
|
+
});
|
|
152
65
|
}
|
|
153
66
|
async loadAttachments({ create, }) {
|
|
154
|
-
var _a, _b;
|
|
155
67
|
if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
|
|
156
|
-
this.sdkState.fromDevRev = {
|
|
157
|
-
filesToLoad: await this.getLoaderBatches(
|
|
158
|
-
supportedItemTypes: ['attachment'],
|
|
159
|
-
}),
|
|
68
|
+
this.adapterState.sdkState.fromDevRev = {
|
|
69
|
+
filesToLoad: await this.getLoaderBatches(['attachment']),
|
|
160
70
|
};
|
|
161
71
|
}
|
|
162
|
-
|
|
163
|
-
|
|
72
|
+
return this.loadFiles('attachment', () => async (item) => this.loadAttachment({ item, create }));
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Walks the transformer files in `fromDevRev.filesToLoad`, resuming from
|
|
76
|
+
* each file's `lineToProcess`. `getLoader` returns undefined when a file has
|
|
77
|
+
* an item type the connector can't load, which fails the phase.
|
|
78
|
+
*/
|
|
79
|
+
async loadFiles(kind, getLoader) {
|
|
80
|
+
var _a;
|
|
81
|
+
const filesToLoad = (_a = this.adapterState.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad;
|
|
82
|
+
if (!(filesToLoad === null || filesToLoad === void 0 ? void 0 : filesToLoad.length)) {
|
|
164
83
|
console.log('No files to load, returning.');
|
|
165
84
|
return { status: 'success' };
|
|
166
85
|
}
|
|
167
|
-
|
|
86
|
+
console.log('Files to load in state', filesToLoad);
|
|
168
87
|
try {
|
|
169
88
|
for (const fileToLoad of filesToLoad) {
|
|
170
|
-
if (
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
89
|
+
if (fileToLoad.completed) {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const load = getLoader(fileToLoad);
|
|
93
|
+
if (!load) {
|
|
94
|
+
const message = `Item type to load not found for item type: ${fileToLoad.itemType}.`;
|
|
95
|
+
console.error(message);
|
|
96
|
+
return { status: 'error', error: { message } };
|
|
97
|
+
}
|
|
98
|
+
const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
99
|
+
artifactId: fileToLoad.id,
|
|
100
|
+
isGzipped: true,
|
|
101
|
+
});
|
|
102
|
+
if (transformerFileError) {
|
|
103
|
+
const message = `Transformer file not found for artifact ID: ${fileToLoad.id}.`;
|
|
104
|
+
console.error(message);
|
|
105
|
+
return { status: 'error', error: { message } };
|
|
106
|
+
}
|
|
107
|
+
const transformerFile = response;
|
|
108
|
+
for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
|
|
109
|
+
if (this.isTimeout) {
|
|
110
|
+
console.log(`Timeout detected during ${kind} loading. Returning progress to allow continuation.`);
|
|
111
|
+
return { status: 'progress' };
|
|
112
|
+
}
|
|
113
|
+
const { report, rateLimit } = await load(transformerFile[i]);
|
|
114
|
+
if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
|
|
115
|
+
return { status: 'delay', delaySeconds: rateLimit.delay };
|
|
184
116
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
190
|
-
const { report, rateLimit } = await this.loadAttachment({
|
|
191
|
-
item: transformerFile[i],
|
|
192
|
-
create,
|
|
117
|
+
if (report) {
|
|
118
|
+
(0, loading_adapter_helpers_1.addReportToLoaderReport)({
|
|
119
|
+
loaderReports: this.loaderReports,
|
|
120
|
+
report,
|
|
193
121
|
});
|
|
194
|
-
|
|
195
|
-
return { status: 'delay', delaySeconds: rateLimit.delay };
|
|
196
|
-
}
|
|
197
|
-
if (report) {
|
|
198
|
-
(0, loading_adapter_helpers_1.addReportToLoaderReport)({
|
|
199
|
-
loaderReports: this.loaderReports,
|
|
200
|
-
report,
|
|
201
|
-
});
|
|
202
|
-
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
203
|
-
}
|
|
122
|
+
fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
|
|
204
123
|
}
|
|
205
|
-
fileToLoad.completed = true;
|
|
206
|
-
this._processedFiles.push(fileToLoad.id);
|
|
207
124
|
}
|
|
125
|
+
fileToLoad.completed = true;
|
|
126
|
+
this._processedFiles.push(fileToLoad.id);
|
|
208
127
|
}
|
|
209
128
|
}
|
|
210
129
|
catch (error) {
|
|
211
|
-
console.error(
|
|
130
|
+
console.error(`Error during ${kind} loading.`, (0, logger_1.serializeError)(error));
|
|
212
131
|
return {
|
|
213
132
|
status: 'error',
|
|
214
133
|
error: {
|
|
215
|
-
message: `Error during
|
|
134
|
+
message: `Error during ${kind} loading. ${(0, logger_1.serializeError)(error)}`,
|
|
216
135
|
},
|
|
217
136
|
};
|
|
218
137
|
}
|
|
219
138
|
return { status: 'success' };
|
|
220
139
|
}
|
|
221
|
-
async
|
|
140
|
+
async getLoaderBatches(supportedItemTypes) {
|
|
222
141
|
var _a;
|
|
142
|
+
const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
|
|
143
|
+
if (!statsFileArtifactId) {
|
|
144
|
+
return [];
|
|
145
|
+
}
|
|
146
|
+
const { response, error: statsFileError } = await this.uploader.getJsonObjectByArtifactId({
|
|
147
|
+
artifactId: statsFileArtifactId,
|
|
148
|
+
});
|
|
149
|
+
const statsFile = response;
|
|
150
|
+
if (statsFileError || statsFile.length === 0) {
|
|
151
|
+
return [];
|
|
152
|
+
}
|
|
153
|
+
return (0, loading_adapter_helpers_1.getFilesToLoad)({ supportedItemTypes, statsFile });
|
|
154
|
+
}
|
|
155
|
+
async loadItem({ item, itemTypeToLoad, }) {
|
|
223
156
|
const devrevId = item.id.devrev;
|
|
157
|
+
const syncUnit = this.event.payload.event_context.sync_unit;
|
|
158
|
+
let syncMapperRecord;
|
|
224
159
|
try {
|
|
225
|
-
|
|
226
|
-
sync_unit:
|
|
160
|
+
syncMapperRecord = await this._mappers.getByTargetId({
|
|
161
|
+
sync_unit: syncUnit,
|
|
227
162
|
target: devrevId,
|
|
228
163
|
});
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
}
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
|
|
167
|
+
return {
|
|
168
|
+
error: {
|
|
169
|
+
message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const loadParams = { item, mappers: this._mappers, event: this.event };
|
|
174
|
+
if (syncMapperRecord) {
|
|
175
|
+
const response = await itemTypeToLoad.update(loadParams);
|
|
176
|
+
if (response.status !== 'success') {
|
|
177
|
+
return this.toFailedLoad(itemTypeToLoad.itemType, 'update', response);
|
|
236
178
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
add: [
|
|
247
|
-
{
|
|
248
|
-
modified_date: modifiedDate,
|
|
249
|
-
recipe_version: 0,
|
|
250
|
-
},
|
|
251
|
-
],
|
|
252
|
-
},
|
|
253
|
-
})), { external_ids: {
|
|
254
|
-
add: [id],
|
|
255
|
-
}, targets: {
|
|
256
|
-
add: [devrevId],
|
|
257
|
-
} }));
|
|
258
|
-
console.log('Successfully updated sync mapper record.', syncMapperRecordUpdate);
|
|
259
|
-
}
|
|
260
|
-
catch (error) {
|
|
261
|
-
console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
|
|
262
|
-
return {
|
|
263
|
-
error: {
|
|
264
|
-
message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
|
|
265
|
-
},
|
|
266
|
-
};
|
|
267
|
-
}
|
|
268
|
-
return {
|
|
269
|
-
report: {
|
|
270
|
-
item_type: itemTypeToLoad.itemType,
|
|
271
|
-
[loading_1.ActionType.UPDATED]: 1,
|
|
179
|
+
try {
|
|
180
|
+
const syncMapperRecordUpdate = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL }, (response.modifiedDate && {
|
|
181
|
+
external_versions: {
|
|
182
|
+
add: [
|
|
183
|
+
{
|
|
184
|
+
modified_date: response.modifiedDate,
|
|
185
|
+
recipe_version: 0,
|
|
186
|
+
},
|
|
187
|
+
],
|
|
272
188
|
},
|
|
273
|
-
}
|
|
189
|
+
})), { external_ids: {
|
|
190
|
+
add: [response.id],
|
|
191
|
+
}, targets: {
|
|
192
|
+
add: [devrevId],
|
|
193
|
+
} }));
|
|
194
|
+
console.log('Successfully updated sync mapper record.', syncMapperRecordUpdate);
|
|
274
195
|
}
|
|
275
|
-
|
|
276
|
-
console.
|
|
196
|
+
catch (error) {
|
|
197
|
+
console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
|
|
277
198
|
return {
|
|
278
|
-
|
|
279
|
-
|
|
199
|
+
error: {
|
|
200
|
+
message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
|
|
280
201
|
},
|
|
281
202
|
};
|
|
282
203
|
}
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
204
|
+
return {
|
|
205
|
+
report: {
|
|
206
|
+
item_type: itemTypeToLoad.itemType,
|
|
207
|
+
[loading_1.ActionType.UPDATED]: 1,
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const response = await itemTypeToLoad.create(loadParams);
|
|
212
|
+
if (response.status !== 'success') {
|
|
213
|
+
return this.toFailedLoad(itemTypeToLoad.itemType, 'create', response);
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
const syncMapperRecordCreate = await this._mappers.create(Object.assign({ sync_unit: syncUnit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [response.id], targets: [devrevId] }, (response.modifiedDate && {
|
|
217
|
+
external_versions: [
|
|
218
|
+
{
|
|
219
|
+
modified_date: response.modifiedDate,
|
|
220
|
+
recipe_version: 0,
|
|
289
221
|
},
|
|
290
|
-
|
|
291
|
-
}
|
|
292
|
-
|
|
222
|
+
],
|
|
223
|
+
})));
|
|
224
|
+
console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
|
|
293
225
|
}
|
|
294
226
|
catch (error) {
|
|
295
|
-
|
|
296
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
297
|
-
// Create item in external system if mapper record not found
|
|
298
|
-
const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
|
|
299
|
-
item,
|
|
300
|
-
mappers: this._mappers,
|
|
301
|
-
event: this.event,
|
|
302
|
-
});
|
|
303
|
-
if (id) {
|
|
304
|
-
try {
|
|
305
|
-
const syncMapperRecordCreate = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
|
|
306
|
-
external_versions: [
|
|
307
|
-
{
|
|
308
|
-
modified_date: modifiedDate,
|
|
309
|
-
recipe_version: 0,
|
|
310
|
-
},
|
|
311
|
-
],
|
|
312
|
-
})));
|
|
313
|
-
console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
|
|
314
|
-
return {
|
|
315
|
-
report: {
|
|
316
|
-
item_type: itemTypeToLoad.itemType,
|
|
317
|
-
[loading_1.ActionType.CREATED]: 1,
|
|
318
|
-
},
|
|
319
|
-
};
|
|
320
|
-
}
|
|
321
|
-
catch (error) {
|
|
322
|
-
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
323
|
-
return {
|
|
324
|
-
error: {
|
|
325
|
-
message: 'Failed to create sync mapper record. ' +
|
|
326
|
-
(0, logger_1.serializeError)(error),
|
|
327
|
-
},
|
|
328
|
-
};
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
else if (delay) {
|
|
332
|
-
return {
|
|
333
|
-
rateLimit: {
|
|
334
|
-
delay,
|
|
335
|
-
},
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
else {
|
|
339
|
-
console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
|
|
340
|
-
return {
|
|
341
|
-
report: {
|
|
342
|
-
item_type: itemTypeToLoad.itemType,
|
|
343
|
-
[loading_1.ActionType.FAILED]: 1,
|
|
344
|
-
},
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
else {
|
|
349
|
-
console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
|
|
350
|
-
return {
|
|
351
|
-
error: {
|
|
352
|
-
message: error.message,
|
|
353
|
-
},
|
|
354
|
-
};
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
|
|
227
|
+
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
358
228
|
return {
|
|
359
229
|
error: {
|
|
360
|
-
message: 'Failed to
|
|
230
|
+
message: 'Failed to create sync mapper record. ' + (0, logger_1.serializeError)(error),
|
|
361
231
|
},
|
|
362
232
|
};
|
|
363
233
|
}
|
|
234
|
+
return {
|
|
235
|
+
report: {
|
|
236
|
+
item_type: itemTypeToLoad.itemType,
|
|
237
|
+
[loading_1.ActionType.CREATED]: 1,
|
|
238
|
+
},
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/** Maps a non-success connector response to a delay or a failed report. */
|
|
242
|
+
toFailedLoad(itemType, action, response) {
|
|
243
|
+
if (response.status === 'delay') {
|
|
244
|
+
console.log(`Rate limited during ${action} in external system, delaying for ${response.delaySeconds} seconds.`);
|
|
245
|
+
return { rateLimit: { delay: response.delaySeconds } };
|
|
246
|
+
}
|
|
247
|
+
console.warn(`Failed to ${action} item in external system.`, response.error.message);
|
|
248
|
+
return {
|
|
249
|
+
report: {
|
|
250
|
+
item_type: itemType,
|
|
251
|
+
[loading_1.ActionType.FAILED]: 1,
|
|
252
|
+
},
|
|
253
|
+
};
|
|
364
254
|
}
|
|
365
255
|
async loadAttachment({ item, create, }) {
|
|
366
|
-
const
|
|
256
|
+
const response = await create({
|
|
367
257
|
item,
|
|
368
258
|
mappers: this._mappers,
|
|
369
259
|
event: this.event,
|
|
370
260
|
});
|
|
371
|
-
if (
|
|
372
|
-
return
|
|
373
|
-
rateLimit: {
|
|
374
|
-
delay,
|
|
375
|
-
},
|
|
376
|
-
};
|
|
261
|
+
if (response.status !== 'success') {
|
|
262
|
+
return this.toFailedLoad('attachments', 'create', response);
|
|
377
263
|
}
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
|
|
387
|
-
}
|
|
388
|
-
catch (error) {
|
|
389
|
-
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
390
|
-
}
|
|
391
|
-
return {
|
|
392
|
-
report: {
|
|
393
|
-
item_type: 'attachments',
|
|
394
|
-
[loading_1.ActionType.CREATED]: 1,
|
|
395
|
-
},
|
|
396
|
-
};
|
|
264
|
+
try {
|
|
265
|
+
const syncMapperRecordCreate = await this._mappers.create({
|
|
266
|
+
sync_unit: this.event.payload.event_context.sync_unit,
|
|
267
|
+
external_ids: [response.id],
|
|
268
|
+
targets: [item.reference_id],
|
|
269
|
+
status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL,
|
|
270
|
+
});
|
|
271
|
+
console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
|
|
397
272
|
}
|
|
398
|
-
|
|
399
|
-
console.warn('Failed to create
|
|
400
|
-
return {
|
|
401
|
-
report: {
|
|
402
|
-
item_type: 'attachments',
|
|
403
|
-
[loading_1.ActionType.FAILED]: 1,
|
|
404
|
-
},
|
|
405
|
-
};
|
|
273
|
+
catch (error) {
|
|
274
|
+
console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
|
|
406
275
|
}
|
|
276
|
+
return {
|
|
277
|
+
report: {
|
|
278
|
+
item_type: 'attachments',
|
|
279
|
+
[loading_1.ActionType.CREATED]: 1,
|
|
280
|
+
},
|
|
281
|
+
};
|
|
407
282
|
}
|
|
408
283
|
}
|
|
409
284
|
exports.LoadingAdapter = LoadingAdapter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emit.d.ts","sourceRoot":"","sources":["../../src/multithreading/emit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAItC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,
|
|
1
|
+
{"version":3,"file":"emit.d.ts","sourceRoot":"","sources":["../../src/multithreading/emit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAItC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAClB,cAAc,EACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,eAAO,MAAM,IAAI,GAAU,8CAKxB,aAAa,KAAG,OAAO,CAAC,aAAa,CA+BvC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,oBAAoB,EAIrB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAuF5D,oDAAoD;AACpD,wBAAgB,qBAAqB,CAAC,cAAc,EAAE,EACpD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,QA6BzD;AAED,gDAAgD;AAChD,wBAAgB,kBAAkB,CAAC,cAAc,EAAE,EACjD,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,QA6BtD"}
|
|
@@ -13,22 +13,28 @@ const spawn_helpers_1 = require("./spawn/spawn.helpers");
|
|
|
13
13
|
/**
|
|
14
14
|
* Shared worker-thread driver: builds the adapter, runs the task, maps the
|
|
15
15
|
* returned {@link TaskResult} to a platform event and emits it exactly once.
|
|
16
|
-
* On soft timeout
|
|
17
|
-
* a phase-appropriate default)
|
|
18
|
-
*
|
|
16
|
+
* On soft timeout a returned `success`/`progress` is replaced by the
|
|
17
|
+
* `onTimeout` result (or a phase-appropriate default); a returned `error` or
|
|
18
|
+
* `delay` is emitted as is.
|
|
19
19
|
*/
|
|
20
20
|
async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
21
|
+
let adapter;
|
|
21
22
|
try {
|
|
22
|
-
|
|
23
|
+
adapter = await buildAdapter();
|
|
24
|
+
const workerAdapter = adapter;
|
|
23
25
|
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
|
|
24
26
|
if (message.subject !== workers_1.WorkerMessageSubject.WorkerMessageExit) {
|
|
25
27
|
return;
|
|
26
28
|
}
|
|
27
29
|
console.log('Timeout received. Waiting for the task to finish.');
|
|
28
|
-
|
|
30
|
+
workerAdapter.isTimeout = true;
|
|
29
31
|
});
|
|
32
|
+
console.log('Event passed from SDK to the connector:', adapter.event);
|
|
30
33
|
let result = await task({ adapter });
|
|
31
|
-
|
|
34
|
+
// A returned error or delay is a real outcome and must reach the platform;
|
|
35
|
+
// only a success/progress is replaced so a timed-out phase hands off.
|
|
36
|
+
const canBeReplaced = result.status === 'success' || result.status === 'progress';
|
|
37
|
+
if (adapter.isTimeout && canBeReplaced) {
|
|
32
38
|
if (onTimeout) {
|
|
33
39
|
result = await onTimeout({ adapter });
|
|
34
40
|
}
|
|
@@ -50,6 +56,18 @@ async function runWorkerTask(buildAdapter, { task, onTimeout }) {
|
|
|
50
56
|
process.exit(0);
|
|
51
57
|
}
|
|
52
58
|
catch (error) {
|
|
59
|
+
if (adapter && !adapter.hasWorkerEmitted) {
|
|
60
|
+
try {
|
|
61
|
+
await adapter.emitError(error);
|
|
62
|
+
}
|
|
63
|
+
catch (emitError) {
|
|
64
|
+
console.error('Error while emitting task failure.', (0, logger_1.serializeError)(emitError));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (adapter === null || adapter === void 0 ? void 0 : adapter.hasWorkerEmitted) {
|
|
68
|
+
process.exit(1);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
53
71
|
const errorMessage = `Error while processing task. ${(0, logger_1.serializeError)(error)}`;
|
|
54
72
|
console.error(errorMessage);
|
|
55
73
|
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { SpawnFactoryInterface, SpawnInterface } from '../../types/workers';
|
|
2
|
+
/**
|
|
3
|
+
* Local mode is opt-in through AIRSYNC_LOCAL_DEVELOPMENT only, and refused on
|
|
4
|
+
* Lambda so a stray flag can never change production behavior.
|
|
5
|
+
*/
|
|
6
|
+
export declare function isLocalDevelopment(env?: NodeJS.ProcessEnv): boolean;
|
|
2
7
|
/**
|
|
3
8
|
* Spawns a worker thread for the event and manages its lifecycle (control
|
|
4
9
|
* events, graceful exit, lambda timeout handling). Resolves when the worker
|
|
5
10
|
* run is fully finished.
|
|
6
11
|
*/
|
|
7
|
-
export declare function spawn<ConnectorState>({ event, initialState, initialDomainMapping, options, baseWorkerPath, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
|
|
12
|
+
export declare function spawn<ConnectorState>({ event, initialState, initialDomainMapping, options: spawnOptions, baseWorkerPath, workerPathOverrides, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
|
|
8
13
|
export declare class Spawn {
|
|
9
14
|
private event;
|
|
10
15
|
private alreadyEmitted;
|