@golemio/core 1.2.17-dev.623758334 → 1.2.18-dev.631214275
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integration-engine/data-access/StorageService.d.ts +10 -0
- package/dist/integration-engine/data-access/StorageService.js +68 -0
- package/dist/integration-engine/data-access/StorageService.js.map +1 -0
- package/dist/integration-engine/data-access/index.d.ts +1 -0
- package/dist/integration-engine/data-access/index.js +18 -0
- package/dist/integration-engine/data-access/index.js.map +1 -0
- package/dist/integration-engine/helpers/ContentTypeHelper.d.ts +1 -39
- package/dist/integration-engine/helpers/ContentTypeHelper.js.map +1 -1
- package/dist/integration-engine/helpers/RawDataStore.d.ts +1 -1
- package/dist/integration-engine/helpers/RawDataStore.js +38 -63
- package/dist/integration-engine/helpers/RawDataStore.js.map +1 -1
- package/dist/integration-engine/workers/AbstractEmptyTask.d.ts +1 -2
- package/dist/integration-engine/workers/AbstractEmptyTask.js +2 -2
- package/dist/integration-engine/workers/AbstractEmptyTask.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTask.d.ts +3 -3
- package/dist/integration-engine/workers/AbstractTask.js +7 -38
- package/dist/integration-engine/workers/AbstractTask.js.map +1 -1
- package/dist/integration-engine/workers/helpers/MessageDataParser.d.ts +5 -0
- package/dist/integration-engine/workers/helpers/MessageDataParser.js +27 -0
- package/dist/integration-engine/workers/helpers/MessageDataParser.js.map +1 -0
- package/dist/integration-engine/workers/helpers/MessageDataValidator.d.ts +4 -0
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js +44 -0
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js.map +1 -0
- package/package.json +3 -2
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
declare class StorageService {
|
|
2
|
+
private readonly s3;
|
|
3
|
+
constructor();
|
|
4
|
+
/**
|
|
5
|
+
* Save data to our storage
|
|
6
|
+
*/
|
|
7
|
+
saveData(data: string, pathPrefix: string, ext?: string, fileName?: string): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export declare const storageService: StorageService;
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.storageService = void 0;
|
|
16
|
+
const aws_sdk_1 = require("aws-sdk");
|
|
17
|
+
const moment_timezone_1 = __importDefault(require("moment-timezone"));
|
|
18
|
+
const config_1 = require("../config");
|
|
19
|
+
const helpers_1 = require("../helpers");
|
|
20
|
+
class StorageService {
|
|
21
|
+
constructor() {
|
|
22
|
+
this.s3 = new aws_sdk_1.S3({
|
|
23
|
+
accessKeyId: config_1.config.s3.access_key_id,
|
|
24
|
+
endpoint: config_1.config.s3.endpoint,
|
|
25
|
+
s3ForcePathStyle: true,
|
|
26
|
+
secretAccessKey: config_1.config.s3.secret_access_key,
|
|
27
|
+
signatureVersion: "v4",
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Save data to our storage
|
|
32
|
+
*/
|
|
33
|
+
saveData(data, pathPrefix, ext = "", fileName) {
|
|
34
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
35
|
+
if (!config_1.config.s3.enabled || !config_1.config.s3.bucket_name) {
|
|
36
|
+
helpers_1.log.silly("[S3] Service is disabled. Skipping");
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const now = (0, moment_timezone_1.default)().tz("Europe/Prague");
|
|
40
|
+
const fileNamePrefix = fileName !== null && fileName !== void 0 ? fileName : `${now.format("YYYY-MM-DD")}/${now.format("HH_mm_ss.SSS")}`;
|
|
41
|
+
const fileNameKey = `${pathPrefix}/${fileNamePrefix}`;
|
|
42
|
+
const fileNameExt = ext ? `.${ext}` : "";
|
|
43
|
+
const paramsBodyData = {
|
|
44
|
+
Body: data,
|
|
45
|
+
Bucket: config_1.config.s3.bucket_name,
|
|
46
|
+
Key: `${fileNameKey}${fileNameExt}`,
|
|
47
|
+
};
|
|
48
|
+
const options = {
|
|
49
|
+
partSize: config_1.config.s3.upload_part_size * 1024 * 1024,
|
|
50
|
+
queueSize: config_1.config.s3.upload_queue_size,
|
|
51
|
+
};
|
|
52
|
+
try {
|
|
53
|
+
yield this.s3
|
|
54
|
+
.upload(paramsBodyData, options)
|
|
55
|
+
.on("httpUploadProgress", (progress) => {
|
|
56
|
+
helpers_1.log.verbose(`[S3] Data S3 upload ` +
|
|
57
|
+
`\`${config_1.config.s3.bucket_name}/${fileName}: \`${progress.loaded} / ${progress.total}.`);
|
|
58
|
+
})
|
|
59
|
+
.promise();
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
helpers_1.log.error(`[S3] Saving of data failed (${fileName})`, err);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.storageService = new StorageService();
|
|
68
|
+
//# sourceMappingURL=StorageService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"StorageService.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/StorageService.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,qCAA6B;AAC7B,sEAAqC;AACrC,sCAAoC;AACpC,wCAAkC;AAElC,MAAM,cAAc;IAGhB;QACI,IAAI,CAAC,EAAE,GAAG,IAAI,YAAE,CAAC;YACb,WAAW,EAAE,eAAM,CAAC,EAAE,CAAC,aAAa;YACpC,QAAQ,EAAE,eAAM,CAAC,EAAE,CAAC,QAAQ;YAC5B,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,eAAM,CAAC,EAAE,CAAC,iBAAiB;YAC5C,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACU,QAAQ,CAAC,IAAY,EAAE,UAAkB,EAAE,MAAc,EAAE,EAAE,QAAiB;;YACvF,IAAI,CAAC,eAAM,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,eAAM,CAAC,EAAE,CAAC,WAAW,EAAE;gBAC9C,aAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBAChD,OAAO;aACV;YAED,MAAM,GAAG,GAAG,IAAA,yBAAM,GAAE,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;YACzC,MAAM,cAAc,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/F,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEzC,MAAM,cAAc,GAAwB;gBACxC,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,eAAM,CAAC,EAAE,CAAC,WAAW;gBAC7B,GAAG,EAAE,GAAG,WAAW,GAAG,WAAW,EAAE;aACtC,CAAC;YAEF,MAAM,OAAO,GAAG;gBACZ,QAAQ,EAAE,eAAM,CAAC,EAAE,CAAC,gBAAgB,GAAG,IAAI,GAAG,IAAI;gBAClD,SAAS,EAAE,eAAM,CAAC,EAAE,CAAC,iBAAiB;aACzC,CAAC;YAEF,IAAI;gBACA,MAAM,IAAI,CAAC,EAAE;qBACR,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC;qBAC/B,EAAE,CAAC,oBAAoB,EAAE,CAAC,QAAQ,EAAE,EAAE;oBACnC,aAAG,CAAC,OAAO,CACP,sBAAsB;wBAClB,KAAK,eAAM,CAAC,EAAE,CAAC,WAAW,IAAI,QAAQ,OAAO,QAAQ,CAAC,MAAM,MAAM,QAAQ,CAAC,KAAK,GAAG,CAC1F,CAAC;gBACN,CAAC,CAAC;qBACD,OAAO,EAAE,CAAC;aAClB;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,KAAK,CAAC,+BAA+B,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;aAC9D;QACL,CAAC;KAAA;CACJ;AAEY,QAAA,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./StorageService";
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./StorageService"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,mDAAiC"}
|
|
@@ -1,39 +1 @@
|
|
|
1
|
-
export declare const contentTypeToExtension:
|
|
2
|
-
"application/geo+json": string;
|
|
3
|
-
"application/gnutar": string;
|
|
4
|
-
"application/gzip": string;
|
|
5
|
-
"application/javascript": string;
|
|
6
|
-
"application/json": string;
|
|
7
|
-
"application/ld+json": string;
|
|
8
|
-
"application/rar": string;
|
|
9
|
-
"application/tar": string;
|
|
10
|
-
"application/tar+gzip": string;
|
|
11
|
-
"application/vnd.ms-excel": string;
|
|
12
|
-
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": string;
|
|
13
|
-
"application/x-compressed": string;
|
|
14
|
-
"application/x-gzip": string;
|
|
15
|
-
"application/x-javascript": string;
|
|
16
|
-
"application/x-json": string;
|
|
17
|
-
"application/x-rar": string;
|
|
18
|
-
"application/x-rar-compressed": string;
|
|
19
|
-
"application/x-tar": string;
|
|
20
|
-
"application/x-zip": string;
|
|
21
|
-
"application/x-zip-compressed": string;
|
|
22
|
-
"application/xml": string;
|
|
23
|
-
"application/zip": string;
|
|
24
|
-
"image/gif": string;
|
|
25
|
-
"image/jpeg": string;
|
|
26
|
-
"image/png": string;
|
|
27
|
-
"image/tiff": string;
|
|
28
|
-
"multipart/x-gzip": string;
|
|
29
|
-
"multipart/x-zip": string;
|
|
30
|
-
"text/csv": string;
|
|
31
|
-
"text/html": string;
|
|
32
|
-
"text/javascript": string;
|
|
33
|
-
"text/json": string;
|
|
34
|
-
"text/plain": string;
|
|
35
|
-
"text/tab-separated-values": string;
|
|
36
|
-
"text/x-javascript": string;
|
|
37
|
-
"text/x-json": string;
|
|
38
|
-
"text/xml": string;
|
|
39
|
-
};
|
|
1
|
+
export declare const contentTypeToExtension: Record<string, string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ContentTypeHelper.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/ContentTypeHelper.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,
|
|
1
|
+
{"version":3,"file":"ContentTypeHelper.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/ContentTypeHelper.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAA2B;IAC1D,sBAAsB,EAAE,MAAM;IAC9B,oBAAoB,EAAE,KAAK;IAC3B,kBAAkB,EAAE,IAAI;IACxB,wBAAwB,EAAE,MAAM;IAChC,kBAAkB,EAAE,MAAM;IAC1B,qBAAqB,EAAE,QAAQ;IAC/B,iBAAiB,EAAE,KAAK;IACxB,iBAAiB,EAAE,KAAK;IACxB,sBAAsB,EAAE,QAAQ;IAChC,0BAA0B,EAAE,KAAK;IACjC,mEAAmE,EAAE,MAAM;IAC3E,0BAA0B,EAAE,KAAK;IACjC,oBAAoB,EAAE,IAAI;IAC1B,0BAA0B,EAAE,MAAM;IAClC,oBAAoB,EAAE,MAAM;IAC5B,mBAAmB,EAAE,KAAK;IAC1B,8BAA8B,EAAE,KAAK;IACrC,mBAAmB,EAAE,KAAK;IAC1B,mBAAmB,EAAE,KAAK;IAC1B,8BAA8B,EAAE,KAAK;IACrC,iBAAiB,EAAE,KAAK;IACxB,iBAAiB,EAAE,KAAK;IACxB,WAAW,EAAE,KAAK;IAClB,YAAY,EAAE,KAAK;IACnB,WAAW,EAAE,KAAK;IAClB,YAAY,EAAE,MAAM;IACpB,kBAAkB,EAAE,IAAI;IACxB,iBAAiB,EAAE,KAAK;IACxB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,MAAM;IACnB,iBAAiB,EAAE,MAAM;IACzB,WAAW,EAAE,MAAM;IACnB,YAAY,EAAE,KAAK;IACnB,2BAA2B,EAAE,KAAK;IAClC,mBAAmB,EAAE,MAAM;IAC3B,aAAa,EAAE,MAAM;IACrB,UAAU,EAAE,KAAK;CACpB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const save: (data: any, meta: any, name?: string) => void
|
|
1
|
+
export declare const save: (data: any, meta: any, name?: string) => Promise<void>;
|
|
@@ -1,80 +1,55 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
2
11
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
12
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
13
|
};
|
|
5
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
15
|
exports.save = void 0;
|
|
7
|
-
const aws_sdk_1 = require("aws-sdk");
|
|
8
16
|
const content_type_1 = __importDefault(require("content-type"));
|
|
9
17
|
const moment_timezone_1 = __importDefault(require("moment-timezone"));
|
|
10
18
|
const helpers_1 = require("./");
|
|
11
19
|
const config_1 = require("../config");
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}
|
|
28
|
-
catch (_a) {
|
|
29
|
-
null;
|
|
30
|
-
}
|
|
31
|
-
try {
|
|
32
|
-
// @ts-ignore
|
|
33
|
-
ext = `.${helpers_1.contentTypeToExtension[(content_type_1.default.parse(meta.headers["content-type"]) || {}).type]}`;
|
|
34
|
-
}
|
|
35
|
-
catch (_b) {
|
|
36
|
-
null;
|
|
37
|
-
}
|
|
20
|
+
const data_access_1 = require("../data-access");
|
|
21
|
+
const save = (data, meta, name = "Unknown") => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
|
+
if (!config_1.config.s3.enabled || !config_1.config.saveRawDataWhitelist[name]) {
|
|
23
|
+
helpers_1.log.verbose(`Raw data S3 upload not enabled for \`${name}\` datasource.`);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const now = (0, moment_timezone_1.default)().tz("Europe/Prague");
|
|
27
|
+
let ext = null;
|
|
28
|
+
let headers = null;
|
|
29
|
+
if (meta.headers) {
|
|
30
|
+
try {
|
|
31
|
+
headers = JSON.stringify(meta.headers, null, 2);
|
|
32
|
+
}
|
|
33
|
+
catch (_a) {
|
|
34
|
+
null;
|
|
38
35
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
Key: `${fileName}${ext || ""}`,
|
|
44
|
-
};
|
|
45
|
-
const paramsBodyHeaders = {
|
|
46
|
-
Body: headers,
|
|
47
|
-
Bucket: config_1.config.s3.bucket_name,
|
|
48
|
-
Key: `${fileName}_headers.json`,
|
|
49
|
-
};
|
|
50
|
-
const options = {
|
|
51
|
-
partSize: config_1.config.s3.upload_part_size * 1024 * 1024,
|
|
52
|
-
queueSize: config_1.config.s3.upload_queue_size,
|
|
53
|
-
};
|
|
54
|
-
const uploadData = s3.upload(paramsBodyData, options, (err) => {
|
|
55
|
-
if (err) {
|
|
56
|
-
helpers_1.log.error(`Saving raw data failed. (${fileName})`, { error: err });
|
|
36
|
+
try {
|
|
37
|
+
const parsedMediaType = content_type_1.default.parse(meta.headers["content-type"]);
|
|
38
|
+
if (parsedMediaType === null || parsedMediaType === void 0 ? void 0 : parsedMediaType.type) {
|
|
39
|
+
ext = helpers_1.contentTypeToExtension[parsedMediaType.type];
|
|
57
40
|
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
${config_1.config.s3.bucket_name}/${fileName}: \`${progress.loaded} / ${progress.total}.`);
|
|
62
|
-
});
|
|
63
|
-
if (config_1.config.saveRawDataWhitelist[name].saveHeaders && headers && meta.statusCode && meta.statusCode !== 200) {
|
|
64
|
-
const uploadHeaders = s3.upload(paramsBodyHeaders, options, (err) => {
|
|
65
|
-
if (err) {
|
|
66
|
-
helpers_1.log.error(`Saving raw data headers failed. (${fileName}.headers)`, { error: err });
|
|
67
|
-
}
|
|
68
|
-
});
|
|
69
|
-
uploadHeaders.on("httpUploadProgress", (progress) => {
|
|
70
|
-
helpers_1.log.verbose(`Raw data headers S3 upload \`
|
|
71
|
-
${config_1.config.s3.bucket_name}/${fileName}.headers: \`${progress.loaded} / ${progress.total}.`);
|
|
72
|
-
});
|
|
41
|
+
}
|
|
42
|
+
catch (_b) {
|
|
43
|
+
null;
|
|
73
44
|
}
|
|
74
45
|
}
|
|
75
|
-
|
|
76
|
-
|
|
46
|
+
const fileName = `${now.format("YYYY-MM-DD")}/${now.format("HH_mm_ss.SSS")}`;
|
|
47
|
+
// Save data
|
|
48
|
+
yield data_access_1.storageService.saveData(data, name, ext !== null && ext !== void 0 ? ext : "", fileName);
|
|
49
|
+
if (config_1.config.saveRawDataWhitelist[name].saveHeaders && headers && meta.statusCode && meta.statusCode !== 200) {
|
|
50
|
+
// Save headers
|
|
51
|
+
yield data_access_1.storageService.saveData(headers, name, "json", `${fileName}_headers`);
|
|
77
52
|
}
|
|
78
|
-
};
|
|
53
|
+
});
|
|
79
54
|
exports.save = save;
|
|
80
55
|
//# sourceMappingURL=RawDataStore.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RawDataStore.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/RawDataStore.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"RawDataStore.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/RawDataStore.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,gEAAuC;AACvC,sEAAqC;AACrC,gCAA0D;AAC1D,sCAAoC;AACpC,gDAAiD;AAE1C,MAAM,IAAI,GAAG,CAAO,IAAS,EAAE,IAAS,EAAE,IAAI,GAAG,SAAS,EAAE,EAAE;IACjE,IAAI,CAAC,eAAM,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,eAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE;QAC1D,aAAG,CAAC,OAAO,CAAC,wCAAwC,IAAI,gBAAgB,CAAC,CAAC;QAC1E,OAAO;KACV;IAED,MAAM,GAAG,GAAG,IAAA,yBAAM,GAAE,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,IAAI,IAAI,CAAC,OAAO,EAAE;QACd,IAAI;YACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;SACnD;QAAC,WAAM;YACJ,IAAI,CAAC;SACR;QACD,IAAI;YACA,MAAM,eAAe,GAAG,sBAAW,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;YACxE,IAAI,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,IAAI,EAAE;gBACvB,GAAG,GAAG,gCAAsB,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;aACtD;SACJ;QAAC,WAAM;YACJ,IAAI,CAAC;SACR;KACJ;IAED,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;IAE7E,YAAY;IACZ,MAAM,4BAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;IAE/D,IAAI,eAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,KAAK,GAAG,EAAE;QACxG,eAAe;QACf,MAAM,4BAAc,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC;KAC/E;AACL,CAAC,CAAA,CAAC;AAnCW,QAAA,IAAI,QAmCf"}
|
|
@@ -3,8 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.AbstractEmptyTask = void 0;
|
|
4
4
|
const AbstractTask_1 = require("./AbstractTask");
|
|
5
5
|
class AbstractEmptyTask extends AbstractTask_1.AbstractTask {
|
|
6
|
-
constructor(
|
|
7
|
-
super(
|
|
6
|
+
constructor() {
|
|
7
|
+
super(...arguments);
|
|
8
8
|
this.schema = undefined;
|
|
9
9
|
}
|
|
10
10
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractEmptyTask.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/AbstractEmptyTask.ts"],"names":[],"mappings":";;;AAAA,iDAA8C;AAE9C,MAAsB,iBAAkB,SAAQ,2BAAoB;
|
|
1
|
+
{"version":3,"file":"AbstractEmptyTask.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/AbstractEmptyTask.ts"],"names":[],"mappings":";;;AAAA,iDAA8C;AAE9C,MAAsB,iBAAkB,SAAQ,2BAAoB;IAApE;;QACc,WAAM,GAAG,SAAS,CAAC;IACjC,CAAC;CAAA;AAFD,8CAEC"}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { Message } from "amqplib";
|
|
1
|
+
import { Message, MessageProperties } from "amqplib";
|
|
2
2
|
import { ClassConstructor } from "class-transformer";
|
|
3
3
|
import { ITask } from "./interfaces";
|
|
4
4
|
export declare abstract class AbstractTask<TRawData extends object> implements ITask {
|
|
5
5
|
abstract queueName: string;
|
|
6
6
|
queueTtl?: number;
|
|
7
7
|
queuePrefix: string;
|
|
8
|
-
protected abstract schema
|
|
8
|
+
protected abstract schema: ClassConstructor<TRawData> | undefined;
|
|
9
9
|
constructor(queuePrefix: string);
|
|
10
10
|
validateAndExecute(msg: Message | null): Promise<void>;
|
|
11
11
|
protected abstract execute(data: TRawData): void | Promise<void>;
|
|
12
|
-
|
|
12
|
+
protected abstract execute(data: TRawData, msgProperties?: MessageProperties): void | Promise<void>;
|
|
13
13
|
}
|
|
@@ -10,52 +10,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.AbstractTask = void 0;
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const errors_1 = require("@golemio/errors");
|
|
16
|
-
const helpers_1 = require("../helpers");
|
|
13
|
+
const MessageDataParser_1 = require("./helpers/MessageDataParser");
|
|
14
|
+
const MessageDataValidator_1 = require("./helpers/MessageDataValidator");
|
|
17
15
|
class AbstractTask {
|
|
18
16
|
constructor(queuePrefix) {
|
|
19
17
|
this.queuePrefix = queuePrefix;
|
|
20
18
|
}
|
|
21
19
|
validateAndExecute(msg) {
|
|
22
|
-
var _a;
|
|
23
20
|
return __awaiter(this, void 0, void 0, function* () {
|
|
24
21
|
if (!this.schema) {
|
|
25
|
-
return this.execute({});
|
|
26
|
-
}
|
|
27
|
-
let msgContent = "";
|
|
28
|
-
let data;
|
|
29
|
-
try {
|
|
30
|
-
msgContent = (_a = msg === null || msg === void 0 ? void 0 : msg.content.toString()) !== null && _a !== void 0 ? _a : "";
|
|
31
|
-
data = (0, class_transformer_1.plainToInstance)(this.schema, JSON.parse(msgContent));
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
const exception = new errors_1.CustomError(`[${this.constructor.name}] Message parsing failed: ${`'${msgContent.substring(0, 2000)}'`}`, true, this.constructor.name, undefined, err);
|
|
35
|
-
helpers_1.log.error(exception);
|
|
36
|
-
throw exception;
|
|
37
|
-
}
|
|
38
|
-
yield this.validate(data);
|
|
39
|
-
return this.execute(data);
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
validate(data) {
|
|
43
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
44
|
-
let errors = [];
|
|
45
|
-
try {
|
|
46
|
-
errors = yield (0, class_validator_1.validate)(data);
|
|
47
|
-
}
|
|
48
|
-
catch (err) {
|
|
49
|
-
const exception = new errors_1.CustomError(`[${this.constructor.name}] Message validation failed: ${err.message}`, true, this.constructor.name, undefined, err);
|
|
50
|
-
helpers_1.log.error(exception);
|
|
51
|
-
throw exception;
|
|
52
|
-
}
|
|
53
|
-
if (errors.length > 0) {
|
|
54
|
-
const constraints = errors.flatMap(({ constraints }) => (constraints ? Object.values(constraints) : []));
|
|
55
|
-
const exception = new errors_1.CustomError(`[${this.constructor.name}] Message validation failed: [${constraints.join(", ")}]`, true, this.constructor.name);
|
|
56
|
-
helpers_1.log.error(exception);
|
|
57
|
-
throw exception;
|
|
22
|
+
return this.execute({}, msg === null || msg === void 0 ? void 0 : msg.properties);
|
|
58
23
|
}
|
|
24
|
+
const queueKey = `${this.queuePrefix}.${this.queueName}`;
|
|
25
|
+
const data = MessageDataParser_1.MessageDataParser.parse(queueKey, msg, this.schema);
|
|
26
|
+
yield MessageDataValidator_1.MessageDataValidator.validate(queueKey, data);
|
|
27
|
+
return this.execute(data, msg === null || msg === void 0 ? void 0 : msg.properties);
|
|
59
28
|
});
|
|
60
29
|
}
|
|
61
30
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractTask.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/AbstractTask.ts"],"names":[],"mappings":";;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"AbstractTask.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/AbstractTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,mEAAgE;AAChE,yEAAsE;AAGtE,MAAsB,YAAY;IAM9B,YAAY,WAAmB;QAC3B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAEY,kBAAkB,CAAC,GAAmB;;YAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBACd,OAAO,IAAI,CAAC,OAAO,CAAC,EAAc,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,UAAU,CAAC,CAAC;aACxD;YAED,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,qCAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,2CAAoB,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEpD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,UAAU,CAAC,CAAC;QAC/C,CAAC;KAAA;CAIJ;AAxBD,oCAwBC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MessageDataParser = void 0;
|
|
4
|
+
const class_transformer_1 = require("class-transformer");
|
|
5
|
+
const errors_1 = require("@golemio/errors");
|
|
6
|
+
const helpers_1 = require("../../helpers");
|
|
7
|
+
const data_access_1 = require("../../data-access");
|
|
8
|
+
class MessageDataParser {
|
|
9
|
+
static parse(queueKey, msg, schema) {
|
|
10
|
+
var _a;
|
|
11
|
+
let msgContent = "";
|
|
12
|
+
let data;
|
|
13
|
+
try {
|
|
14
|
+
msgContent = (_a = msg === null || msg === void 0 ? void 0 : msg.content.toString()) !== null && _a !== void 0 ? _a : "";
|
|
15
|
+
data = (0, class_transformer_1.plainToInstance)(schema, JSON.parse(msgContent));
|
|
16
|
+
}
|
|
17
|
+
catch (err) {
|
|
18
|
+
const exception = new errors_1.CustomError(`[Queue ${queueKey}] Message parsing failed: ${`'${msgContent.substring(0, 2000)}'`}`, true, "MessageDataParser", undefined, err);
|
|
19
|
+
helpers_1.log.error(exception);
|
|
20
|
+
data_access_1.storageService.saveData(JSON.stringify({ exception: exception.toObject(), msgContent }), `validation-errors/${queueKey}`, "json");
|
|
21
|
+
throw exception;
|
|
22
|
+
}
|
|
23
|
+
return data;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
exports.MessageDataParser = MessageDataParser;
|
|
27
|
+
//# sourceMappingURL=MessageDataParser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MessageDataParser.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/helpers/MessageDataParser.ts"],"names":[],"mappings":";;;AACA,yDAAsE;AACtE,4CAA8C;AAC9C,2CAAkC;AAClC,mDAAiD;AAEjD,MAAa,iBAAiB;IACnB,MAAM,CAAC,KAAK,CACf,QAAgB,EAChB,GAAmB,EACnB,MAAkC;;QAElC,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,IAAc,CAAC;QAEnB,IAAI;YACA,UAAU,GAAG,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,QAAQ,EAAE,mCAAI,EAAE,CAAC;YAC3C,IAAI,GAAG,IAAA,mCAAe,EAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;SAC1D;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,SAAS,GAAG,IAAI,oBAAW,CAC7B,UAAU,QAAQ,6BAA6B,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,EACrF,IAAI,EACJ,mBAAmB,EACnB,SAAS,EACT,GAAG,CACN,CAAC;YAEF,aAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACrB,4BAAc,CAAC,QAAQ,CACnB,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAC/D,qBAAqB,QAAQ,EAAE,EAC/B,MAAM,CACT,CAAC;YAEF,MAAM,SAAS,CAAC;SACnB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AAjCD,8CAiCC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.MessageDataValidator = void 0;
|
|
13
|
+
const class_validator_1 = require("class-validator");
|
|
14
|
+
const errors_1 = require("@golemio/errors");
|
|
15
|
+
const helpers_1 = require("../../helpers");
|
|
16
|
+
const data_access_1 = require("../../data-access");
|
|
17
|
+
class MessageDataValidator {
|
|
18
|
+
static validate(queueKey, data) {
|
|
19
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
20
|
+
let validationErrors = [];
|
|
21
|
+
try {
|
|
22
|
+
validationErrors = yield (0, class_validator_1.validate)(data);
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
const exception = new errors_1.CustomError(`[Queue ${queueKey}] Message validation failed: ${err.message}`, true, "MessageDataValidator", undefined, err);
|
|
26
|
+
helpers_1.log.error(exception);
|
|
27
|
+
data_access_1.storageService.saveData(JSON.stringify({ exception: exception.toObject(), data }), `validation-errors/${queueKey}`, "json");
|
|
28
|
+
throw exception;
|
|
29
|
+
}
|
|
30
|
+
if (validationErrors.length > 0) {
|
|
31
|
+
const constraints = this.traverseValidationErrors(validationErrors);
|
|
32
|
+
const exception = new errors_1.CustomError(`[Queue ${queueKey}] Message validation failed: [${constraints.slice(0, 10).join(", ")}]`, true, "MessageDataValidator");
|
|
33
|
+
helpers_1.log.error(exception);
|
|
34
|
+
data_access_1.storageService.saveData(JSON.stringify({ validationErrors }), `validation-errors/${queueKey}`, "json");
|
|
35
|
+
throw exception;
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
static traverseValidationErrors(errors) {
|
|
40
|
+
return errors.flatMap(({ constraints, children }) => constraints ? Object.values(constraints) : children ? this.traverseValidationErrors(children) : []);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.MessageDataValidator = MessageDataValidator;
|
|
44
|
+
//# sourceMappingURL=MessageDataValidator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MessageDataValidator.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/helpers/MessageDataValidator.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qDAA4D;AAC5D,4CAA8C;AAC9C,2CAAkC;AAClC,mDAAiD;AAEjD,MAAa,oBAAoB;IACtB,MAAM,CAAO,QAAQ,CAAC,QAAgB,EAAE,IAAY;;YACvD,IAAI,gBAAgB,GAAsB,EAAE,CAAC;YAC7C,IAAI;gBACA,gBAAgB,GAAG,MAAM,IAAA,0BAAQ,EAAC,IAAI,CAAC,CAAC;aAC3C;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,SAAS,GAAG,IAAI,oBAAW,CAC7B,UAAU,QAAQ,gCAAgC,GAAG,CAAC,OAAO,EAAE,EAC/D,IAAI,EACJ,sBAAsB,EACtB,SAAS,EACT,GAAG,CACN,CAAC;gBAEF,aAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACrB,4BAAc,CAAC,QAAQ,CACnB,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EACzD,qBAAqB,QAAQ,EAAE,EAC/B,MAAM,CACT,CAAC;gBAEF,MAAM,SAAS,CAAC;aACnB;YAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;gBACpE,MAAM,SAAS,GAAG,IAAI,oBAAW,CAC7B,UAAU,QAAQ,iCAAiC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EACzF,IAAI,EACJ,sBAAsB,CACzB,CAAC;gBAEF,aAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACrB,4BAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,qBAAqB,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;gBAEvG,MAAM,SAAS,CAAC;aACnB;QACL,CAAC;KAAA;IAEO,MAAM,CAAC,wBAAwB,CAAC,MAAyB;QAC7D,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE,CAChD,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CACrG,CAAC;IACN,CAAC;CACJ;AA5CD,oDA4CC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/core",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.18-dev.631214275",
|
|
4
4
|
"description": "Golemio Core Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -54,6 +54,7 @@
|
|
|
54
54
|
"nyc": "^15.1.0",
|
|
55
55
|
"prettier": "^2.5.1",
|
|
56
56
|
"pretty-quick": "^3.1.0",
|
|
57
|
+
"reflect-metadata": "^0.1.13",
|
|
57
58
|
"rimraf": "^3.0.2",
|
|
58
59
|
"sinon": "7.1.1",
|
|
59
60
|
"source-map-support": "0.5.9",
|
|
@@ -66,7 +67,7 @@
|
|
|
66
67
|
"dependencies": {
|
|
67
68
|
"@golemio/errors": "^1.0.1",
|
|
68
69
|
"@golemio/utils": "^1.0.0",
|
|
69
|
-
"@golemio/validator": "0.2.4
|
|
70
|
+
"@golemio/validator": "0.2.4",
|
|
70
71
|
"@google-cloud/storage": "^5.18.0",
|
|
71
72
|
"@opentelemetry/api": "1.0.4",
|
|
72
73
|
"@opentelemetry/exporter-jaeger": "^1.2.0",
|