@hestia-earth/pipeline-utils 0.13.0 → 0.13.2

package/dist/sqs.js

@@ -7,7 +7,8 @@ const operators_1 = require("rxjs/operators");
 const log_1 = require("./log");
 const s3_1 = require("./s3");
 const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL, extras = {}) => s3_1.isS3Mode
-    ? (0, rxjs_1.from)(new AWS.SQS({ region: s3_1.region }).sendMessage({
+    ? (0, rxjs_1.from)(new AWS.SQS({ region: s3_1.region })
+        .sendMessage({
         QueueUrl: queueUrl,
         MessageBody: JSON.stringify(params),
         MessageAttributes: {
@@ -15,15 +16,16 @@ const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL, extra
                 DataType: 'String',
                 StringValue: functionName
             },
-            ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
+            ...Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
                 key,
                 {
                     DataType: 'String',
                     StringValue: value
                 }
-            ])))
+            ]))
         }
-    }).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
+    })
+        .promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
         (0, log_1.debug)(`SQS send message ${functionName}`, res);
         return res;
     }))

package/dist/sqs.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAwD;AAMjD,MAAM,WAAW,GAAG,CACzB,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,SAAqB,EAAE,EACrG,EAAE,CAAC,aAAQ;IACX,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC,CAAC,WAAW,CAAC;QAClC,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACxE,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CAAC,CAAC;SACL;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,oBAAoB,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACD,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AA5BA,QAAA,WAAW,eA4BX"}
+{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAwD;AAMjD,MAAM,WAAW,GAAG,CACzB,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAC9B,SAAqB,EAAE,EACvB,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC;SACpB,WAAW,CAAC;QACX,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;KACF,CAAC;SACD,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,oBAAoB,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AApCF,QAAA,WAAW,eAoCT"}

package/dist/utils.js

@@ -3,15 +3,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.stripBOM = exports.prependWithKey = exports.refToSchemaType = exports.isEmpty = exports.spliceAndProcess = exports.returnError = exports.returnResponse = exports.functionParamToSNSRecord = exports.functionParamToS3Record = exports.recordToFunctionParam = void 0;
 const utils_1 = require("@hestia-earth/utils");
 const log_1 = require("./log");
-const recordToFunctionParam = (event) => (0, utils_1.reduceUndefinedValues)('Sns' in event ? {
-    ...JSON.parse(event.Sns.Message),
-    timestamp: event.Sns.Timestamp ? new Date(event.Sns.Timestamp) : undefined
-} : {
-    bucket: event.s3.bucket.name,
-    key: event.s3.object.key,
-    filepath: event.s3.object.key,
-    timestamp: event.eventTime ? new Date(event.eventTime) : undefined
-});
+const recordToFunctionParam = (event) => (0, utils_1.reduceUndefinedValues)('Sns' in event
+    ? {
+        ...JSON.parse(event.Sns.Message),
+        timestamp: event.Sns.Timestamp ? new Date(event.Sns.Timestamp) : undefined
+    }
+    : {
+        bucket: event.s3.bucket.name,
+        key: event.s3.object.key,
+        filepath: event.s3.object.key,
+        timestamp: event.eventTime ? new Date(event.eventTime) : undefined
+    });
 exports.recordToFunctionParam = recordToFunctionParam;
 const functionParamToS3Record = ({ bucket, key, timestamp }) => ({
     s3: {
@@ -40,7 +42,7 @@ const returnResponse = (data, statusCode = 200) => {
 };
 exports.returnResponse = returnResponse;
 const defaultErrorStatus = 500;
-const returnError = (err) => {
+const returnError = err => {
     (0, log_1.error)('handleError', err, err === null || err === void 0 ? void 0 : err.stack, err === null || err === void 0 ? void 0 : err.meta);
     const { error: errorContent, status = defaultErrorStatus } = typeof err === 'string'
         ? { error: err }
@@ -55,20 +57,21 @@ const spliceAndProcess = (docs, func, count = 1) => new Promise(resolve => {
         return resolve();
     }
     const remaining = docs.splice(docs.length - count);
-    void func(remaining).then(() => (0, exports.spliceAndProcess)(docs, func, count)).then(resolve);
+    void func(remaining)
+        .then(() => (0, exports.spliceAndProcess)(docs, func, count))
+        .then(resolve);
 });
 exports.spliceAndProcess = spliceAndProcess;
-const isEmpty = (value, minKeys = 1) => typeof value === 'object' ?
-    (Array.isArray(value) ?
-        !value.length :
-        Object.keys(value).filter(key => key !== 'type').length < minKeys) :
-    typeof value === 'undefined' || value === null || value === '';
+const isEmpty = (value, minKeys = 1) => typeof value === 'object'
+    ? Array.isArray(value)
+        ? !value.length
+        : Object.keys(value).filter(key => key !== 'type').length < minKeys
+    : typeof value === 'undefined' || value === null || value === '';
 exports.isEmpty = isEmpty;
 const refToSchemaType = ref => ref.substring(2).replace('.json#', '');
 exports.refToSchemaType = refToSchemaType;
-const prependWithKey = (data, key) => Object.keys(data)
-    .reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
+const prependWithKey = (data, key) => Object.keys(data).reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
 exports.prependWithKey = prependWithKey;
-const stripBOM = (content) => content.charCodeAt(0) === 0xFEFF ? content.slice(1) : content;
+const stripBOM = (content) => (content.charCodeAt(0) === 0xfeff ? content.slice(1) : content);
 exports.stripBOM = stripBOM;
 //# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AACA,+CAA4D;AAG5D,+BAA8B;AAEvB,MAAM,qBAAqB,GAAG,CAAC,KAAqC,EAAkB,EAAE,CAC7F,IAAA,6BAAqB,EACnB,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC;IACf,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;IAChC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CAC3E,CAAC,CAAC,CAAC;IACF,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI;IAC5B,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IACxB,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IAC7B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CACnE,CACgB,CAAC;AAXT,QAAA,qBAAqB,yBAWZ;AAEf,MAAM,uBAAuB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACtF,EAAE,EAAE;QACF,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACxB,MAAM,EAAE,EAAE,GAAG,EAAE;KAChB;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA2B,CAAC;AANhB,QAAA,uBAAuB,2BAMP;AAEtB,MAAM,wBAAwB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACvF,GAAG,EAAE;QACH,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACzC;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA4B,CAAC;AALjB,QAAA,wBAAwB,4BAKP;AAEvB,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,UAAU,GAAG,GAAG,EAAE,EAAE;IAC5D,OAAO;QACL,UAAU;QACV,OAAO,EAAE;YACP,6BAA6B,EAAE,GAAG;YAClC,kCAAkC,EAAE,IAAI;SACzC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;KACpC,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB;AAEF,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAExB,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,EAAE;IACjC,IAAA,WAAK,EAAC,aAAa,EAAE,GAAG,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,CAAC;IACjD,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,GAAG,kBAAkB,EAAE,GAAG,OAAO,GAAG,KAAK,QAAQ;QAClF,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;QAChB,CAAC,CAAC,GAAG,CAAC,IAAI;YACR,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI;YACf,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACzD,OAAO,IAAA,sBAAc,EAAC,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,MAAM,IAAI,kBAAkB,CAAC,CAAC;AAC/E,CAAC,CAAC;AARW,QAAA,WAAW,eAQtB;AAEK,MAAM,gBAAgB,GAAG,CAAO,IAAS,EAAE,IAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAC3F,IAAI,OAAO,CAAS,OAAO,CAAC,EAAE;IAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAChB,OAAO,OAAO,EAAE,CAAC;KAClB;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACnD,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAA,wBAAgB,EAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACrF,CAAC,CAAC,CAAC;AAPQ,QAAA,gBAAgB,oBAOxB;AAEE,MAAM,OAAO,GAAG,CAAC,KAAU,EAAE,OAAO,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;IAC7E,CACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO,CACpE,CAAC,CAAC;IACH,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;AANpD,QAAA,OAAO,WAM6C;AAE1D,MAAM,eAAe,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAhE,QAAA,eAAe,mBAAiD;AAEtE,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;KACxE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AADhE,QAAA,cAAc,kBACkD;AAEtE,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAA9F,QAAA,QAAQ,YAAsF"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AACA,+CAA4D;AAG5D,+BAA8B;AAEvB,MAAM,qBAAqB,GAAG,CAAC,KAAqC,EAAkB,EAAE,CAC7F,IAAA,6BAAqB,EACnB,KAAK,IAAI,KAAK;IACZ,CAAC,CAAC;QACE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;QAChC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;KAC3E;IACH,CAAC,CAAC;QACE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI;QAC5B,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;QACxB,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;QAC7B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;KACnE,CACY,CAAC;AAbT,QAAA,qBAAqB,yBAaZ;AAEf,MAAM,uBAAuB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CACpF,CAAC;IACC,EAAE,EAAE;QACF,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACxB,MAAM,EAAE,EAAE,GAAG,EAAE;KAChB;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA2B,CAAC;AAPlB,QAAA,uBAAuB,2BAOL;AAExB,MAAM,wBAAwB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CACrF,CAAC;IACC,GAAG,EAAE;QACH,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACzC;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA4B,CAAC;AANnB,QAAA,wBAAwB,4BAML;AAEzB,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,UAAU,GAAG,GAAG,EAAE,EAAE;IAC5D,OAAO;QACL,UAAU;QACV,OAAO,EAAE;YACP,6BAA6B,EAAE,GAAG;YAClC,kCAAkC,EAAE,IAAI;SACzC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;KACpC,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB;AAEF,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAExB,MAAM,WAAW,GAAG,GAAG,CAAC,EAAE;IAC/B,IAAA,WAAK,EAAC,aAAa,EAAE,GAAG,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,CAAC;IACjD,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,GAAG,kBAAkB,EAAE,GACxD,OAAO,GAAG,KAAK,QAAQ;QACrB,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;QAChB,CAAC,CAAC,GAAG,CAAC,IAAI;YACV,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI;YACf,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACzD,OAAO,IAAA,sBAAc,EAAC,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,MAAM,IAAI,kBAAkB,CAAC,CAAC;AAC/E,CAAC,CAAC;AATW,QAAA,WAAW,eAStB;AAEK,MAAM,gBAAgB,GAAG,CAAO,IAAS,EAAE,IAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAC3F,IAAI,OAAO,CAAW,OAAO,CAAC,EAAE;IAC9B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAEnD,KAAK,IAAI,CAAC,SAAS,CAAC;SACjB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAA,wBAAgB,EAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;SAC/C,IAAI,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAVQ,QAAA,gBAAgB,oBAUxB;AAEE,MAAM,OAAO,GAAG,CAAC,KAAU,EAAE,OAAO,GAAG,CAAC,EAAE,EAAE,CACjD,OAAO,KAAK,KAAK,QAAQ;IACvB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM;QACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO;IACrE,CAAC,CAAC,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;AALxD,QAAA,OAAO,WAKiD;AAE9D,MAAM,eAAe,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAhE,QAAA,eAAe,mBAAiD;AAEtE,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,GAAW,EAAE,EAAE,CACvD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AADjF,QAAA,cAAc,kBACmE;AAEvF,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AAAhG,QAAA,QAAQ,YAAwF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/pipeline-utils",
-  "version": "0.13.0",
+  "version": "0.13.2",
   "description": "Utilities for running pipelines",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -43,38 +43,40 @@
     "@hestia-earth/utils": ">=0.11.2"
   },
   "devDependencies": {
-    "@commitlint/cli": "^17.4.2",
-    "@commitlint/config-conventional": "^17.4.2",
-    "@hestia-earth/api": "^0.21.9",
-    "@hestia-earth/eslint-config": "^0.0.5",
-    "@hestia-earth/json-schema": "^22.3.0",
-    "@hestia-earth/schema": "^22.3.0",
-    "@hestia-earth/utils": "^0.11.2",
-    "@types/aws-lambda": "^8.10.109",
-    "@types/chai": "^4.3.4",
-    "@types/lodash.get": "^4.4.7",
-    "@types/lodash.pick": "^4.4.7",
-    "@types/mocha": "^10.0.1",
-    "@types/node": "^16.18.39",
-    "@typescript-eslint/eslint-plugin": "^5.27.1",
-    "@typescript-eslint/parser": "^5.27.1",
-    "chai": "^4.3.7",
+    "@commitlint/cli": "^17.8.1",
+    "@commitlint/config-conventional": "^17.8.1",
+    "@hestia-earth/api": "^0.21.13",
+    "@hestia-earth/eslint-config": "^0.1.0",
+    "@hestia-earth/json-schema": "^22.4.0",
+    "@hestia-earth/schema": "^22.4.0",
+    "@hestia-earth/utils": "^0.11.13",
+    "@types/aws-lambda": "^8.10.130",
+    "@types/chai": "^4.3.11",
+    "@types/lodash.get": "^4.4.9",
+    "@types/lodash.pick": "^4.4.9",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^16.18.68",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "chai": "^4.3.10",
     "eslint": "^7.32.0",
     "eslint-plugin-jsdoc": "^30.7.13",
     "husky": "^4.3.8",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
+    "prettier": "^3.1.1",
     "sinon": "^15.2.0",
     "source-map-support": "^0.5.21",
     "standard-version": "^9.5.0",
-    "ts-node": "^10.9.1",
-    "typescript": "^5.1.6"
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
   },
   "husky": {
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
     }
   },
+  "prettier": "@hestia-earth/eslint-config/prettier.json",
   "engines": {
     "npm": ">=8.0.0",
     "node": ">=16.0.0"

package/src/api.ts

@@ -20,8 +20,7 @@ export const getNode = async (id: string, type: NodeType) => {
     const token = apiAccessToken();
     const headers = token ? { 'X-Access-Token': token } : {};
     return (await axios.get<any>(url, { headers })).data;
-  }
-  catch (_err) {
+  } catch (_err) {
     return null;
   }
 };

package/src/ec2.ts

@@ -7,16 +7,19 @@ import { isS3Mode } from './s3';
 
 const defaultInstanceIds = (process.env.START_INSTANCE_ID || '').split(',');
 
-export const startInstances = (functionName: string, instanceIds = defaultInstanceIds) => isS3Mode
-  ? from(
-    new AWS.EC2().startInstances({
-      InstanceIds: instanceIds
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`EC2 start instances ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+export const startInstances = (functionName: string, instanceIds = defaultInstanceIds) =>
+  isS3Mode
+    ? from(
+        new AWS.EC2()
+          .startInstances({
+            InstanceIds: instanceIds
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`EC2 start instances ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);

package/src/find-nodes.ts

@@ -19,16 +19,19 @@ const filteredKeys = (keys: string[]) => ['@id', '@type', ...keys];
 
 export const search = async (query: any, limit = 2, keys: string[] = []) => {
   debug('search', JSON.stringify(query));
-  const hits = (await client.search({
-    index: indexName,
-    size: limit,
-    body: {
-      _source: {
-        includes: filteredKeys(keys)
-      },
-      query
-    }
-  })).body.hits.hits || [];
+  const hits =
+    (
+      await client.search({
+        index: indexName,
+        size: limit,
+        body: {
+          _source: {
+            includes: filteredKeys(keys)
+          },
+          query
+        }
+      })
+    ).body.hits.hits || [];
   return hits.map(({ _source }) => _source);
 };
 
@@ -43,19 +46,18 @@ interface IQueries {
   should: ITypedQuery[];
 }
 
-const prependWithKey = (data: any, key: string) => Object.entries(data)
-  .reduce((prev, [curr, value]) => ({ ...prev, [`${key}.${curr}`]: value }), {});
+const prependWithKey = (data: any, key: string) =>
+  Object.entries(data).reduce((prev, [curr, value]) => ({ ...prev, [`${key}.${curr}`]: value }), {});
 
 const buildFindNodeQuery = (type: SchemaType, must?: ITypedQuery[], should?: ITypedQuery[]) => ({
   bool: {
-    must: [
-      { match: { '@type': type } },
-      ...must
-    ],
-    ...(should.length ? {
-      should,
-      minimum_should_match: must.length === 0 ? 1 : 0
-    } : {})
+    must: [{ match: { '@type': type } }, ...must],
+    ...(should.length
+      ? {
+          should,
+          minimum_should_match: must.length === 0 ? 1 : 0
+        }
+      : {})
   }
 });
 
@@ -65,16 +67,19 @@ const canUseCache = (value: any) => {
   return totalSize + valueSize <= cacheMaxSize;
 };
 
-const setCache = (key: string, value: any) => canUseCache(value) ? cache.put(key, value) : null;
+const setCache = (key: string, value: any) => (canUseCache(value) ? cache.put(key, value) : null);
 
 export const cacheSearch = (query: any, ...args) => {
   const cacheKey = JSON.stringify(query);
   const cacheEntry = cache.get(cacheKey);
-  return cacheEntry || (async () => {
-    const results = await search(query, ...args);
-    setCache(cacheKey, results);
-    return results;
-  })();
+  return (
+    cacheEntry ||
+    (async () => {
+      const results = await search(query, ...args);
+      setCache(cacheKey, results);
+      return results;
+    })()
+  );
 };
 
 export const execFindNode = async (query: any, keys: string[] = []) => {
@@ -91,31 +96,24 @@ const throwNotFound = () => {
 };
 
 // errors coming from ES, like server is overloaded
-const retryErrors = [
-  'ResponseError',
-  'DeserializationError'
-];
+const retryErrors = ['ResponseError', 'DeserializationError'];
 
 const retryFindNodeOnFailure = async (func: (retry: number) => Promise<any>, retries = 0) => {
   try {
     return await func(retries);
-  }
-  catch (err) {
+  } catch (err) {
     const queryError = retryErrors.includes(err.name) || err.message === 'search_phase_execution_exception';
-    return queryError && retries < MAX_RETRIES ?
-      retryFindNodeOnFailure(func, retries + 1) :
-      null;
+    return queryError && retries < MAX_RETRIES ? retryFindNodeOnFailure(func, retries + 1) : null;
   }
 };
 
 /* eslint-disable complexity */
 export const findNodeByQueries = (type: SchemaType, { must, should }: IQueries, keys?: string[]) =>
-  retryFindNodeOnFailure(async (retries) => {
+  retryFindNodeOnFailure(async retries => {
     const query = buildFindNodeQuery(type, must, should);
     try {
-      return must.length || should.length ? (await execFindNode(query, keys) || throwNotFound()) : null;
-    }
-    catch (err) {
+      return must.length || should.length ? (await execFindNode(query, keys)) || throwNotFound() : null;
+    } catch (err) {
       const logFunction = retries < MAX_RETRIES ? debug : error;
       logFunction('Node not found after', retries, 'retries', JSON.stringify(query), err);
       throw err;
@@ -139,22 +137,23 @@ export const findNodeById = async (type: NodeType | SchemaType, id: string, keys
   return node ? pick(node, filteredKeys(keys)) : null;
 };
 
-const findChildNodes = (schemas: definitions, data: any) => Promise.all(
-  Object.entries(data)
-    .filter(([_key, value]) => isExpandable(value))
-    .map(([key, value]) => ({ key, value: value as any, isArray: Array.isArray(value) }))
-    .map(async ({ isArray, key, value }) => {
-      const res = isArray ?
-        await Promise.all(value.map(val => isExpandable(val) ? findNodes(schemas, val) : val)) :
-        await findNodes(schemas, value);
-      return { [key]: res };
-    })
-);
+const findChildNodes = (schemas: definitions, data: any) =>
+  Promise.all(
+    Object.entries(data)
+      .filter(([_key, value]) => isExpandable(value))
+      .map(([key, value]) => ({ key, value: value as any, isArray: Array.isArray(value) }))
+      .map(async ({ isArray, key, value }) => {
+        const res = isArray
+          ? await Promise.all(value.map(val => (isExpandable(val) ? findNodes(schemas, val) : val)))
+          : await findNodes(schemas, value);
+        return { [key]: res };
+      })
+  );
 
 const findNodeField = (type: SchemaType, field: string) =>
-  searchNormalizedEnabled && [SchemaType.Term].includes(type) && ['name'].includes(field) ?
-    `${field}Normalized` :
-    `${field}.keyword`;
+  searchNormalizedEnabled && [SchemaType.Term].includes(type) && ['name'].includes(field)
+    ? `${field}Normalized`
+    : `${field}.keyword`;
 
 const findNodeQuery = (type: SchemaType, data: any, uniqueFields: string[], requiredFields: string[]): IQueries => ({
   must: uniqueFields
@@ -166,19 +165,24 @@ const findNodeQuery = (type: SchemaType, data: any, uniqueFields: string[], requ
 });
 
 const findChildNodesQuery = (schemas: definitions, data: any, uniqueFields: string[]) =>
-  uniqueFields.reduce((queries, key) => {
-    const { queries: { must, should } } = findNodeData(schemas, data[key]);
-    return {
-      must: [
-        ...queries.must,
-        ...must.map(({ match_phrase }) => ({ match_phrase: prependWithKey(match_phrase, key) }))
-      ],
-      should: [
-        ...queries.should,
-        ...should.map(({ match_phrase }) => ({ match_phrase: prependWithKey(match_phrase, key) }))
-      ]
-    };
-  }, { must: [], should: [] } as IQueries);
+  uniqueFields.reduce(
+    (queries, key) => {
+      const {
+        queries: { must, should }
+      } = findNodeData(schemas, data[key]);
+      return {
+        must: [
+          ...queries.must,
+          ...must.map(({ match_phrase }) => ({ match_phrase: prependWithKey(match_phrase, key) }))
+        ],
+        should: [
+          ...queries.should,
+          ...should.map(({ match_phrase }) => ({ match_phrase: prependWithKey(match_phrase, key) }))
+        ]
+      };
+    },
+    { must: [], should: [] } as IQueries
+  );
 
 const compileQueries = (schemas: definitions, type: SchemaType, data: any): IQueries => {
   const schema = schemas[type];
@@ -201,13 +205,12 @@ const findByIdQueries = (id: string): IQueries => ({ must: [{ match_phrase: { '@
  *
  * @param param0 The node
  */
-const nodeExistingId = (type: SchemaType, { '@id': _id, id }: any) =>
-  type === SchemaType.Term ? _id || id : _id;
+const nodeExistingId = (type: SchemaType, { '@id': _id, id }: any) => (type === SchemaType.Term ? _id || id : _id);
 
 const nodeExistingType = ({ '@type': _type, type }: any) =>
-  isTypeValid({ type: _type || type }) ? (_type || type) as SchemaType : undefined;
+  isTypeValid({ type: _type || type }) ? ((_type || type) as SchemaType) : undefined;
 
-const schemaByType = (schemas: definitions, type?: SchemaType) => type && type in schemas ? schemas[type] : null;
+const schemaByType = (schemas: definitions, type?: SchemaType) => (type && type in schemas ? schemas[type] : null);
 
 export const findNodeData = (schemas: definitions, { '@context': context, ...node }: any) => {
   const type = nodeExistingType(node);
@@ -215,19 +218,21 @@ export const findNodeData = (schemas: definitions, { '@context': context, ...nod
   // remove keys from data
   const { '@id': _id, '@type': _2, id, type: _4, ...data } = node;
   const schema = schemaByType(schemas, type);
-  return schema ? {
-    id: (_id || id) as string,
-    type,
-    existingId,
-    queries: existingId ? findByIdQueries(existingId) : compileQueries(schemas, type, data),
-    // can only fetch searchable properties
-    queryKeys: extendableProperties(schema),
-    data
-  } : { data: node };
+  return schema
+    ? {
+        id: (_id || id) as string,
+        type,
+        existingId,
+        queries: existingId ? findByIdQueries(existingId) : compileQueries(schemas, type, data),
+        // can only fetch searchable properties
+        queryKeys: extendableProperties(schema),
+        data
+      }
+    : { data: node };
 };
 
 const findNode = async (id: string, type: SchemaType, queries: IQueries, queryKeys: string[]) =>
-  await findNodeById(type, id, queryKeys) || await findNodeByQueries(type, queries, queryKeys);
+  (await findNodeById(type, id, queryKeys)) || (await findNodeByQueries(type, queries, queryKeys));
 
 export const findNodes = async (
   schemas: definitions,

package/src/gitlab.ts

@@ -24,10 +24,7 @@ export const listFiles = async (path: string, page = 1, limit = 100): Promise<IG
     headers,
     params: { page, per_page: limit, ref: branch, path }
   });
-  return [
-    ...data,
-    ...(data.length === limit ? await listFiles(path, page + 1) : [])
-  ];
+  return [...data, ...(data.length === limit ? await listFiles(path, page + 1) : [])];
 };
 
 export const readFile = async ({ path }: Partial<IGitlabTreeFile>) => {

package/src/lambda.ts

@@ -5,19 +5,22 @@ import { take, map } from 'rxjs/operators';
 import { debug } from './log';
 import { IFunctionParam, isS3Mode } from './s3';
 
-export const invoke = (functionName: string, params: IFunctionParam) => isS3Mode
-  ? from(
-    new AWS.Lambda().invoke({
-      FunctionName: functionName,
-      InvocationType: 'Event',
-      LogType: 'None',
-      Payload: JSON.stringify(params)
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`invoke ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+export const invoke = (functionName: string, params: IFunctionParam | Record<string, any>) =>
+  isS3Mode
+    ? from(
+        new AWS.Lambda()
+          .invoke({
+            FunctionName: functionName,
+            InvocationType: 'Event',
+            LogType: 'None',
+            Payload: JSON.stringify(params)
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`invoke ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);

package/src/local.ts

@@ -17,25 +17,22 @@ const currentDir = (key: string) => {
 };
 
 export const loadTextFile = (folder: string) => (file: string) =>
-  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true })
-    .pipe(
-      mergeMap(() => readFileAsObservable(join(folder, file), 'utf8')),
-      take(1),
-      map((res: any) => res as string)
-    );
+  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true }).pipe(
+    mergeMap(() => readFileAsObservable(join(folder, file), 'utf8')),
+    take(1),
+    map((res: any) => res as string)
+  );
 
-export const loadJSONFile = (folder: string) => <T>(file: string) =>
-  loadTextFile(folder)(file)
-    .pipe(
-      map(res => JSON.parse(stripBOM(res)) as T)
-    );
+export const loadJSONFile =
+  (folder: string) =>
+  <T>(file: string) =>
+    loadTextFile(folder)(file).pipe(map(res => JSON.parse(stripBOM(res)) as T));
 
 export const uploadText = (folder: string) => (file: string, data: string) =>
-  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true })
-    .pipe(
-      mergeMap(() => writeFileAsObservable(join(folder, file), data, 'utf8')),
-      map(() => data)
-    );
+  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true }).pipe(
+    mergeMap(() => writeFileAsObservable(join(folder, file), data, 'utf8')),
+    map(() => data)
+  );
 
 export const uploadJSON = (folder: string) => (file: string, data) =>
   uploadText(folder)(file, JSON.stringify(data, null, 2));
@@ -43,8 +40,7 @@ export const uploadJSON = (folder: string) => (file: string, data) =>
 export const lastModified = (folder: string) => (file: string) => {
   try {
     return of(statSync(join(folder, file)).mtime);
-  }
-  catch (err) {
+  } catch (err) {
     return of(new Date());
   }
 };
@@ -53,8 +49,7 @@ export const fileExists = (folder: string) => (file: string) => {
   try {
     const success = statSync(join(folder, file)).isFile();
     return of(success);
-  }
-  catch (err) {
+  } catch (err) {
     return of(false);
   }
 };
@@ -62,11 +57,12 @@ export const fileExists = (folder: string) => (file: string) => {
 export const fileSize = (folder: string) => (file: string) => {
   try {
     return of(statSync(join(folder, file)).size);
-  }
-  catch (err) {
+  } catch (err) {
     return of(undefined as number);
   }
 };
 
-export const deleteFile = (folder: string) => (file: string): Observable<any> =>
-  unlinkAsObservable(join(folder, file));
+export const deleteFile =
+  (folder: string) =>
+  (file: string): Observable<any> =>
+    unlinkAsObservable(join(folder, file));