@hestia-earth/pipeline-utils 0.15.5 → 0.16.1

package/dist/sns.js

@@ -1,15 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.publishSnsMessage = void 0;
-const AWS = require("aws-sdk");
+const client_sns_1 = require("@aws-sdk/client-sns");
 const rxjs_1 = require("rxjs");
-const operators_1 = require("rxjs/operators");
 const log_1 = require("./log");
 const s3_1 = require("./s3");
 const utils_1 = require("./utils");
 const publishSnsMessage = (functionName, params, topicArn = process.env.SNS_TOPIC, extras = {}) => s3_1.isS3Mode
-    ? (0, rxjs_1.from)(new AWS.SNS()
-        .publish({
+    ? (0, rxjs_1.from)(new client_sns_1.SNSClient({}).send(new client_sns_1.PublishCommand({
         TopicArn: topicArn,
         Message: JSON.stringify(params),
         MessageAttributes: {
@@ -25,11 +23,10 @@ const publishSnsMessage = (functionName, params, topicArn = process.env.SNS_TOPI
                 }
             ]))
         }
-    })
-        .promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
+    }))).pipe((0, rxjs_1.take)(1), (0, rxjs_1.map)(res => {
         (0, log_1.info)(`SNS publish ${functionName}`, { params }, ', params byte size:', (0, utils_1.byteSize)(params), res);
         return res;
-    }), (0, operators_1.catchError)(err => {
+    }), (0, rxjs_1.catchError)(err => {
         (0, log_1.error)(`SNS publish ${functionName}`, { params }, ', params byte size:', (0, utils_1.byteSize)(params), err);
         return (0, rxjs_1.throwError)(err);
     }))

package/dist/sns.js.map

@@ -1 +1 @@
-{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAA4C;AAC5C,8CAAuD;AAEvD,+BAAoC;AACpC,6BAAgD;AAChD,mCAAmC;AAM5B,MAAM,iBAAiB,GAAG,CAC/B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAChC,SAAqB,EAAE,EACvB,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,GAAG,CAAC,GAAG,EAAE;SACV,OAAO,CAAC;QACP,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;KACF,CAAC;SACD,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9F,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,EACF,IAAA,sBAAU,EAAC,GAAG,CAAC,EAAE;QACf,IAAA,WAAK,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/F,OAAO,IAAA,iBAAU,EAAC,GAAG,CAAC,CAAC;IACzB,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAxCF,QAAA,iBAAiB,qBAwCf"}
+{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,oDAAgE;AAChE,+BAAmE;AAEnE,+BAAoC;AACpC,6BAAgD;AAChD,mCAAmC;AAM5B,MAAM,iBAAiB,GAAG,CAC/B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAChC,SAAqB,EAAE,EACvB,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,sBAAS,CAAC,EAAE,CAAC,CAAC,IAAI,CACpB,IAAI,2BAAc,CAAC;QACjB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;KACF,CAAC,CACH,CACF,CAAC,IAAI,CACJ,IAAA,WAAI,EAAC,CAAC,CAAC,EACP,IAAA,UAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9F,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,EACF,IAAA,iBAAU,EAAC,GAAG,CAAC,EAAE;QACf,IAAA,WAAK,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/F,OAAO,IAAA,iBAAU,EAAC,GAAG,CAAC,CAAC;IACzB,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAxCF,QAAA,iBAAiB,qBAwCf"}

package/dist/sqs.d.ts

@@ -1,7 +1,7 @@
-import * as AWS from 'aws-sdk';
+import { SendMessageCommandInput } from '@aws-sdk/client-sqs';
 import { IFunctionParam } from './s3';
 type attributes = {
     [key: string]: string;
 };
-export declare const sendSqsMessage: (functionName: string, params: IFunctionParam | Record<string, any>, queueUrl?: string, extras?: attributes, requestParams?: Partial<Omit<AWS.SQS.Types.SendMessageRequest, 'QueueUrl' | 'MessageBody' | 'MessageAttributes'>>) => import("rxjs").Observable<any>;
+export declare const sendSqsMessage: (functionName: string, params: IFunctionParam | Record<string, any>, queueUrl?: string, extras?: attributes, requestParams?: Partial<Omit<SendMessageCommandInput, 'QueueUrl' | 'MessageBody' | 'MessageAttributes'>>) => import("rxjs").Observable<import("@aws-sdk/client-sqs").SendMessageCommandOutput>;
 export {};

package/dist/sqs.js

@@ -1,14 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.sendSqsMessage = void 0;
-const AWS = require("aws-sdk");
+const client_sqs_1 = require("@aws-sdk/client-sqs");
 const rxjs_1 = require("rxjs");
-const operators_1 = require("rxjs/operators");
 const log_1 = require("./log");
 const s3_1 = require("./s3");
 const sendSqsMessage = (functionName, params, queueUrl = process.env.SQS_URL, extras = {}, requestParams = {}) => s3_1.isS3Mode
-    ? (0, rxjs_1.from)(new AWS.SQS({ region: s3_1.region })
-        .sendMessage({
+    ? (0, rxjs_1.from)(new client_sqs_1.SQSClient({ region: s3_1.region }).send(new client_sqs_1.SendMessageCommand({
         QueueUrl: queueUrl,
         MessageBody: JSON.stringify(params),
         MessageAttributes: {
@@ -25,8 +23,7 @@ const sendSqsMessage = (functionName, params, queueUrl = process.env.SQS_URL, ex
             ]))
         },
         ...requestParams
-    })
-        .promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
+    }))).pipe((0, rxjs_1.take)(1), (0, rxjs_1.map)(res => {
         (0, log_1.info)(`SQS send message ${functionName}`, { params }, res);
         return res;
     }))

package/dist/sqs.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA6B;AAC7B,6BAAwD;AAMjD,MAAM,cAAc,GAAG,CAC5B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAC9B,SAAqB,EAAE,EACvB,gBAAmH,EAAE,EACrH,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC;SACpB,WAAW,CAAC;QACX,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;QACD,GAAG,aAAa;KACjB,CAAC;SACD,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,oBAAoB,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAtCF,QAAA,cAAc,kBAsCZ"}
+{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,oDAA6F;AAC7F,+BAA2C;AAE3C,+BAA6B;AAC7B,6BAAwD;AAMjD,MAAM,cAAc,GAAG,CAC5B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAC9B,SAAqB,EAAE,EACvB,gBAA0G,EAAE,EAC5G,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,sBAAS,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC,CAAC,IAAI,CAC5B,IAAI,+BAAkB,CAAC;QACrB,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;QACD,GAAG,aAAa;KACjB,CAAC,CACH,CACF,CAAC,IAAI,CACJ,IAAA,WAAI,EAAC,CAAC,CAAC,EACP,IAAA,UAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,oBAAoB,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAtCF,QAAA,cAAc,kBAsCZ"}

package/dist/write-json-stream.d.ts

@@ -1 +1 @@
-export declare const streamJSONToFile: (filepath: string, data: Record<string, any>) => Promise<unknown>;
+export declare const streamJSONToFile: (filepath: string, data: Record<string, any>) => Promise<void>;

package/dist/write-json-stream.js

@@ -34,7 +34,7 @@ const stringify = (writeStream, data) => data !== null && typeof data === 'objec
     : writeStream.write(JSON.stringify(data), encoding);
 const streamJSONToFile = (filepath, data) => new Promise((resolve, reject) => {
     const writeStream = (0, fs_1.createWriteStream)(filepath);
-    writeStream.on('finish', resolve);
+    writeStream.on('finish', () => resolve());
     writeStream.on('error', reject);
     stringify(writeStream, data);
     writeStream.end();

package/dist/write-json-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-json-stream.js","sourceRoot":"","sources":["../src/write-json-stream.ts"],"names":[],"mappings":";;;AAAA,2BAAoD;AAEpD,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB,MAAM,WAAW,GAAG,CAAC,KAAU,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,WAAW,CAAC;AAEjE,MAAM,cAAc,GAAG,CAAC,WAAwB,EAAE,IAAW,EAAE,EAAE;IAC/D,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAEjC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,WAAwB,EAAE,IAAyB,EAAE,EAAE;IAC9E,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAEjC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;SACjB,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SAC7C,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACL,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,WAAwB,EAAE,IAAS,EAAE,EAAE,CACxD,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;IACvC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QACnB,CAAC,CAAC,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC;QACnC,CAAC,CAAC,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC;IACtC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;AAEjD,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,IAAyB,EAAE,EAAE,CAC9E,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IAC9B,MAAM,WAAW,GAAG,IAAA,sBAAiB,EAAC,QAAQ,CAAC,CAAC;IAChD,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC7B,WAAW,CAAC,GAAG,EAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAPQ,QAAA,gBAAgB,oBAOxB"}
+{"version":3,"file":"write-json-stream.js","sourceRoot":"","sources":["../src/write-json-stream.ts"],"names":[],"mappings":";;;AAAA,2BAAoD;AAEpD,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB,MAAM,WAAW,GAAG,CAAC,KAAU,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,WAAW,CAAC;AAEjE,MAAM,cAAc,GAAG,CAAC,WAAwB,EAAE,IAAW,EAAE,EAAE;IAC/D,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAEjC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,WAAwB,EAAE,IAAyB,EAAE,EAAE;IAC9E,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAEjC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;SACjB,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SAC7C,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACL,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,WAAwB,EAAE,IAAS,EAAE,EAAE,CACxD,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;IACvC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QACnB,CAAC,CAAC,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC;QACnC,CAAC,CAAC,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC;IACtC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;AAEjD,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,IAAyB,EAAE,EAAE,CAC9E,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IACpC,MAAM,WAAW,GAAG,IAAA,sBAAiB,EAAC,QAAQ,CAAC,CAAC;IAChD,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAC1C,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC7B,WAAW,CAAC,GAAG,EAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAPQ,QAAA,gBAAgB,oBAOxB"}

package/jest.setup.js

@@ -0,0 +1,15 @@
+const defaultEnv = (key, value) => {
+  process.env[key] = process.env[key] || value;
+  return process.env[key];
+};
+
+defaultEnv('TMP_DIR', './');
+defaultEnv('ELASTICSEARCH_URL', 'http://localhost:9200');
+defaultEnv('ELASTICSEARCH_INDEX', 'test');
+// enable search by normalized field
+defaultEnv('SEARCH_NORMALIZED', 'true');
+// enable search by API
+defaultEnv('API_URL', 'http://localhost:3001');
+defaultEnv('API_ACCESS_TOKEN', 'token');
+// enable cache requests
+defaultEnv('CACHE_MAX_SIZE', '1');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/pipeline-utils",
-  "version": "0.15.5",
+  "version": "0.16.1",
   "description": "Utilities for running pipelines",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -8,8 +8,8 @@
     "build": "rimraf ./dist && tsc -p tsconfig.dist.json",
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "npm run lint -- --fix",
-    "test": "nyc mocha \"src/**/*.spec.ts\"",
-    "test:watch": "npm run test -- --watch ",
+    "test": "jest",
+    "test:watch": "jest --watch",
     "release": "standard-version",
     "postrelease": "git push origin master --follow-tags"
   },
@@ -28,9 +28,15 @@
   },
   "homepage": "https://gitlab.com/hestia-earth/hestia-pipeline-utils#readme",
   "dependencies": {
+    "@aws-sdk/client-ec2": "^3.525.0",
+    "@aws-sdk/client-lambda": "^3.525.0",
+    "@aws-sdk/client-s3": "^3.525.0",
+    "@aws-sdk/client-sns": "^3.525.0",
+    "@aws-sdk/client-sqs": "^3.525.0",
+    "@aws-sdk/lib-storage": "^3.525.0",
+    "@aws-sdk/util-utf8-node": "^3.259.0",
     "@elastic/elasticsearch": "7.13.0",
     "@slack/web-api": "^7.7.0",
-    "aws-sdk": "^2.1692.0",
     "axios": "^1.4.0",
     "bson": "^4.7.2",
     "lodash.get": "^4.4.2",
@@ -38,7 +44,7 @@
     "lodash.pick": "^4.4.0",
     "memory-cache": "^0.2.0",
     "mongodb": "^3.7.4",
-    "rxjs": "^6.6.7"
+    "rxjs": "^7.8.1"
   },
   "peerDependencies": {
     "@hestia-earth/api": ">=0.21.0",
@@ -55,26 +61,22 @@
     "@hestia-earth/schema": "^30.3.2",
     "@hestia-earth/utils": "^0.13.9",
     "@types/aws-lambda": "^8.10.130",
-    "@types/chai": "^4.3.11",
+    "@types/jest": "^30.0.0",
     "@types/lodash.get": "^4.4.9",
     "@types/lodash.omit": "^4.5.9",
     "@types/lodash.pick": "^4.4.9",
-    "@types/mocha": "^10.0.6",
     "@types/mongodb": "^3.6.20",
-    "@types/node": "^18.19.64",
+    "@types/node": "^22.19.3",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
-    "chai": "^4.3.10",
     "dotenv": "^16.4.7",
     "eslint": "^7.32.0",
     "eslint-plugin-jsdoc": "^30.7.13",
     "husky": "^4.3.8",
-    "mocha": "^10.2.0",
-    "nyc": "^15.1.0",
+    "jest": "^30.2.0",
     "prettier": "^3.1.1",
-    "sinon": "^15.2.0",
-    "source-map-support": "^0.5.21",
     "standard-version": "^9.5.0",
+    "ts-jest": "^29.4.6",
     "ts-node": "^10.9.2",
     "typescript": "^5.3.3"
   },
@@ -86,6 +88,6 @@
   "prettier": "@hestia-earth/eslint-config/prettier.json",
   "engines": {
     "npm": ">=8.0.0",
-    "node": ">=16.0.0"
+    "node": ">=22.0.0"
   }
 }

package/src/ec2.ts

@@ -1,6 +1,5 @@
-import * as AWS from 'aws-sdk';
-import { from, of } from 'rxjs';
-import { take, map } from 'rxjs/operators';
+import { EC2Client, StartInstancesCommand } from '@aws-sdk/client-ec2';
+import { from, of, take, map } from 'rxjs';
 
 import { debug } from './log';
 import { isS3Mode } from './s3';
@@ -10,11 +9,11 @@ const defaultInstanceIds = (process.env.START_INSTANCE_ID || '').split(',');
 export const startInstances = (functionName: string, instanceIds = defaultInstanceIds) =>
   isS3Mode
     ? from(
-        new AWS.EC2()
-          .startInstances({
+        new EC2Client({}).send(
+          new StartInstancesCommand({
             InstanceIds: instanceIds
           })
-          .promise()
+        )
       ).pipe(
         take(1),
         map(res => {

package/src/find-nodes.ts

@@ -5,8 +5,7 @@ const cache = require('memory-cache');
 import { SchemaType, JSON as HestiaJson, isExpandable, NodeType, isTypeValid, isTypeNode } from '@hestia-earth/schema';
 import { definitions, uniqueProperties, extendableProperties } from '@hestia-earth/json-schema';
 import { reduceUndefinedValues } from '@hestia-earth/utils';
-import { from, of } from 'rxjs';
-import { filter, map, mergeMap, reduce, toArray } from 'rxjs/operators';
+import { from, of, filter, map, mergeMap, reduce, toArray } from 'rxjs';
 
 import { debug, error } from './log';
 import { client, indexName } from './es';

package/src/lambda.ts

@@ -1,6 +1,6 @@
-import * as AWS from 'aws-sdk';
-import { from, of } from 'rxjs';
-import { take, map } from 'rxjs/operators';
+import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda';
+import { from, of, take, map } from 'rxjs';
+import { fromUtf8 } from '@aws-sdk/util-utf8-node';
 
 import { info } from './log';
 import { IFunctionParam, isS3Mode } from './s3';
@@ -8,14 +8,14 @@ import { IFunctionParam, isS3Mode } from './s3';
 export const invoke = (functionName: string, params: IFunctionParam | Record<string, any>) =>
   isS3Mode
     ? from(
-        new AWS.Lambda()
-          .invoke({
+        new LambdaClient({}).send(
+          new InvokeCommand({
             FunctionName: functionName,
             InvocationType: 'Event',
             LogType: 'None',
-            Payload: JSON.stringify(params)
+            Payload: fromUtf8(JSON.stringify(params))
           })
-          .promise()
+        )
       ).pipe(
         take(1),
         map(res => {

package/src/progress.ts

@@ -1,5 +1,4 @@
-import { catchError, mergeMap } from 'rxjs/operators';
-import { of, throwError } from 'rxjs';
+import { catchError, mergeMap, of, throwError } from 'rxjs';
 import { FileProgress, fileToExt, HestiaExtensions } from '@hestia-earth/api';
 import { NodeType } from '@hestia-earth/schema';
 

package/src/s3.ts

@@ -1,16 +1,46 @@
 /* eslint-disable complexity */
-import * as AWS from 'aws-sdk';
+import {
+  S3Client,
+  GetObjectCommand,
+  GetObjectCommandInput,
+  PutObjectCommand,
+  PutObjectCommandInput,
+  DeleteObjectsCommand,
+  CopyObjectCommand,
+  waitUntilObjectExists,
+  HeadObjectCommand,
+  DeleteObjectCommand,
+  ListObjectsV2Command,
+  MetadataDirective,
+  _Object,
+  ObjectIdentifier,
+  DeletedObject
+} from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
 import { createReadStream, unlinkSync } from 'fs';
 import { join } from 'path';
-import { PassThrough } from 'stream';
-import { Subject, Observable, from, of } from 'rxjs';
-import { bufferCount, catchError, concatMap, delay, map, mergeMap, take, tap, toArray } from 'rxjs/operators';
+import {
+  from,
+  of,
+  Observable,
+  Subject,
+  catchError,
+  map,
+  mergeMap,
+  tap,
+  take,
+  bufferCount,
+  concatMap,
+  delay,
+  toArray
+} from 'rxjs';
 import { filenameFromPath, fileToExt, sizeInMb } from '@hestia-earth/api';
 
 import { debug } from './log';
 import * as local from './local';
 import { stripBOM } from './utils';
 import { streamJSONToFile } from './write-json-stream';
+import { PassThrough, Readable } from 'stream';
 
 const encoding = 'utf-8';
 export const isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
@@ -29,79 +59,83 @@ export interface IFunctionParam {
 /**
  * Unsafe, but needed otherwise it is impossible to run unit tests
  */
-let _s3: AWS.S3;
+let _s3: S3Client;
 
 export const getS3 = () => {
-  _s3 = _s3 || new AWS.S3();
+  _s3 = _s3 || new S3Client({ region });
   return _s3;
 };
 
-export const getObject = ({ Key, ...params }: AWS.S3.GetObjectRequest) =>
-  from(
-    getS3()
-      .getObject({ ...params, Key: decodeURI(Key) })
-      .promise()
-  ).pipe(
-    take(1),
-    map(({ Body }) => Body.toString())
-  );
+const getBody = async (command: GetObjectCommand) => {
+  const response = await getS3().send(command);
+  return response.Body.transformToString();
+};
+
+export const getObject = ({ Key, Bucket, ...params }: GetObjectCommandInput) =>
+  from(getBody(new GetObjectCommand({ ...params, Key: decodeURI(Key), Bucket }))).pipe(map(Body => Body.toString()));
 
-export const putObject = ({ Key, ...params }: AWS.S3.PutObjectRequest) =>
+export const putObject = ({ Key, Bucket, ...params }: PutObjectCommandInput) =>
+  from(getS3().send(new PutObjectCommand({ ...params, Key: decodeURI(Key), Bucket })));
+
+const execDeleteObjects = (Bucket: string, Objects: ObjectIdentifier[]) =>
   from(
-    getS3()
-      .putObject({ ...params, Key: decodeURI(Key) })
-      .promise()
+    getS3().send(
+      new DeleteObjectsCommand({
+        Bucket,
+        Delete: { Objects }
+      })
+    )
   ).pipe(
     take(1),
-    map(res => (res.$response.data || '').toString())
+    map(res => res.Deleted || [])
   );
 
-export const deleteObjects = (objects: AWS.S3.Object[], Bucket = bucket) =>
+export const deleteObjects = (objects: ObjectIdentifier[], Bucket = bucket) =>
   objects.length === 0
-    ? of('')
+    ? of([] as DeletedObject[])
     : from(objects).pipe(
         // max objects to delete is 1000
         bufferCount(1000),
         // delay each operation by 500ms to avoid "Slow down" errors
         concatMap(values => of(values.map(({ Key }) => ({ Key }))).pipe(delay(500))),
-        mergeMap(Objects =>
-          from(getS3().deleteObjects({ Bucket, Delete: { Objects } }).promise()).pipe(
-            take(1),
-            map(res => (res.$response.data || '').toString())
-          )
-        ),
+        mergeMap(Objects => execDeleteObjects(Bucket, Objects)),
         toArray(),
-        map(res => res.join(''))
+        map(res => res.flat())
       );
 
 export const copyObject = (
   fromParams: { bucket: string; key: string },
   toParams: { bucket: string; key: string },
-  Metadata: any = {},
-  MetadataDirective = 'REPLACE'
+  metadata: any = {},
+  metadataDirective: MetadataDirective = 'REPLACE'
 ) =>
   from(
-    getS3()
-      .copyObject({
+    getS3().send(
+      new CopyObjectCommand({
         Bucket: toParams.bucket,
         Key: decodeURI(toParams.key),
         CopySource: `${fromParams.bucket}/${fromParams.key}`,
-        Metadata,
-        MetadataDirective
+        Metadata: metadata,
+        MetadataDirective: metadataDirective
       })
-      .promise()
-  ).pipe(
-    take(1),
-    map(res => (res.$response.data || '').toString())
+    )
   );
 
-export const loadStream = (key: string, Bucket = bucket) =>
+export const loadStream = (key: string, Bucket = bucket): Readable => {
+  const stream = new PassThrough();
+
+  const command = new GetObjectCommand({
+    Bucket,
+    Key: decodeURI(key)
+  });
+
   getS3()
-    .getObject({
-      Bucket,
-      Key: decodeURI(key)
-    })
-    .createReadStream();
+    .send(command)
+    .then(response => (response.Body as Readable).pipe(stream))
+    .catch(err => stream.emit('error', err));
+
+  return stream;
+};
 
 export const loadFile = (key: string, Bucket = bucket) => {
   const stream = loadStream(key, Bucket);
@@ -183,24 +217,7 @@ export const uploadText = (key: string, data: string, Bucket = bucket) =>
       })
     : local.uploadText(Bucket)(key, data);
 
-/**
- * Use this method to upload large JSON files.
- *
- * @param Key
- * @param data
- * @param Bucket
- * @returns
- */
-export const uploadStreamJSON = (Key: string, data: Record<string, any>, Bucket = bucket) => {
-  const stream = new PassThrough();
-
-  const uploader = getS3().upload({
-    Bucket,
-    Key,
-    Body: stream,
-    ContentType: 'application/json'
-  });
-
+const uploadDataToStream = (stream: PassThrough, data: Record<string, any>) => {
   // beginning of JSON object
   stream.write('{', encoding);
 
@@ -224,8 +241,30 @@ export const uploadStreamJSON = (Key: string, data: Record<string, any>, Bucket
   // end of JSON object
   stream.write('}', encoding);
   stream.end();
+};
+
+export const uploadStreamJSON = (Key: string, data: Record<string, any>, Bucket = bucket) => {
+  const stream = new PassThrough();
+
+  const uploader = new Upload({
+    client: getS3(),
+    params: {
+      Bucket,
+      Key,
+      Body: stream,
+      ContentType: 'application/json'
+    }
+  });
+
+  process.nextTick(() => {
+    try {
+      uploadDataToStream(stream, data);
+    } catch (err) {
+      stream.destroy(err as Error);
+    }
+  });
 
-  return from(uploader.promise());
+  return from(uploader.done());
 };
 
 /**
@@ -243,23 +282,21 @@ export const uploadFileStream = (
   type = 'application/json',
   Bucket = bucket,
   partSize = 100 * 1024 * 1024
-) =>
-  from(
-    getS3()
-      .upload(
-        {
-          Bucket,
-          Key: key,
-          Body: createReadStream(filepath),
-          ContentType: type
-        },
-        {
-          partSize,
-          queueSize: 1
-        }
-      )
-      .promise()
-  );
+) => {
+  const fileStream = createReadStream(filepath);
+  const upload = new Upload({
+    client: getS3(),
+    params: {
+      Bucket,
+      Key: key,
+      Body: fileStream,
+      ContentType: type
+    },
+    partSize,
+    queueSize: 1
+  });
+  return from(upload.done());
+};
 
 export const uploadJSON = (key: string, data: any, type = 'application/json', Bucket = bucket) =>
   isS3Mode
@@ -283,26 +320,27 @@ export const uploadJSONLarge = (key: string, data: any, type = 'application/json
 export const waitObjectCreated = (key: string, Bucket = bucket) =>
   isS3Mode
     ? from(
-        getS3()
-          .waitFor('objectExists', {
+        waitUntilObjectExists(
+          {
+            client: getS3(),
+            maxWaitTime: 20
+          },
+          {
             Bucket,
             Key: decodeURI(key)
-          })
-          .promise()
-      ).pipe(
-        take(1),
-        map(() => true)
-      )
+          }
+        )
+      ).pipe(map(() => true))
     : of(true);
 
 export const headObject = (key: string, Bucket = bucket) =>
   from(
-    getS3()
-      .headObject({
+    getS3().send(
+      new HeadObjectCommand({
         Bucket,
         Key: decodeURI(key)
       })
-      .promise()
+    )
   );
 
 export const lastModified = (key: string, Bucket = bucket) =>
@@ -318,8 +356,7 @@ export const fileExists = (key: string, Bucket = bucket) =>
   isS3Mode
     ? headObject(key, Bucket).pipe(
         map(() => true),
-        catchError(() => of(false)),
-        take(1)
+        catchError(() => of(false))
       )
     : local.fileExists(Bucket)(key);
 
@@ -327,37 +364,43 @@ export const fileSize = (key: string, Bucket = bucket) =>
   isS3Mode
     ? headObject(key, Bucket).pipe(
         map(({ ContentLength }) => ContentLength),
-        catchError(() => of(undefined as number)),
-        take(1)
+        catchError(() => of(undefined as number))
       )
     : local.fileSize(Bucket)(key);
 
 export const deleteObject = (key: string, Bucket = bucket) =>
   isS3Mode
     ? from(
-        getS3()
-          .deleteObject({
+        getS3().send(
+          new DeleteObjectCommand({
             Bucket,
             Key: decodeURI(key)
           })
-          .promise()
+        )
       )
     : local.deleteFile(Bucket)(key);
 
-export const listFolder = async (Prefix = '', Bucket = bucket, nextToken?: string): Promise<AWS.S3.Object[]> => {
-  const { Contents, IsTruncated, NextContinuationToken } = await getS3()
-    .listObjectsV2({
+export const listFolder = async (Prefix = '', Bucket = bucket, nextToken?: string): Promise<_Object[]> => {
+  const { Contents, IsTruncated, NextContinuationToken } = await getS3().send(
+    new ListObjectsV2Command({
       Bucket,
       Prefix,
       ContinuationToken: nextToken,
       MaxKeys: 1000
     })
-    .promise();
-  return [...Contents, ...(IsTruncated ? await listFolder(Prefix, bucket, NextContinuationToken) : [])];
+  );
+  return [...(Contents || []), ...(IsTruncated ? await listFolder(Prefix, bucket, NextContinuationToken) : [])];
 };
 
 export const listFolders = async (Bucket = bucket) =>
-  (await getS3().listObjectsV2({ Bucket, Delimiter: '/' }).promise()).CommonPrefixes?.map(({ Prefix }) => Prefix);
+  (
+    await getS3().send(
+      new ListObjectsV2Command({
+        Bucket,
+        Delimiter: '/'
+      })
+    )
+  ).CommonPrefixes?.map(({ Prefix }) => Prefix);
 
 export const largeFileSize = +(process.env.LARGE_SIZE_MB || '100');