@hestia-earth/pipeline-utils 0.8.2 → 0.9.1

package/CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.9.1](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.9.0...v0.9.1) (2023-04-18)
+
+
+### Features
+
+* **progress:** log time since datetime ([1dc6664](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/1dc66648da05cbb4049b5c6cf46796c206e18870))
+
+
+### Bug Fixes
+
+* **progress:** fix wrong step name ([2f17942](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/2f179425fd682f8850f01e0683d24b7a9737848a))
+
+## [0.9.0](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.8.2...v0.9.0) (2023-04-18)
+
+
+### ⚠ BREAKING CHANGES
+
+* **progress:** `@hestia-earth/api` required above version `0.18.0`
+
+### Features
+
+* **progress:** log time for each step ([51bf21b](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/51bf21be9cd1cfa2124ff480da9c31a228632e60))
+
 ### [0.8.2](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.8.1...v0.8.2) (2023-04-06)
 
 

package/dist/progress.d.ts

@@ -1,3 +1,4 @@
+import { FileProgress } from '@hestia-earth/api';
 import { IFunctionParam } from './s3';
 export declare enum publishFunctionSuffix {
     Success = "Done",
@@ -15,6 +16,6 @@ export declare class HestiaError extends Error {
         stack: string;
     };
 }
-export declare const handleError: (step: string, params: IFunctionParam, time?: number, Bucket?: string) => (err: HestiaError) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
-export declare const handleSuccess: (step: string, params: IFunctionParam, time?: number, Bucket?: string) => (details?: any) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
+export declare const handleError: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (err: HestiaError) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
+export declare const handleSuccess: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (details?: any) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
 export declare const defaultErrorHandler: (key: string) => (err: Error) => import("rxjs").Observable<never>;

package/dist/progress.js

@@ -29,16 +29,19 @@ class HestiaError extends Error {
 }
 exports.HestiaError = HestiaError;
 const fileToProgress = (path) => `${path.split('.')[0]}.progress`;
-const uploadProgress = (step, path, Bucket, time, err, details) => (0, s3_1.uploadJSON)(fileToProgress(path), {
+const uploadProgress = (step, path, Bucket, since, err, details) => (0, rxjs_1.of)(fileToProgress(path)).pipe((0, operators_1.mergeMap)(filepath => (0, s3_1.loadJSONFile)(filepath, Bucket).pipe((0, operators_1.catchError)(() => (0, rxjs_1.of)({})), (0, operators_1.mergeMap)(data => (0, s3_1.uploadJSON)(filepath, {
     step,
     success: !err,
-    time,
+    time: {
+        ...(data.time || {}),
+        ...(since ? { [step]: (0, exports.timeSpent)(since) } : {})
+    },
     details,
     error: err ? err.toString() : null
-}, 'application/json', Bucket);
-const handleError = (step, params, time = 0, Bucket = s3_1.bucket) => (err) => uploadProgress(step, params.key, Bucket, time, err).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Error}`, params)));
+}, 'application/json', Bucket)))));
+const handleError = (step, params, since, Bucket = s3_1.bucket) => (err) => uploadProgress(step, params.key, Bucket, since, err).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Error}`, params)));
 exports.handleError = handleError;
-const handleSuccess = (step, params, time = 0, Bucket = s3_1.bucket) => (details) => uploadProgress(step, params.key, Bucket, time, null, details).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Success}`, params)));
+const handleSuccess = (step, params, since, Bucket = s3_1.bucket) => (details) => uploadProgress(step, params.key, Bucket, since, null, details).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Success}`, params)));
 exports.handleSuccess = handleSuccess;
 const defaultErrorHandler = (key) => (err) => (0, rxjs_1.throwError)(new HestiaError(err.message, key, err.stack));
 exports.defaultErrorHandler = defaultErrorHandler;

package/dist/progress.js.map

@@ -1 +1 @@
-{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,8CAA0C;AAC1C,+BAAkC;AAElC,6BAA0D;AAC1D,+BAAgC;AAEhC,IAAY,qBAGX;AAHD,WAAY,qBAAqB;IAC/B,yCAAgB,CAAA;IAChB,wCAAe,CAAA;AACjB,CAAC,EAHW,qBAAqB,GAArB,6BAAqB,KAArB,6BAAqB,QAGhC;AAEM,MAAM,SAAS,GAAG,CAAC,KAAW,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;AAApE,QAAA,SAAS,aAA2D;AAEjF,MAAa,WAAY,SAAQ,KAAK;IAOpC,YAAmB,OAAO,EAAS,GAAW,EAAS,SAAc,EAAE;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC;QADE,YAAO,GAAP,OAAO,CAAA;QAAS,QAAG,GAAH,GAAG,CAAQ;QAAS,WAAM,GAAN,MAAM,CAAU;IAEvE,CAAC;IAED,QAAQ;QACN,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAlBD,kCAkBC;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;AAE1E,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,MAAc,EAAE,IAAY,EAAE,GAAiB,EAAE,OAAa,EAAE,EAAE,CACpH,IAAA,eAAU,EAAC,cAAc,CAAC,IAAI,CAAC,EAAE;IAC/B,IAAI;IACJ,OAAO,EAAE,CAAC,GAAG;IACb,IAAI;IACJ,OAAO;IACP,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;CACnC,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;AAE1B,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,MAAsB,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CAAC,CAAC,GAAgB,EAAE,EAAE,CACnH,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CACtD,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CACzE,CAAC;AAHS,QAAA,WAAW,eAGpB;AAEG,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,MAAsB,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CAAC,CAAC,OAAa,EAAE,EAAE,CAClH,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAChE,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC,CAC3E,CAAC;AAHS,QAAA,aAAa,iBAGtB;AAEG,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAU,EAAE,EAAE,CACjE,IAAA,iBAAU,EAAC,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAD9C,QAAA,mBAAmB,uBAC2B"}
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,8CAAsD;AACtD,+BAAsC;AAGtC,6BAAwE;AACxE,+BAAgC;AAYhC,IAAY,qBAGX;AAHD,WAAY,qBAAqB;IAC/B,yCAAgB,CAAA;IAChB,wCAAe,CAAA;AACjB,CAAC,EAHW,qBAAqB,GAArB,6BAAqB,KAArB,6BAAqB,QAGhC;AAEM,MAAM,SAAS,GAAG,CAAC,KAAW,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;AAApE,QAAA,SAAS,aAA2D;AAEjF,MAAa,WAAY,SAAQ,KAAK;IAOpC,YAAmB,OAAO,EAAS,GAAW,EAAS,SAAc,EAAE;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC;QADE,YAAO,GAAP,OAAO,CAAA;QAAS,QAAG,GAAH,GAAG,CAAQ;QAAS,WAAM,GAAN,MAAM,CAAU;IAEvE,CAAC;IAED,QAAQ;QACN,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAlBD,kCAkBC;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;AAE1E,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,MAAc,EAAE,KAAY,EAAE,GAAiB,EAAE,OAAa,EAAE,EAAE,CACpH,IAAA,SAAE,EAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAC3B,IAAA,oBAAQ,EAAC,QAAQ,CAAC,EAAE,CAAC,IAAA,iBAAY,EAAyB,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAC9E,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,EAA4B,CAAC,CAAC,EAClD,IAAA,oBAAQ,EAAC,IAAI,CAAC,EAAE,CAAC,IAAA,eAAU,EAAC,QAAQ,EAAE;IACpC,IAAI;IACJ,OAAO,EAAE,CAAC,GAAG;IACb,IAAI,EAAE;QACJ,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACpB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAA,iBAAS,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C;IACD,OAAO;IACP,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;CACnC,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAChC,CAAC,CACH,CAAC;AAEG,MAAM,WAAW,GAAG,CAAC,IAAkB,EAAE,MAAsB,EAAE,KAAY,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CACvG,CAAC,GAAgB,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAC7E,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CACzE,CAAC;AAHS,QAAA,WAAW,eAGpB;AAEG,MAAM,aAAa,GAAG,CAAC,IAAkB,EAAE,MAAsB,EAAE,KAAY,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CACzG,CAAC,OAAa,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CACpF,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC,CAC3E,CAAC;AAHS,QAAA,aAAa,iBAGtB;AAEG,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAU,EAAE,EAAE,CACjE,IAAA,iBAAU,EAAC,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAD9C,QAAA,mBAAmB,uBAC2B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/pipeline-utils",
-  "version": "0.8.2",
+  "version": "0.9.1",
   "description": "Utilities for running pipelines",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -38,15 +38,18 @@
     "rxjs": "^6.6.7"
   },
   "peerDependencies": {
+    "@hestia-earth/api": ">=0.18.0",
     "@hestia-earth/json-schema": ">=15.0.0",
     "@hestia-earth/schema": ">=15.0.0"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.4.2",
     "@commitlint/config-conventional": "^17.4.2",
+    "@hestia-earth/api": "^0.18.0",
     "@hestia-earth/eslint-config": "^0.0.5",
     "@hestia-earth/json-schema": "^15.0.0",
     "@hestia-earth/schema": "^15.0.0",
+    "@hestia-earth/utils": "^0.11.0",
     "@types/aws-lambda": "^8.10.109",
     "@types/aws-sdk": "^2.7.0",
     "@types/chai": "^4.3.4",

package/src/progress.ts

@@ -1,9 +1,20 @@
-import { mergeMap } from 'rxjs/operators';
-import { throwError } from 'rxjs';
+import { catchError, mergeMap } from 'rxjs/operators';
+import { of, throwError } from 'rxjs';
+import { FileProgress } from '@hestia-earth/api';
 
-import { IFunctionParam, uploadJSON, bucket } from './s3';
+import { IFunctionParam, uploadJSON, bucket, loadJSONFile } from './s3';
 import { publish } from './sns';
 
+interface IFileProgress {
+  step: FileProgress;
+  success: boolean;
+  time: {
+    [step in FileProgress]?: number;
+  };
+  details?: any;
+  error?: any;
+}
+
 export enum publishFunctionSuffix {
   Success = 'Done',
   Error = 'Error'
@@ -33,22 +44,30 @@ export class HestiaError extends Error {
 
 const fileToProgress = (path: string) => `${path.split('.')[0]}.progress`;
 
-const uploadProgress = (step: string, path: string, Bucket: string, time: number, err?: HestiaError, details?: any) =>
-  uploadJSON(fileToProgress(path), {
-    step,
-    success: !err,
-    time,
-    details,
-    error: err ? err.toString() : null
-  }, 'application/json', Bucket);
-
-export const handleError = (step: string, params: IFunctionParam, time = 0, Bucket = bucket) => (err: HestiaError) =>
-  uploadProgress(step, params.key, Bucket, time, err).pipe(
+const uploadProgress = (step: string, path: string, Bucket: string, since?: Date, err?: HestiaError, details?: any) =>
+  of(fileToProgress(path)).pipe(
+    mergeMap(filepath => loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
+      catchError(() => of({} as Partial<IFileProgress>)),
+      mergeMap(data => uploadJSON(filepath, {
+        step,
+        success: !err,
+        time: {
+          ...(data.time || {}),
+          ...(since ? { [step]: timeSpent(since) } : {})
+        },
+        details,
+        error: err ? err.toString() : null
+      }, 'application/json', Bucket))
+    ))
+  );
+
+export const handleError = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (err: HestiaError) => uploadProgress(step, params.key, Bucket, since, err).pipe(
     mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
   );
 
-export const handleSuccess = (step: string, params: IFunctionParam, time = 0, Bucket = bucket) => (details?: any) =>
-  uploadProgress(step, params.key, Bucket, time, null, details).pipe(
+export const handleSuccess = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (details?: any) => uploadProgress(step, params.key, Bucket, since, null, details).pipe(
     mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
   );