@hestia-earth/pipeline-utils 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/local.js +2 -1
- package/dist/local.js.map +1 -1
- package/dist/progress.d.ts +3 -2
- package/dist/progress.js +6 -3
- package/dist/progress.js.map +1 -1
- package/dist/s3.js +2 -1
- package/dist/s3.js.map +1 -1
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +3 -1
- package/dist/utils.js.map +1 -1
- package/package.json +3 -1
- package/src/local.ts +3 -1
- package/src/progress.ts +34 -15
- package/src/s3.ts +2 -1
- package/src/utils.ts +2 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
## [0.9.0](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.8.2...v0.9.0) (2023-04-18)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### ⚠ BREAKING CHANGES
|
|
9
|
+
|
|
10
|
+
* **progress:** `@hestia-earth/api` required above version `0.18.0`
|
|
11
|
+
|
|
12
|
+
### Features
|
|
13
|
+
|
|
14
|
+
* **progress:** log time for each step ([51bf21b](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/51bf21be9cd1cfa2124ff480da9c31a228632e60))
|
|
15
|
+
|
|
16
|
+
### [0.8.2](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.8.1...v0.8.2) (2023-04-06)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### Bug Fixes
|
|
20
|
+
|
|
21
|
+
* **s3:** handle BOM when reading JSON files ([2c195c7](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/2c195c7860cd932ad2bccf59074a1fb83f10f419))
|
|
22
|
+
|
|
5
23
|
### [0.8.1](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.8.0...v0.8.1) (2023-02-15)
|
|
6
24
|
|
|
7
25
|
|
package/dist/local.js
CHANGED
|
@@ -5,6 +5,7 @@ const fs_1 = require("fs");
|
|
|
5
5
|
const path_1 = require("path");
|
|
6
6
|
const rxjs_1 = require("rxjs");
|
|
7
7
|
const operators_1 = require("rxjs/operators");
|
|
8
|
+
const utils_1 = require("./utils");
|
|
8
9
|
const readFileAsObservable = (0, rxjs_1.bindNodeCallback)(fs_1.readFile.bind(fs_1.readFile));
|
|
9
10
|
const writeFileAsObservable = (0, rxjs_1.bindNodeCallback)(fs_1.writeFile.bind(fs_1.writeFile));
|
|
10
11
|
const unlinkAsObservable = (0, rxjs_1.bindNodeCallback)(fs_1.unlink.bind(fs_1.unlink));
|
|
@@ -18,7 +19,7 @@ const loadTextFile = (folder) => (file) => mkdirAsObservable(currentDir((0, path
|
|
|
18
19
|
.pipe((0, operators_1.mergeMap)(() => readFileAsObservable((0, path_1.join)(folder, file), 'utf8')), (0, operators_1.take)(1), (0, operators_1.map)((res) => res));
|
|
19
20
|
exports.loadTextFile = loadTextFile;
|
|
20
21
|
const loadJSONFile = (folder) => (file) => (0, exports.loadTextFile)(folder)(file)
|
|
21
|
-
.pipe((0, operators_1.map)(res => JSON.parse(res)));
|
|
22
|
+
.pipe((0, operators_1.map)(res => JSON.parse((0, utils_1.stripBOM)(res))));
|
|
22
23
|
exports.loadJSONFile = loadJSONFile;
|
|
23
24
|
const uploadText = (folder) => (file, data) => mkdirAsObservable(currentDir((0, path_1.join)(folder, file)), { recursive: true })
|
|
24
25
|
.pipe((0, operators_1.mergeMap)(() => writeFileAsObservable((0, path_1.join)(folder, file), data, 'utf8')), (0, operators_1.map)(() => data));
|
package/dist/local.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local.js","sourceRoot":"","sources":["../src/local.ts"],"names":[],"mappings":";;;AAAA,2BAAkE;AAClE,+BAA4B;AAC5B,+BAAwD;AACxD,8CAAqD;AAErD,MAAM,oBAAoB,GAAG,IAAA,uBAAgB,EAAC,aAAQ,CAAC,IAAI,CAAC,aAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,qBAAqB,GAAG,IAAA,uBAAgB,EAAC,cAAS,CAAC,IAAI,CAAC,cAAS,CAAC,CAAC,CAAC;AAC1E,MAAM,kBAAkB,GAAG,IAAA,uBAAgB,EAAC,WAAM,CAAC,IAAI,CAAC,WAAM,CAAC,CAAC,CAAC;AACjE,MAAM,iBAAiB,GAAG,IAAA,uBAAgB,EAAC,UAAK,CAAC,IAAI,CAAC,UAAK,CAAC,CAAC,CAAC;AAE9D,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAE;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE,CAC/D,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAChE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAa,CAAC,CACjC,CAAC;AANO,QAAA,YAAY,gBAMnB;AAEC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAI,IAAY,EAAE,EAAE,CAClE,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC;KACvB,IAAI,CACH,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"local.js","sourceRoot":"","sources":["../src/local.ts"],"names":[],"mappings":";;;AAAA,2BAAkE;AAClE,+BAA4B;AAC5B,+BAAwD;AACxD,8CAAqD;AAErD,mCAAmC;AAEnC,MAAM,oBAAoB,GAAG,IAAA,uBAAgB,EAAC,aAAQ,CAAC,IAAI,CAAC,aAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,qBAAqB,GAAG,IAAA,uBAAgB,EAAC,cAAS,CAAC,IAAI,CAAC,cAAS,CAAC,CAAC,CAAC;AAC1E,MAAM,kBAAkB,GAAG,IAAA,uBAAgB,EAAC,WAAM,CAAC,IAAI,CAAC,WAAM,CAAC,CAAC,CAAC;AACjE,MAAM,iBAAiB,GAAG,IAAA,uBAAgB,EAAC,UAAK,CAAC,IAAI,CAAC,UAAK,CAAC,CAAC,CAAC;AAE9D,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAE;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE,CAC/D,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAChE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAa,CAAC,CACjC,CAAC;AANO,QAAA,YAAY,gBAMnB;AAEC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAI,IAAY,EAAE,EAAE,CAClE,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC;KACvB,IAAI,CACH,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAA,gBAAQ,EAAC,GAAG,CAAC,CAAM,CAAC,CAC3C,CAAC;AAJO,QAAA,YAAY,gBAInB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAC3E,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EACvE,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC;AALO,QAAA,UAAU,cAKjB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAI,EAAE,EAAE,CACnE,IAAA,kBAAU,EAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAD7C,QAAA,UAAU,cACmC;AAEnD,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC/D,IAAI;QACF,OAAO,IAAA,SAAE,EAAC,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;KAC/C;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,IAAI,IAAI,EAAE,CAAC,CAAC;KACvB;AACH,CAAC,CAAC;AAPW,QAAA,YAAY,gBAOvB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC7D,IAAI;QACF,MAAM,OAAO,GAAG,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,OAAO,IAAA,SAAE,EAAC,OAAO,CAAC,CAAC;KACpB;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC;KAClB;AACH,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB;AAEK,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC3D,IAAI;QACF,OAAO,IAAA,SAAE,EAAC,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KAC9C;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,SAAmB,CAAC,CAAC;KAChC;AACH,CAAC,CAAC;AAPW,QAAA,QAAQ,YAOnB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAmB,EAAE,CAC9E,kBAAkB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAD5B,QAAA,UAAU,cACkB"}
|
package/dist/progress.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FileProgress } from '@hestia-earth/api';
|
|
1
2
|
import { IFunctionParam } from './s3';
|
|
2
3
|
export declare enum publishFunctionSuffix {
|
|
3
4
|
Success = "Done",
|
|
@@ -15,6 +16,6 @@ export declare class HestiaError extends Error {
|
|
|
15
16
|
stack: string;
|
|
16
17
|
};
|
|
17
18
|
}
|
|
18
|
-
export declare const handleError: (step:
|
|
19
|
-
export declare const handleSuccess: (step:
|
|
19
|
+
export declare const handleError: (step: FileProgress, params: IFunctionParam, time?: number, Bucket?: string) => (err: HestiaError) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
|
|
20
|
+
export declare const handleSuccess: (step: FileProgress, params: IFunctionParam, time?: number, Bucket?: string) => (details?: any) => import("rxjs").Observable<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/sns").PublishResponse, import("aws-sdk").AWSError>>;
|
|
20
21
|
export declare const defaultErrorHandler: (key: string) => (err: Error) => import("rxjs").Observable<never>;
|
package/dist/progress.js
CHANGED
|
@@ -29,13 +29,16 @@ class HestiaError extends Error {
|
|
|
29
29
|
}
|
|
30
30
|
exports.HestiaError = HestiaError;
|
|
31
31
|
const fileToProgress = (path) => `${path.split('.')[0]}.progress`;
|
|
32
|
-
const uploadProgress = (step, path, Bucket, time, err, details) => (0, s3_1.uploadJSON)(fileToProgress(path), {
|
|
32
|
+
const uploadProgress = (step, path, Bucket, time, err, details) => (0, rxjs_1.of)(fileToProgress(path)).pipe((0, operators_1.mergeMap)(filepath => (0, s3_1.loadJSONFile)(filepath, Bucket).pipe((0, operators_1.catchError)(() => (0, rxjs_1.of)({})), (0, operators_1.mergeMap)(data => (0, s3_1.uploadJSON)(fileToProgress(path), {
|
|
33
33
|
step,
|
|
34
34
|
success: !err,
|
|
35
|
-
time
|
|
35
|
+
time: {
|
|
36
|
+
...(data.time || {}),
|
|
37
|
+
step: time
|
|
38
|
+
},
|
|
36
39
|
details,
|
|
37
40
|
error: err ? err.toString() : null
|
|
38
|
-
}, 'application/json', Bucket);
|
|
41
|
+
}, 'application/json', Bucket)))));
|
|
39
42
|
const handleError = (step, params, time = 0, Bucket = s3_1.bucket) => (err) => uploadProgress(step, params.key, Bucket, time, err).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Error}`, params)));
|
|
40
43
|
exports.handleError = handleError;
|
|
41
44
|
const handleSuccess = (step, params, time = 0, Bucket = s3_1.bucket) => (details) => uploadProgress(step, params.key, Bucket, time, null, details).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publish)(`${step}${publishFunctionSuffix.Success}`, params)));
|
package/dist/progress.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,8CAAsD;AACtD,+BAAsC;AAGtC,6BAAwE;AACxE,+BAAgC;AAYhC,IAAY,qBAGX;AAHD,WAAY,qBAAqB;IAC/B,yCAAgB,CAAA;IAChB,wCAAe,CAAA;AACjB,CAAC,EAHW,qBAAqB,GAArB,6BAAqB,KAArB,6BAAqB,QAGhC;AAEM,MAAM,SAAS,GAAG,CAAC,KAAW,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;AAApE,QAAA,SAAS,aAA2D;AAEjF,MAAa,WAAY,SAAQ,KAAK;IAOpC,YAAmB,OAAO,EAAS,GAAW,EAAS,SAAc,EAAE;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC;QADE,YAAO,GAAP,OAAO,CAAA;QAAS,QAAG,GAAH,GAAG,CAAQ;QAAS,WAAM,GAAN,MAAM,CAAU;IAEvE,CAAC;IAED,QAAQ;QACN,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAlBD,kCAkBC;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;AAE1E,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,MAAc,EAAE,IAAY,EAAE,GAAiB,EAAE,OAAa,EAAE,EAAE,CACpH,IAAA,SAAE,EAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAC3B,IAAA,oBAAQ,EAAC,QAAQ,CAAC,EAAE,CAAC,IAAA,iBAAY,EAAyB,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAC9E,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,EAA4B,CAAC,CAAC,EAClD,IAAA,oBAAQ,EAAC,IAAI,CAAC,EAAE,CAAC,IAAA,eAAU,EAAC,cAAc,CAAC,IAAI,CAAC,EAAE;IAChD,IAAI;IACJ,OAAO,EAAE,CAAC,GAAG;IACb,IAAI,EAAE;QACJ,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACpB,IAAI,EAAE,IAAI;KACX;IACD,OAAO;IACP,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;CACnC,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAChC,CAAC,CACH,CAAC;AAEG,MAAM,WAAW,GAAG,CAAC,IAAkB,EAAE,MAAsB,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CACnG,CAAC,GAAgB,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAC5E,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CACzE,CAAC;AAHS,QAAA,WAAW,eAGpB;AAEG,MAAM,aAAa,GAAG,CAAC,IAAkB,EAAE,MAAsB,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CACrG,CAAC,OAAa,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CACnF,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,aAAO,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC,CAC3E,CAAC;AAHS,QAAA,aAAa,iBAGtB;AAEG,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAU,EAAE,EAAE,CACjE,IAAA,iBAAU,EAAC,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAD9C,QAAA,mBAAmB,uBAC2B"}
|
package/dist/s3.js
CHANGED
|
@@ -6,6 +6,7 @@ const fs_1 = require("fs");
|
|
|
6
6
|
const rxjs_1 = require("rxjs");
|
|
7
7
|
const operators_1 = require("rxjs/operators");
|
|
8
8
|
const local = require("./local");
|
|
9
|
+
const utils_1 = require("./utils");
|
|
9
10
|
const isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
|
|
10
11
|
exports.bucket = process.env.BUCKET;
|
|
11
12
|
const filePath = (key) => `https://${exports.bucket}.s3.amazonaws.com/${key || ''}`;
|
|
@@ -62,7 +63,7 @@ const loadJSONFile = (key, Bucket = exports.bucket) => isS3Mode ? (0, exports.ge
|
|
|
62
63
|
Bucket,
|
|
63
64
|
Key: key,
|
|
64
65
|
ResponseContentType: 'application/json'
|
|
65
|
-
}).pipe((0, operators_1.map)(res => JSON.parse(res))) : local.loadJSONFile(Bucket)(key);
|
|
66
|
+
}).pipe((0, operators_1.map)(res => JSON.parse((0, utils_1.stripBOM)(res)))) : local.loadJSONFile(Bucket)(key);
|
|
66
67
|
exports.loadJSONFile = loadJSONFile;
|
|
67
68
|
const uploadFile = (filepath, key, Bucket = exports.bucket) => (0, exports.putObject)({
|
|
68
69
|
Bucket,
|
package/dist/s3.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,8CAAuD;AAEvD,iCAAiC;
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,8CAAuD;AAEvD,iCAAiC;AACjC,mCAAmC;AAEnC,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;AAC/C,QAAA,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;AAElC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,WAAW,cAAM,qBAAqB,GAAG,IAAI,EAAE,EAAE,CAAC;AAA9E,QAAA,QAAQ,YAAsE;AAY3F,IAAI,GAAW,CAAC;AAET,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,GAAG,GAAG,GAAG,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAHW,QAAA,KAAK,SAGhB;AAEK,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CACnC,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,aAAa,GAAG,CAAC,OAAwB,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CACzE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,aAAa,CAAC;IACzB,MAAM;IACN,MAAM,EAAE;QACN,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;KAC7C;CACF,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AATS,QAAA,aAAa,iBAStB;AAEG,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,iBAA0B,EAAE,EAAE,CAC7E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;IACvB,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;IAC5B,UAAU,EAAE,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE;IACpD,iBAAiB;CAClB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AATS,QAAA,UAAU,cASnB;AAEG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC;IAC5E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,gBAAgB,EAAE,CAAC;AAHT,QAAA,UAAU,cAGD;AAEf,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;IACtC,MAAM,MAAM,GAAG,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,cAAO,EAAU,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;AAChC,CAAC,CAAC;AAdW,QAAA,QAAQ,YAcnB;AAEK,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACjF,MAAM;IACN,GAAG,EAAE,GAAG;CACT,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjE,QAAA,YAAY,gBAGqD;AAEvE,MAAM,YAAY,GAAG,CAAI,GAAW,EAAE,MAAM,GAAG,cAAM,EAAiB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACnG,MAAM;IACN,GAAG,EAAE,GAAG;IACR,mBAAmB,EAAE,kBAAkB;CACxC,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAA,gBAAQ,EAAC,GAAG,CAAC,CAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJzE,QAAA,YAAY,gBAI6D;AAE/E,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAE,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,iBAAS,EAAC;IACtF,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAA,qBAAgB,EAAC,QAAQ,EAAE,MAAM,CAAC;CACzC,CAAC,CAAC;AAJU,QAAA,UAAU,cAIpB;AAEI,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IAC7F,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;CACX,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAJ5B,QAAA,UAAU,cAIkB;AAElC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAI,EAAE,IAAI,GAAG,kBAAkB,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IAChH,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAL5B,QAAA,UAAU,cAKkB;AAElC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC3E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,OAAO,CAAC,cAAc,EAAE;QACnC,MAAM;QACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;KACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAPF,QAAA,iBAAiB,qBAOf;AAEf,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IAC3E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AAEP,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACnG,IAAA,eAAG,EAAC,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EACvC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,IAAY,CAAC,CAAC,EAClC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJvB,QAAA,YAAY,gBAIW;AAE7B,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACjG,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EACf,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC,EAC3B,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJrB,QAAA,UAAU,cAIW;AAE3B,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAC/F,IAAA,eAAG,EAAC,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,EACzC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,SAAmB,CAAC,CAAC,EACzC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJnB,QAAA,QAAQ,YAIW;AAEzB,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,YAAY,CAAC;IACjG,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjC,QAAA,YAAY,gBAGqB;AAEvC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,cAAM,EAAE,SAAkB,EAA4B,EAAE;IAC7G,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAA,aAAK,GAAE,CAAC,aAAa,CAAC;QACnF,MAAM;QACN,MAAM;QACN,iBAAiB,EAAE,SAAS;QAC5B,OAAO,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CAAC;IACb,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,IAAA,kBAAU,EAAC,MAAM,EAAE,cAAM,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxG,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB"}
|
package/dist/utils.d.ts
CHANGED
|
@@ -26,3 +26,4 @@ export declare const isUndefined: <T>(value: T, allowNull: boolean) => boolean;
|
|
|
26
26
|
export declare const reduceUndefinedValues: <T>(obj: T, allowNull?: boolean) => Partial<T>;
|
|
27
27
|
export declare const filterUndefinedValues: <T>(values: T[], allowNull?: boolean) => T[];
|
|
28
28
|
export declare const prependWithKey: (data: any, key: string) => {};
|
|
29
|
+
export declare const stripBOM: (content: string) => string;
|
package/dist/utils.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.prependWithKey = exports.filterUndefinedValues = exports.reduceUndefinedValues = exports.isUndefined = exports.refToSchemaType = exports.isEmpty = exports.spliceAndProcess = exports.returnError = exports.returnResponse = exports.functionParamToSNSRecord = exports.functionParamToS3Record = exports.recordToFunctionParam = void 0;
|
|
3
|
+
exports.stripBOM = exports.prependWithKey = exports.filterUndefinedValues = exports.reduceUndefinedValues = exports.isUndefined = exports.refToSchemaType = exports.isEmpty = exports.spliceAndProcess = exports.returnError = exports.returnResponse = exports.functionParamToSNSRecord = exports.functionParamToS3Record = exports.recordToFunctionParam = void 0;
|
|
4
4
|
const log_1 = require("./log");
|
|
5
5
|
const recordToFunctionParam = (event) => (0, exports.reduceUndefinedValues)('Sns' in event ? {
|
|
6
6
|
...JSON.parse(event.Sns.Message),
|
|
@@ -72,4 +72,6 @@ exports.filterUndefinedValues = filterUndefinedValues;
|
|
|
72
72
|
const prependWithKey = (data, key) => Object.keys(data)
|
|
73
73
|
.reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
|
|
74
74
|
exports.prependWithKey = prependWithKey;
|
|
75
|
+
const stripBOM = (content) => content.charCodeAt(0) === 0xFEFF ? content.slice(1) : content;
|
|
76
|
+
exports.stripBOM = stripBOM;
|
|
75
77
|
//# sourceMappingURL=utils.js.map
|
package/dist/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AAGA,+BAA8B;AAEvB,MAAM,qBAAqB,GAAG,CAAC,KAAqC,EAAkB,EAAE,CAC7F,IAAA,6BAAqB,EACnB,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC;IACf,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;IAChC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CAC3E,CAAC,CAAC,CAAC;IACF,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI;IAC5B,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IACxB,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IAC7B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CACnE,CACgB,CAAC;AAXT,QAAA,qBAAqB,yBAWZ;AAEf,MAAM,uBAAuB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACtF,EAAE,EAAE;QACF,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACxB,MAAM,EAAE,EAAE,GAAG,EAAE;KAChB;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA2B,CAAC;AANhB,QAAA,uBAAuB,2BAMP;AAEtB,MAAM,wBAAwB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACvF,GAAG,EAAE;QACH,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACzC;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA4B,CAAC;AALjB,QAAA,wBAAwB,4BAKP;AAEvB,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,UAAU,GAAG,GAAG,EAAE,EAAE;IAC5D,OAAO;QACL,UAAU;QACV,OAAO,EAAE;YACP,6BAA6B,EAAE,GAAG;YAClC,kCAAkC,EAAE,IAAI;SACzC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;KACpC,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB;AAEK,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,EAAE;IACjC,IAAA,WAAK,EAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACvG,OAAO,IAAA,sBAAc,EAAC,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,MAAM,IAAI,GAAG,CAAC,CAAC;AAChE,CAAC,CAAC;AAJW,QAAA,WAAW,eAItB;AAEK,MAAM,gBAAgB,GAAG,CAAO,IAAS,EAAE,IAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAC3F,IAAI,OAAO,CAAS,OAAO,CAAC,EAAE;IAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAChB,OAAO,OAAO,EAAE,CAAC;KAClB;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACnD,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAA,wBAAgB,EAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACrF,CAAC,CAAC,CAAC;AAPQ,QAAA,gBAAgB,oBAOxB;AAEE,MAAM,OAAO,GAAG,CAAC,KAAU,EAAE,OAAO,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;IAC7E,CACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO,CACpE,CAAC,CAAC;IACH,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;AANpD,QAAA,OAAO,WAM6C;AAE1D,MAAM,eAAe,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAhE,QAAA,eAAe,mBAAiD;AAGtE,MAAM,WAAW,GAAG,CAAI,KAAQ,EAAE,SAAkB,EAAE,EAAE,CAC7D,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;IAC9B,OAAO,KAAK,KAAK,WAAW,IAAI,CAC9B,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CACtG,CAAC;AAJS,QAAA,WAAW,eAIpB;AAGG,MAAM,qBAAqB,GAAG,CAAI,GAAM,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE,CACpE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;IAChD,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,IAAA,mBAAW,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC,EAAE,EAAgB,CAAC,CAAC;AAHV,QAAA,qBAAqB,yBAGX;AAEhB,MAAM,qBAAqB,GAAG,CAAI,MAAW,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE,CACzE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAA,mBAAW,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;AAD5C,QAAA,qBAAqB,yBACuB;AAElD,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;KACxE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AADhE,QAAA,cAAc,kBACkD"}
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AAGA,+BAA8B;AAEvB,MAAM,qBAAqB,GAAG,CAAC,KAAqC,EAAkB,EAAE,CAC7F,IAAA,6BAAqB,EACnB,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC;IACf,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;IAChC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CAC3E,CAAC,CAAC,CAAC;IACF,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI;IAC5B,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IACxB,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;IAC7B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;CACnE,CACgB,CAAC;AAXT,QAAA,qBAAqB,yBAWZ;AAEf,MAAM,uBAAuB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACtF,EAAE,EAAE;QACF,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACxB,MAAM,EAAE,EAAE,GAAG,EAAE;KAChB;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA2B,CAAC;AANhB,QAAA,uBAAuB,2BAMP;AAEtB,MAAM,wBAAwB,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAkB,EAAE,EAAE,CAAC,CAAC;IACvF,GAAG,EAAE;QACH,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACzC;IACD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CACxD,CAA4B,CAAC;AALjB,QAAA,wBAAwB,4BAKP;AAEvB,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,UAAU,GAAG,GAAG,EAAE,EAAE;IAC5D,OAAO;QACL,UAAU;QACV,OAAO,EAAE;YACP,6BAA6B,EAAE,GAAG;YAClC,kCAAkC,EAAE,IAAI;SACzC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;KACpC,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB;AAEK,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,EAAE;IACjC,IAAA,WAAK,EAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACvG,OAAO,IAAA,sBAAc,EAAC,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,MAAM,IAAI,GAAG,CAAC,CAAC;AAChE,CAAC,CAAC;AAJW,QAAA,WAAW,eAItB;AAEK,MAAM,gBAAgB,GAAG,CAAO,IAAS,EAAE,IAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAC3F,IAAI,OAAO,CAAS,OAAO,CAAC,EAAE;IAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAChB,OAAO,OAAO,EAAE,CAAC;KAClB;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACnD,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAA,wBAAgB,EAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACrF,CAAC,CAAC,CAAC;AAPQ,QAAA,gBAAgB,oBAOxB;AAEE,MAAM,OAAO,GAAG,CAAC,KAAU,EAAE,OAAO,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;IAC7E,CACE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO,CACpE,CAAC,CAAC;IACH,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;AANpD,QAAA,OAAO,WAM6C;AAE1D,MAAM,eAAe,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAhE,QAAA,eAAe,mBAAiD;AAGtE,MAAM,WAAW,GAAG,CAAI,KAAQ,EAAE,SAAkB,EAAE,EAAE,CAC7D,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;IAC9B,OAAO,KAAK,KAAK,WAAW,IAAI,CAC9B,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CACtG,CAAC;AAJS,QAAA,WAAW,eAIpB;AAGG,MAAM,qBAAqB,GAAG,CAAI,GAAM,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE,CACpE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;IAChD,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,IAAA,mBAAW,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC,EAAE,EAAgB,CAAC,CAAC;AAHV,QAAA,qBAAqB,yBAGX;AAEhB,MAAM,qBAAqB,GAAG,CAAI,MAAW,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE,CACzE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAA,mBAAW,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;AAD5C,QAAA,qBAAqB,yBACuB;AAElD,MAAM,cAAc,GAAG,CAAC,IAAS,EAAE,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;KACxE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AADhE,QAAA,cAAc,kBACkD;AAEtE,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAA9F,QAAA,QAAQ,YAAsF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/pipeline-utils",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Utilities for running pipelines",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"typings": "dist/index.d.ts",
|
|
@@ -38,12 +38,14 @@
|
|
|
38
38
|
"rxjs": "^6.6.7"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
|
+
"@hestia-earth/api": ">=0.18.0",
|
|
41
42
|
"@hestia-earth/json-schema": ">=15.0.0",
|
|
42
43
|
"@hestia-earth/schema": ">=15.0.0"
|
|
43
44
|
},
|
|
44
45
|
"devDependencies": {
|
|
45
46
|
"@commitlint/cli": "^17.4.2",
|
|
46
47
|
"@commitlint/config-conventional": "^17.4.2",
|
|
48
|
+
"@hestia-earth/api": "^0.18.0",
|
|
47
49
|
"@hestia-earth/eslint-config": "^0.0.5",
|
|
48
50
|
"@hestia-earth/json-schema": "^15.0.0",
|
|
49
51
|
"@hestia-earth/schema": "^15.0.0",
|
package/src/local.ts
CHANGED
|
@@ -3,6 +3,8 @@ import { join } from 'path';
|
|
|
3
3
|
import { Observable, bindNodeCallback, of } from 'rxjs';
|
|
4
4
|
import { mergeMap, map, take } from 'rxjs/operators';
|
|
5
5
|
|
|
6
|
+
import { stripBOM } from './utils';
|
|
7
|
+
|
|
6
8
|
const readFileAsObservable = bindNodeCallback(readFile.bind(readFile));
|
|
7
9
|
const writeFileAsObservable = bindNodeCallback(writeFile.bind(writeFile));
|
|
8
10
|
const unlinkAsObservable = bindNodeCallback(unlink.bind(unlink));
|
|
@@ -25,7 +27,7 @@ export const loadTextFile = (folder: string) => (file: string) =>
|
|
|
25
27
|
export const loadJSONFile = (folder: string) => <T>(file: string) =>
|
|
26
28
|
loadTextFile(folder)(file)
|
|
27
29
|
.pipe(
|
|
28
|
-
map(res => JSON.parse(res) as T)
|
|
30
|
+
map(res => JSON.parse(stripBOM(res)) as T)
|
|
29
31
|
);
|
|
30
32
|
|
|
31
33
|
export const uploadText = (folder: string) => (file: string, data: string) =>
|
package/src/progress.ts
CHANGED
|
@@ -1,9 +1,20 @@
|
|
|
1
|
-
import { mergeMap } from 'rxjs/operators';
|
|
2
|
-
import { throwError } from 'rxjs';
|
|
1
|
+
import { catchError, mergeMap } from 'rxjs/operators';
|
|
2
|
+
import { of, throwError } from 'rxjs';
|
|
3
|
+
import { FileProgress } from '@hestia-earth/api';
|
|
3
4
|
|
|
4
|
-
import { IFunctionParam, uploadJSON, bucket } from './s3';
|
|
5
|
+
import { IFunctionParam, uploadJSON, bucket, loadJSONFile } from './s3';
|
|
5
6
|
import { publish } from './sns';
|
|
6
7
|
|
|
8
|
+
interface IFileProgress {
|
|
9
|
+
step: FileProgress;
|
|
10
|
+
success: boolean;
|
|
11
|
+
time: {
|
|
12
|
+
[step in FileProgress]?: number;
|
|
13
|
+
};
|
|
14
|
+
details?: any;
|
|
15
|
+
error?: any;
|
|
16
|
+
}
|
|
17
|
+
|
|
7
18
|
export enum publishFunctionSuffix {
|
|
8
19
|
Success = 'Done',
|
|
9
20
|
Error = 'Error'
|
|
@@ -34,21 +45,29 @@ export class HestiaError extends Error {
|
|
|
34
45
|
const fileToProgress = (path: string) => `${path.split('.')[0]}.progress`;
|
|
35
46
|
|
|
36
47
|
const uploadProgress = (step: string, path: string, Bucket: string, time: number, err?: HestiaError, details?: any) =>
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
48
|
+
of(fileToProgress(path)).pipe(
|
|
49
|
+
mergeMap(filepath => loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
|
|
50
|
+
catchError(() => of({} as Partial<IFileProgress>)),
|
|
51
|
+
mergeMap(data => uploadJSON(fileToProgress(path), {
|
|
52
|
+
step,
|
|
53
|
+
success: !err,
|
|
54
|
+
time: {
|
|
55
|
+
...(data.time || {}),
|
|
56
|
+
step: time
|
|
57
|
+
},
|
|
58
|
+
details,
|
|
59
|
+
error: err ? err.toString() : null
|
|
60
|
+
}, 'application/json', Bucket))
|
|
61
|
+
))
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
export const handleError = (step: FileProgress, params: IFunctionParam, time = 0, Bucket = bucket) =>
|
|
65
|
+
(err: HestiaError) => uploadProgress(step, params.key, Bucket, time, err).pipe(
|
|
47
66
|
mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
|
|
48
67
|
);
|
|
49
68
|
|
|
50
|
-
export const handleSuccess = (step:
|
|
51
|
-
uploadProgress(step, params.key, Bucket, time, null, details).pipe(
|
|
69
|
+
export const handleSuccess = (step: FileProgress, params: IFunctionParam, time = 0, Bucket = bucket) =>
|
|
70
|
+
(details?: any) => uploadProgress(step, params.key, Bucket, time, null, details).pipe(
|
|
52
71
|
mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
|
|
53
72
|
);
|
|
54
73
|
|
package/src/s3.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { Subject, Observable, from, of } from 'rxjs';
|
|
|
4
4
|
import { catchError, map, take } from 'rxjs/operators';
|
|
5
5
|
|
|
6
6
|
import * as local from './local';
|
|
7
|
+
import { stripBOM } from './utils';
|
|
7
8
|
|
|
8
9
|
const isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
|
|
9
10
|
export const bucket = process.env.BUCKET;
|
|
@@ -91,7 +92,7 @@ export const loadJSONFile = <T>(key: string, Bucket = bucket): Observable<T> =>
|
|
|
91
92
|
Bucket,
|
|
92
93
|
Key: key,
|
|
93
94
|
ResponseContentType: 'application/json'
|
|
94
|
-
}).pipe(map(res => JSON.parse(res) as T)) : local.loadJSONFile(Bucket)(key);
|
|
95
|
+
}).pipe(map(res => JSON.parse(stripBOM(res)) as T)) : local.loadJSONFile(Bucket)(key);
|
|
95
96
|
|
|
96
97
|
export const uploadFile = (filepath: string, key: string, Bucket = bucket) => putObject({
|
|
97
98
|
Bucket,
|
package/src/utils.ts
CHANGED
|
@@ -85,3 +85,5 @@ export const filterUndefinedValues = <T>(values: T[], allowNull = false) =>
|
|
|
85
85
|
|
|
86
86
|
export const prependWithKey = (data: any, key: string) => Object.keys(data)
|
|
87
87
|
.reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
|
|
88
|
+
|
|
89
|
+
export const stripBOM = (content: string) => content.charCodeAt(0) === 0xFEFF ? content.slice(1) : content;
|