@hestia-earth/pipeline-utils 0.5.13 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/local.d.ts +1 -0
- package/dist/local.js +10 -1
- package/dist/local.js.map +1 -1
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +10 -6
- package/dist/s3.js.map +1 -1
- package/package.json +5 -5
- package/src/local.ts +9 -0
- package/src/s3.ts +8 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,33 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
## [0.7.0](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.6.0...v0.7.0) (2022-08-30)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### ⚠ BREAKING CHANGES
|
|
9
|
+
|
|
10
|
+
* **s3:** min required schema version is `10.0.0`
|
|
11
|
+
|
|
12
|
+
### Features
|
|
13
|
+
|
|
14
|
+
* **s3:** export `getObject` and `putObject` methods ([b43ba47](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/b43ba472e0775fd56beafe941858d9ebf716fdad))
|
|
15
|
+
|
|
16
|
+
## [0.6.0](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.5.14...v0.6.0) (2022-06-14)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### ⚠ BREAKING CHANGES
|
|
20
|
+
|
|
21
|
+
* **package:** schema min version is `8.0.0`
|
|
22
|
+
|
|
23
|
+
* **package:** update schema to `8.0.0` ([5df2387](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/5df2387053bccaea1ffe13e6adac2cf5b01d7d93))
|
|
24
|
+
|
|
25
|
+
### [0.5.14](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.5.13...v0.5.14) (2022-04-13)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### Features
|
|
29
|
+
|
|
30
|
+
* **s3:** add `fileSize` method ([8ed75bb](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/8ed75bb27975d18d77251429930cbd371c5da988))
|
|
31
|
+
|
|
5
32
|
### [0.5.13](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.5.12...v0.5.13) (2022-03-31)
|
|
6
33
|
|
|
7
34
|
|
package/dist/local.d.ts
CHANGED
|
@@ -5,4 +5,5 @@ export declare const uploadText: (folder: string) => (file: string, data: string
|
|
|
5
5
|
export declare const uploadJSON: (folder: string) => (file: string, data: any) => Observable<string>;
|
|
6
6
|
export declare const lastModified: (folder: string) => (file: string) => Observable<Date>;
|
|
7
7
|
export declare const fileExists: (folder: string) => (file: string) => Observable<boolean>;
|
|
8
|
+
export declare const fileSize: (folder: string) => (file: string) => Observable<number>;
|
|
8
9
|
export declare const deleteFile: (folder: string) => (file: string) => Observable<any>;
|
package/dist/local.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.deleteFile = exports.fileExists = exports.lastModified = exports.uploadJSON = exports.uploadText = exports.loadJSONFile = exports.loadTextFile = void 0;
|
|
3
|
+
exports.deleteFile = exports.fileSize = exports.fileExists = exports.lastModified = exports.uploadJSON = exports.uploadText = exports.loadJSONFile = exports.loadTextFile = void 0;
|
|
4
4
|
const fs_1 = require("fs");
|
|
5
5
|
const path_1 = require("path");
|
|
6
6
|
const rxjs_1 = require("rxjs");
|
|
@@ -44,6 +44,15 @@ const fileExists = (folder) => (file) => {
|
|
|
44
44
|
}
|
|
45
45
|
};
|
|
46
46
|
exports.fileExists = fileExists;
|
|
47
|
+
const fileSize = (folder) => (file) => {
|
|
48
|
+
try {
|
|
49
|
+
return (0, rxjs_1.of)((0, fs_1.statSync)((0, path_1.join)(folder, file)).size);
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
return (0, rxjs_1.of)(undefined);
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
exports.fileSize = fileSize;
|
|
47
56
|
const deleteFile = (folder) => (file) => unlinkAsObservable((0, path_1.join)(folder, file));
|
|
48
57
|
exports.deleteFile = deleteFile;
|
|
49
58
|
//# sourceMappingURL=local.js.map
|
package/dist/local.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local.js","sourceRoot":"","sources":["../src/local.ts"],"names":[],"mappings":";;;AAAA,2BAAkE;AAClE,+BAA4B;AAC5B,+BAAwD;AACxD,8CAAqD;AAErD,MAAM,oBAAoB,GAAG,IAAA,uBAAgB,EAAC,aAAQ,CAAC,IAAI,CAAC,aAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,qBAAqB,GAAG,IAAA,uBAAgB,EAAC,cAAS,CAAC,IAAI,CAAC,cAAS,CAAC,CAAC,CAAC;AAC1E,MAAM,kBAAkB,GAAG,IAAA,uBAAgB,EAAC,WAAM,CAAC,IAAI,CAAC,WAAM,CAAC,CAAC,CAAC;AACjE,MAAM,iBAAiB,GAAG,IAAA,uBAAgB,EAAC,UAAK,CAAC,IAAI,CAAC,UAAK,CAAC,CAAC,CAAC;AAE9D,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAE;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE,CAC/D,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAChE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAa,CAAC,CACjC,CAAC;AANO,QAAA,YAAY,gBAMnB;AAEC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAI,IAAY,EAAE,EAAE,CAClE,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC;KACvB,IAAI,CACH,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC,CACjC,CAAC;AAJO,QAAA,YAAY,gBAInB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAC3E,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EACvE,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC;AALO,QAAA,UAAU,cAKjB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAI,EAAE,EAAE,CACnE,IAAA,kBAAU,EAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAD7C,QAAA,UAAU,cACmC;AAEnD,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC/D,IAAI;QACF,OAAO,IAAA,SAAE,EAAC,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;KAC/C;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,IAAI,IAAI,EAAE,CAAC,CAAC;KACvB;AACH,CAAC,CAAC;AAPW,QAAA,YAAY,gBAOvB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC7D,IAAI;QACF,MAAM,OAAO,GAAG,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,OAAO,IAAA,SAAE,EAAC,OAAO,CAAC,CAAC;KACpB;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC;KAClB;AACH,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAmB,EAAE,CAC9E,kBAAkB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAD5B,QAAA,UAAU,cACkB"}
|
|
1
|
+
{"version":3,"file":"local.js","sourceRoot":"","sources":["../src/local.ts"],"names":[],"mappings":";;;AAAA,2BAAkE;AAClE,+BAA4B;AAC5B,+BAAwD;AACxD,8CAAqD;AAErD,MAAM,oBAAoB,GAAG,IAAA,uBAAgB,EAAC,aAAQ,CAAC,IAAI,CAAC,aAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,qBAAqB,GAAG,IAAA,uBAAgB,EAAC,cAAS,CAAC,IAAI,CAAC,cAAS,CAAC,CAAC,CAAC;AAC1E,MAAM,kBAAkB,GAAG,IAAA,uBAAgB,EAAC,WAAM,CAAC,IAAI,CAAC,WAAM,CAAC,CAAC,CAAC;AACjE,MAAM,iBAAiB,GAAG,IAAA,uBAAgB,EAAC,UAAK,CAAC,IAAI,CAAC,UAAK,CAAC,CAAC,CAAC;AAE9D,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAE;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE,CAC/D,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAChE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAa,CAAC,CACjC,CAAC;AANO,QAAA,YAAY,gBAMnB;AAEC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAI,IAAY,EAAE,EAAE,CAClE,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC;KACvB,IAAI,CACH,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC,CACjC,CAAC;AAJO,QAAA,YAAY,gBAInB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAC3E,iBAAiB,CAAC,UAAU,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACnE,IAAI,CACH,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EACvE,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC;AALO,QAAA,UAAU,cAKjB;AAEC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,IAAI,EAAE,EAAE,CACnE,IAAA,kBAAU,EAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAD7C,QAAA,UAAU,cACmC;AAEnD,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC/D,IAAI;QACF,OAAO,IAAA,SAAE,EAAC,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;KAC/C;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,IAAI,IAAI,EAAE,CAAC,CAAC;KACvB;AACH,CAAC,CAAC;AAPW,QAAA,YAAY,gBAOvB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC7D,IAAI;QACF,MAAM,OAAO,GAAG,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,OAAO,IAAA,SAAE,EAAC,OAAO,CAAC,CAAC;KACpB;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC;KAClB;AACH,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB;AAEK,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE;IAC3D,IAAI;QACF,OAAO,IAAA,SAAE,EAAC,IAAA,aAAQ,EAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KAC9C;IACD,OAAO,GAAG,EAAE;QACV,OAAO,IAAA,SAAE,EAAC,SAAmB,CAAC,CAAC;KAChC;AACH,CAAC,CAAC;AAPW,QAAA,QAAQ,YAOnB;AAEK,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,IAAY,EAAmB,EAAE,CAC9E,kBAAkB,CAAC,IAAA,WAAI,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAD5B,QAAA,UAAU,cACkB"}
|
package/dist/s3.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export interface IFunctionParam {
|
|
|
10
10
|
timestamp?: Date;
|
|
11
11
|
}
|
|
12
12
|
export declare const getS3: () => AWS.S3;
|
|
13
|
+
export declare const getObject: ({ Key, ...params }: AWS.S3.GetObjectRequest) => Observable<string>;
|
|
14
|
+
export declare const putObject: ({ Key, ...params }: AWS.S3.PutObjectRequest) => Observable<string>;
|
|
13
15
|
export declare const copyObject: (fromParams: any, toParams: any, MetadataDirective?: string) => Observable<string>;
|
|
14
16
|
export declare const loadStream: (key: string, Bucket?: string) => import("stream").Readable;
|
|
15
17
|
export declare const loadFile: (key: string) => Observable<Buffer>;
|
|
@@ -21,5 +23,6 @@ export declare const uploadJSON: (key: string, data: any, type?: string, Bucket?
|
|
|
21
23
|
export declare const waitObjectCreated: (key: string, Bucket?: string) => Observable<boolean>;
|
|
22
24
|
export declare const lastModified: (key: string, Bucket?: string) => Observable<Date>;
|
|
23
25
|
export declare const fileExists: (key: string, Bucket?: string) => Observable<boolean>;
|
|
26
|
+
export declare const fileSize: (key: string, Bucket?: string) => Observable<number>;
|
|
24
27
|
export declare const deleteObject: (key: string, Bucket?: string) => Observable<any>;
|
|
25
28
|
export declare const listFolder: (Prefix?: string, Bucket?: string, nextToken?: string) => Promise<AWS.S3.Object[]>;
|
package/dist/s3.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.listFolder = exports.deleteObject = exports.fileExists = exports.lastModified = exports.waitObjectCreated = exports.uploadJSON = exports.uploadText = exports.uploadFile = exports.loadJSONFile = exports.loadTextFile = exports.loadFile = exports.loadStream = exports.copyObject = exports.getS3 = exports.filePath = exports.bucket = void 0;
|
|
3
|
+
exports.listFolder = exports.deleteObject = exports.fileSize = exports.fileExists = exports.lastModified = exports.waitObjectCreated = exports.uploadJSON = exports.uploadText = exports.uploadFile = exports.loadJSONFile = exports.loadTextFile = exports.loadFile = exports.loadStream = exports.copyObject = exports.putObject = exports.getObject = exports.getS3 = exports.filePath = exports.bucket = void 0;
|
|
4
4
|
const AWS = require("aws-sdk");
|
|
5
5
|
const fs_1 = require("fs");
|
|
6
6
|
const rxjs_1 = require("rxjs");
|
|
@@ -17,7 +17,9 @@ const getS3 = () => {
|
|
|
17
17
|
};
|
|
18
18
|
exports.getS3 = getS3;
|
|
19
19
|
const getObject = ({ Key, ...params }) => (0, rxjs_1.from)((0, exports.getS3)().getObject({ ...params, Key: decodeURI(Key) }).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(({ Body }) => Body.toString()));
|
|
20
|
+
exports.getObject = getObject;
|
|
20
21
|
const putObject = ({ Key, ...params }) => (0, rxjs_1.from)((0, exports.getS3)().putObject({ ...params, Key: decodeURI(Key) }).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => (res.$response.data || '').toString()));
|
|
22
|
+
exports.putObject = putObject;
|
|
21
23
|
const copyObject = (fromParams, toParams, MetadataDirective) => (0, rxjs_1.from)((0, exports.getS3)().copyObject({
|
|
22
24
|
Bucket: toParams.bucket,
|
|
23
25
|
Key: decodeURI(toParams.key),
|
|
@@ -44,30 +46,30 @@ const loadFile = (key) => {
|
|
|
44
46
|
return subject.asObservable();
|
|
45
47
|
};
|
|
46
48
|
exports.loadFile = loadFile;
|
|
47
|
-
const loadTextFile = (key, Bucket = exports.bucket) => isS3Mode ? getObject({
|
|
49
|
+
const loadTextFile = (key, Bucket = exports.bucket) => isS3Mode ? (0, exports.getObject)({
|
|
48
50
|
Bucket,
|
|
49
51
|
Key: key
|
|
50
52
|
}).pipe((0, operators_1.map)(res => res.replace(/\"/g, ''))) : local.loadTextFile(Bucket)(key);
|
|
51
53
|
exports.loadTextFile = loadTextFile;
|
|
52
|
-
const loadJSONFile = (key, Bucket = exports.bucket) => isS3Mode ? getObject({
|
|
54
|
+
const loadJSONFile = (key, Bucket = exports.bucket) => isS3Mode ? (0, exports.getObject)({
|
|
53
55
|
Bucket,
|
|
54
56
|
Key: key,
|
|
55
57
|
ResponseContentType: 'application/json'
|
|
56
58
|
}).pipe((0, operators_1.map)(res => JSON.parse(res))) : local.loadJSONFile(Bucket)(key);
|
|
57
59
|
exports.loadJSONFile = loadJSONFile;
|
|
58
|
-
const uploadFile = (filepath, key, Bucket = exports.bucket) => putObject({
|
|
60
|
+
const uploadFile = (filepath, key, Bucket = exports.bucket) => (0, exports.putObject)({
|
|
59
61
|
Bucket,
|
|
60
62
|
Key: key,
|
|
61
63
|
Body: (0, fs_1.createReadStream)(filepath, 'utf8')
|
|
62
64
|
});
|
|
63
65
|
exports.uploadFile = uploadFile;
|
|
64
|
-
const uploadText = (key, data, Bucket = exports.bucket) => isS3Mode ? putObject({
|
|
66
|
+
const uploadText = (key, data, Bucket = exports.bucket) => isS3Mode ? (0, exports.putObject)({
|
|
65
67
|
Bucket,
|
|
66
68
|
Key: key,
|
|
67
69
|
Body: data
|
|
68
70
|
}) : local.uploadText(Bucket)(key, data);
|
|
69
71
|
exports.uploadText = uploadText;
|
|
70
|
-
const uploadJSON = (key, data, type = 'application/json', Bucket = exports.bucket) => isS3Mode ? putObject({
|
|
72
|
+
const uploadJSON = (key, data, type = 'application/json', Bucket = exports.bucket) => isS3Mode ? (0, exports.putObject)({
|
|
71
73
|
Bucket,
|
|
72
74
|
Key: key,
|
|
73
75
|
Body: JSON.stringify(data, null, 2),
|
|
@@ -88,6 +90,8 @@ const lastModified = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key
|
|
|
88
90
|
exports.lastModified = lastModified;
|
|
89
91
|
const fileExists = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(() => true), (0, operators_1.catchError)(() => (0, rxjs_1.of)(false)), (0, operators_1.take)(1)) : local.fileExists(Bucket)(key);
|
|
90
92
|
exports.fileExists = fileExists;
|
|
93
|
+
const fileSize = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(({ ContentLength }) => ContentLength), (0, operators_1.catchError)(() => (0, rxjs_1.of)(undefined)), (0, operators_1.take)(1)) : local.fileSize(Bucket)(key);
|
|
94
|
+
exports.fileSize = fileSize;
|
|
91
95
|
const deleteObject = (key, Bucket = exports.bucket) => isS3Mode ? (0, rxjs_1.from)((0, exports.getS3)().deleteObject({
|
|
92
96
|
Bucket,
|
|
93
97
|
Key: decodeURI(key)
|
package/dist/s3.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,8CAAuD;AAEvD,iCAAiC;AAEjC,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;AAC/C,QAAA,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;AAElC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,WAAW,cAAM,qBAAqB,GAAG,IAAI,EAAE,EAAE,CAAC;AAA9E,QAAA,QAAQ,YAAsE;AAY3F,IAAI,GAAW,CAAC;AAET,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,GAAG,GAAG,GAAG,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAHW,QAAA,KAAK,SAGhB;
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,8CAAuD;AAEvD,iCAAiC;AAEjC,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;AAC/C,QAAA,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;AAElC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,WAAW,cAAM,qBAAqB,GAAG,IAAI,EAAE,EAAE,CAAC;AAA9E,QAAA,QAAQ,YAAsE;AAY3F,IAAI,GAAW,CAAC;AAET,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,GAAG,GAAG,GAAG,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAHW,QAAA,KAAK,SAGhB;AAEK,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CACnC,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,iBAA0B,EAAE,EAAE,CAC7E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;IACvB,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;IAC5B,UAAU,EAAE,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE;IACpD,iBAAiB;CAClB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AATS,QAAA,UAAU,cASnB;AAEG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC;IAC5E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,gBAAgB,EAAE,CAAC;AAHT,QAAA,UAAU,cAGD;AAEf,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;IACtC,MAAM,MAAM,GAAG,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,cAAO,EAAU,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;AAChC,CAAC,CAAC;AAZW,QAAA,QAAQ,YAYnB;AAEK,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACjF,MAAM;IACN,GAAG,EAAE,GAAG;CACT,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjE,QAAA,YAAY,gBAGqD;AAEvE,MAAM,YAAY,GAAG,CAAI,GAAW,EAAE,MAAM,GAAG,cAAM,EAAiB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACnG,MAAM;IACN,GAAG,EAAE,GAAG;IACR,mBAAmB,EAAE,kBAAkB;CACxC,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJ/D,QAAA,YAAY,gBAImD;AAErE,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAE,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,iBAAS,EAAC;IACtF,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAA,qBAAgB,EAAC,QAAQ,EAAE,MAAM,CAAC;CACzC,CAAC,CAAC;AAJU,QAAA,UAAU,cAIpB;AAEI,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAI,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACrF,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;CACX,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAJ5B,QAAA,UAAU,cAIkB;AAElC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAI,EAAE,IAAI,GAAG,kBAAkB,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IAChH,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAL5B,QAAA,UAAU,cAKkB;AAElC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC3E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,OAAO,CAAC,cAAc,EAAE;QACnC,MAAM;QACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;KACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAPF,QAAA,iBAAiB,qBAOf;AAEf,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IAC3E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AAEP,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACnG,IAAA,eAAG,EAAC,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EACvC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,IAAY,CAAC,CAAC,EAClC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJvB,QAAA,YAAY,gBAIW;AAE7B,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACjG,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EACf,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC,EAC3B,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJrB,QAAA,UAAU,cAIW;AAE3B,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAC/F,IAAA,eAAG,EAAC,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,EACzC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,SAAmB,CAAC,CAAC,EACzC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJnB,QAAA,QAAQ,YAIW;AAEzB,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,YAAY,CAAC;IACjG,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjC,QAAA,YAAY,gBAGqB;AAEvC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,cAAM,EAAE,SAAkB,EAA4B,EAAE;IAC7G,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAA,aAAK,GAAE,CAAC,aAAa,CAAC;QACnF,MAAM;QACN,MAAM;QACN,iBAAiB,EAAE,SAAS;QAC5B,OAAO,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CAAC;IACb,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,IAAA,kBAAU,EAAC,MAAM,EAAE,cAAM,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxG,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/pipeline-utils",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Utilities for running pipelines",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"typings": "dist/index.d.ts",
|
|
@@ -38,14 +38,14 @@
|
|
|
38
38
|
"rxjs": "^6.6.7"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
|
-
"@hestia-earth/json-schema": "^
|
|
42
|
-
"@hestia-earth/schema": "^
|
|
41
|
+
"@hestia-earth/json-schema": "^10.0.0",
|
|
42
|
+
"@hestia-earth/schema": "^10.0.0"
|
|
43
43
|
},
|
|
44
44
|
"devDependencies": {
|
|
45
45
|
"@commitlint/cli": "^16.2.3",
|
|
46
46
|
"@commitlint/config-conventional": "^16.2.1",
|
|
47
|
-
"@hestia-earth/json-schema": "^
|
|
48
|
-
"@hestia-earth/schema": "^
|
|
47
|
+
"@hestia-earth/json-schema": "^10.0.0",
|
|
48
|
+
"@hestia-earth/schema": "^10.0.0",
|
|
49
49
|
"@types/aws-lambda": "^8.10.93",
|
|
50
50
|
"@types/aws-sdk": "^2.7.0",
|
|
51
51
|
"@types/chai": "^4.3.0",
|
package/src/local.ts
CHANGED
|
@@ -57,5 +57,14 @@ export const fileExists = (folder: string) => (file: string) => {
|
|
|
57
57
|
}
|
|
58
58
|
};
|
|
59
59
|
|
|
60
|
+
export const fileSize = (folder: string) => (file: string) => {
|
|
61
|
+
try {
|
|
62
|
+
return of(statSync(join(folder, file)).size);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
return of(undefined as number);
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
|
|
60
69
|
export const deleteFile = (folder: string) => (file: string): Observable<any> =>
|
|
61
70
|
unlinkAsObservable(join(folder, file));
|
package/src/s3.ts
CHANGED
|
@@ -27,13 +27,13 @@ export const getS3 = () => {
|
|
|
27
27
|
return _s3;
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
-
const getObject = ({ Key, ...params }: AWS.S3.GetObjectRequest) =>
|
|
30
|
+
export const getObject = ({ Key, ...params }: AWS.S3.GetObjectRequest) =>
|
|
31
31
|
from(getS3().getObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
|
|
32
32
|
take(1),
|
|
33
33
|
map(({ Body }) => Body.toString())
|
|
34
34
|
);
|
|
35
35
|
|
|
36
|
-
const putObject = ({ Key, ...params }: AWS.S3.PutObjectRequest) =>
|
|
36
|
+
export const putObject = ({ Key, ...params }: AWS.S3.PutObjectRequest) =>
|
|
37
37
|
from(getS3().putObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
|
|
38
38
|
take(1),
|
|
39
39
|
map(res => (res.$response.data || '').toString())
|
|
@@ -125,6 +125,12 @@ export const fileExists = (key: string, Bucket = bucket) => isS3Mode ? headObjec
|
|
|
125
125
|
take(1)
|
|
126
126
|
) : local.fileExists(Bucket)(key);
|
|
127
127
|
|
|
128
|
+
export const fileSize = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
|
|
129
|
+
map(({ ContentLength }) => ContentLength),
|
|
130
|
+
catchError(() => of(undefined as number)),
|
|
131
|
+
take(1)
|
|
132
|
+
) : local.fileSize(Bucket)(key);
|
|
133
|
+
|
|
128
134
|
export const deleteObject = (key: string, Bucket = bucket) => isS3Mode ? from(getS3().deleteObject({
|
|
129
135
|
Bucket,
|
|
130
136
|
Key: decodeURI(key)
|