@hestia-earth/pipeline-utils 0.9.3 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/progress.d.ts +2 -2
- package/dist/s3.d.ts +2 -0
- package/dist/s3.js +16 -11
- package/dist/s3.js.map +1 -1
- package/dist/sns.d.ts +1 -2
- package/dist/sns.js +15 -12
- package/dist/sns.js.map +1 -1
- package/package.json +7 -7
- package/src/s3.ts +12 -2
- package/src/sns.ts +5 -4
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
4
|
|
|
5
|
+
### [0.9.5](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.9.4...v0.9.5) (2023-05-18)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
* **sns:** disable in local mode ([7c234aa](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/7c234aabbfbaa495a38a3e630b5618289898daf8))
|
|
11
|
+
|
|
12
|
+
### [0.9.4](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.9.3...v0.9.4) (2023-05-03)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Features
|
|
16
|
+
|
|
17
|
+
* **s3:** add function to detect if file is too large ([ceeff46](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/ceeff46035cef3fb2c67df00e56fe35020173429))
|
|
18
|
+
|
|
5
19
|
### [0.9.3](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.9.2...v0.9.3) (2023-04-28)
|
|
6
20
|
|
|
7
21
|
|
package/dist/progress.d.ts
CHANGED
|
@@ -16,6 +16,6 @@ export declare class HestiaError extends Error {
|
|
|
16
16
|
stack: string;
|
|
17
17
|
};
|
|
18
18
|
}
|
|
19
|
-
export declare const handleError: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (err: HestiaError) => import("rxjs").Observable<
|
|
20
|
-
export declare const handleSuccess: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (details?: any) => import("rxjs").Observable<
|
|
19
|
+
export declare const handleError: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (err: HestiaError) => import("rxjs").Observable<any>;
|
|
20
|
+
export declare const handleSuccess: (step: FileProgress, params: IFunctionParam, since?: Date, Bucket?: string) => (details?: any) => import("rxjs").Observable<any>;
|
|
21
21
|
export declare const defaultErrorHandler: (key: string) => (err: Error) => import("rxjs").Observable<never>;
|
package/dist/s3.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import * as AWS from 'aws-sdk';
|
|
3
3
|
import { Observable } from 'rxjs';
|
|
4
|
+
export declare const isS3Mode: boolean;
|
|
4
5
|
export declare const bucket: string;
|
|
5
6
|
export declare const filePath: (key: string) => string;
|
|
6
7
|
export interface IFunctionParam {
|
|
@@ -27,3 +28,4 @@ export declare const fileExists: (key: string, Bucket?: string) => Observable<bo
|
|
|
27
28
|
export declare const fileSize: (key: string, Bucket?: string) => Observable<number>;
|
|
28
29
|
export declare const deleteObject: (key: string, Bucket?: string) => Observable<any>;
|
|
29
30
|
export declare const listFolder: (Prefix?: string, Bucket?: string, nextToken?: string) => Promise<AWS.S3.Object[]>;
|
|
31
|
+
export declare const isLargeFile: (key: string, Bucket?: string) => Observable<boolean>;
|
package/dist/s3.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.listFolder = exports.deleteObject = exports.fileSize = exports.fileExists = exports.lastModified = exports.waitObjectCreated = exports.uploadJSON = exports.uploadText = exports.uploadFile = exports.loadJSONFile = exports.loadTextFile = exports.loadFile = exports.loadStream = exports.copyObject = exports.deleteObjects = exports.putObject = exports.getObject = exports.getS3 = exports.filePath = exports.bucket = void 0;
|
|
3
|
+
exports.isLargeFile = exports.listFolder = exports.deleteObject = exports.fileSize = exports.fileExists = exports.lastModified = exports.waitObjectCreated = exports.uploadJSON = exports.uploadText = exports.uploadFile = exports.loadJSONFile = exports.loadTextFile = exports.loadFile = exports.loadStream = exports.copyObject = exports.deleteObjects = exports.putObject = exports.getObject = exports.getS3 = exports.filePath = exports.bucket = exports.isS3Mode = void 0;
|
|
4
4
|
const AWS = require("aws-sdk");
|
|
5
5
|
const fs_1 = require("fs");
|
|
6
6
|
const rxjs_1 = require("rxjs");
|
|
7
7
|
const operators_1 = require("rxjs/operators");
|
|
8
|
+
const api_1 = require("@hestia-earth/api");
|
|
9
|
+
const log_1 = require("./log");
|
|
8
10
|
const local = require("./local");
|
|
9
11
|
const utils_1 = require("./utils");
|
|
10
|
-
|
|
12
|
+
exports.isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
|
|
11
13
|
exports.bucket = process.env.BUCKET;
|
|
12
14
|
const filePath = (key) => `https://${exports.bucket}.s3.amazonaws.com/${key || ''}`;
|
|
13
15
|
exports.filePath = filePath;
|
|
@@ -54,12 +56,12 @@ const loadFile = (key) => {
|
|
|
54
56
|
return subject.asObservable();
|
|
55
57
|
};
|
|
56
58
|
exports.loadFile = loadFile;
|
|
57
|
-
const loadTextFile = (key, Bucket = exports.bucket) => isS3Mode ? (0, exports.getObject)({
|
|
59
|
+
const loadTextFile = (key, Bucket = exports.bucket) => exports.isS3Mode ? (0, exports.getObject)({
|
|
58
60
|
Bucket,
|
|
59
61
|
Key: key
|
|
60
62
|
}).pipe((0, operators_1.map)(res => res.replace(/\"/g, ''))) : local.loadTextFile(Bucket)(key);
|
|
61
63
|
exports.loadTextFile = loadTextFile;
|
|
62
|
-
const loadJSONFile = (key, Bucket = exports.bucket) => isS3Mode ? (0, exports.getObject)({
|
|
64
|
+
const loadJSONFile = (key, Bucket = exports.bucket) => exports.isS3Mode ? (0, exports.getObject)({
|
|
63
65
|
Bucket,
|
|
64
66
|
Key: key,
|
|
65
67
|
ResponseContentType: 'application/json'
|
|
@@ -71,20 +73,20 @@ const uploadFile = (filepath, key, Bucket = exports.bucket) => (0, exports.putOb
|
|
|
71
73
|
Body: (0, fs_1.createReadStream)(filepath, 'utf8')
|
|
72
74
|
});
|
|
73
75
|
exports.uploadFile = uploadFile;
|
|
74
|
-
const uploadText = (key, data, Bucket = exports.bucket) => isS3Mode ? (0, exports.putObject)({
|
|
76
|
+
const uploadText = (key, data, Bucket = exports.bucket) => exports.isS3Mode ? (0, exports.putObject)({
|
|
75
77
|
Bucket,
|
|
76
78
|
Key: key,
|
|
77
79
|
Body: data
|
|
78
80
|
}) : local.uploadText(Bucket)(key, data);
|
|
79
81
|
exports.uploadText = uploadText;
|
|
80
|
-
const uploadJSON = (key, data, type = 'application/json', Bucket = exports.bucket) => isS3Mode ? (0, exports.putObject)({
|
|
82
|
+
const uploadJSON = (key, data, type = 'application/json', Bucket = exports.bucket) => exports.isS3Mode ? (0, exports.putObject)({
|
|
81
83
|
Bucket,
|
|
82
84
|
Key: key,
|
|
83
85
|
Body: JSON.stringify(data, null, 2),
|
|
84
86
|
ContentType: type
|
|
85
87
|
}) : local.uploadJSON(Bucket)(key, data);
|
|
86
88
|
exports.uploadJSON = uploadJSON;
|
|
87
|
-
const waitObjectCreated = (key, Bucket = exports.bucket) => isS3Mode ?
|
|
89
|
+
const waitObjectCreated = (key, Bucket = exports.bucket) => exports.isS3Mode ?
|
|
88
90
|
(0, rxjs_1.from)((0, exports.getS3)().waitFor('objectExists', {
|
|
89
91
|
Bucket,
|
|
90
92
|
Key: decodeURI(key)
|
|
@@ -94,13 +96,13 @@ const headObject = (key, Bucket = exports.bucket) => (0, rxjs_1.from)((0, export
|
|
|
94
96
|
Bucket,
|
|
95
97
|
Key: decodeURI(key)
|
|
96
98
|
}).promise());
|
|
97
|
-
const lastModified = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(({ LastModified }) => LastModified), (0, operators_1.catchError)(() => (0, rxjs_1.of)(null)), (0, operators_1.take)(1)) : local.lastModified(Bucket)(key);
|
|
99
|
+
const lastModified = (key, Bucket = exports.bucket) => exports.isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(({ LastModified }) => LastModified), (0, operators_1.catchError)(() => (0, rxjs_1.of)(null)), (0, operators_1.take)(1)) : local.lastModified(Bucket)(key);
|
|
98
100
|
exports.lastModified = lastModified;
|
|
99
|
-
const fileExists = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(() => true), (0, operators_1.catchError)(() => (0, rxjs_1.of)(false)), (0, operators_1.take)(1)) : local.fileExists(Bucket)(key);
|
|
101
|
+
const fileExists = (key, Bucket = exports.bucket) => exports.isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(() => true), (0, operators_1.catchError)(() => (0, rxjs_1.of)(false)), (0, operators_1.take)(1)) : local.fileExists(Bucket)(key);
|
|
100
102
|
exports.fileExists = fileExists;
|
|
101
|
-
const fileSize = (key, Bucket = exports.bucket) => isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(({ ContentLength }) => ContentLength), (0, operators_1.catchError)(() => (0, rxjs_1.of)(undefined)), (0, operators_1.take)(1)) : local.fileSize(Bucket)(key);
|
|
103
|
+
const fileSize = (key, Bucket = exports.bucket) => exports.isS3Mode ? headObject(key, Bucket).pipe((0, operators_1.map)(({ ContentLength }) => ContentLength), (0, operators_1.catchError)(() => (0, rxjs_1.of)(undefined)), (0, operators_1.take)(1)) : local.fileSize(Bucket)(key);
|
|
102
104
|
exports.fileSize = fileSize;
|
|
103
|
-
const deleteObject = (key, Bucket = exports.bucket) => isS3Mode ? (0, rxjs_1.from)((0, exports.getS3)().deleteObject({
|
|
105
|
+
const deleteObject = (key, Bucket = exports.bucket) => exports.isS3Mode ? (0, rxjs_1.from)((0, exports.getS3)().deleteObject({
|
|
104
106
|
Bucket,
|
|
105
107
|
Key: decodeURI(key)
|
|
106
108
|
}).promise()) : local.deleteFile(Bucket)(key);
|
|
@@ -115,4 +117,7 @@ const listFolder = async (Prefix = '', Bucket = exports.bucket, nextToken) => {
|
|
|
115
117
|
return [...Contents, ...(IsTruncated ? await (0, exports.listFolder)(Prefix, exports.bucket, NextContinuationToken) : [])];
|
|
116
118
|
};
|
|
117
119
|
exports.listFolder = listFolder;
|
|
120
|
+
const largeFileSize = +(process.env.LARGE_SIZE_MB || '100');
|
|
121
|
+
const isLargeFile = (key, Bucket = exports.bucket) => (0, exports.fileSize)(key, Bucket).pipe((0, operators_1.tap)(size => (0, log_1.debug)('Detected file size:', (0, api_1.sizeInMb)(size), 'max size:', largeFileSize)), (0, operators_1.map)(size => (0, api_1.sizeInMb)(size) > largeFileSize));
|
|
122
|
+
exports.isLargeFile = isLargeFile;
|
|
118
123
|
//# sourceMappingURL=s3.js.map
|
package/dist/s3.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,2BAAsC;AACtC,+BAAqD;AACrD,8CAA4D;AAC5D,2CAA6C;AAE7C,+BAA8B;AAC9B,iCAAiC;AACjC,mCAAmC;AAEtB,QAAA,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;AACtD,QAAA,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;AAElC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,WAAW,cAAM,qBAAqB,GAAG,IAAI,EAAE,EAAE,CAAC;AAA9E,QAAA,QAAQ,YAAsE;AAY3F,IAAI,GAAW,CAAC;AAET,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,GAAG,GAAG,GAAG,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAHW,QAAA,KAAK,SAGhB;AAEK,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CACnC,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,MAAM,EAA2B,EAAE,EAAE,CACvE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACxE,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AAJS,QAAA,SAAS,aAIlB;AAEG,MAAM,aAAa,GAAG,CAAC,OAAwB,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CACzE,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,aAAa,CAAC;IACzB,MAAM;IACN,MAAM,EAAE;QACN,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;KAC7C;CACF,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AATS,QAAA,aAAa,iBAStB;AAEG,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,iBAA0B,EAAE,EAAE,CAC7E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;IACvB,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;IAC5B,UAAU,EAAE,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE;IACpD,iBAAiB;CAClB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;AATS,QAAA,UAAU,cASnB;AAEG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,aAAK,GAAE,CAAC,SAAS,CAAC;IAC5E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,gBAAgB,EAAE,CAAC;AAHT,QAAA,UAAU,cAGD;AAEf,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;IACtC,MAAM,MAAM,GAAG,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,cAAO,EAAU,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;AAChC,CAAC,CAAC;AAdW,QAAA,QAAQ,YAcnB;AAEK,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACjF,MAAM;IACN,GAAG,EAAE,GAAG;CACT,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjE,QAAA,YAAY,gBAGqD;AAEvE,MAAM,YAAY,GAAG,CAAI,GAAW,EAAE,MAAM,GAAG,cAAM,EAAiB,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IACnG,MAAM;IACN,GAAG,EAAE,GAAG;IACR,mBAAmB,EAAE,kBAAkB;CACxC,CAAC,CAAC,IAAI,CAAC,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAA,gBAAQ,EAAC,GAAG,CAAC,CAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJzE,QAAA,YAAY,gBAI6D;AAE/E,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAE,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,iBAAS,EAAC;IACtF,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAA,qBAAgB,EAAC,QAAQ,EAAE,MAAM,CAAC;CACzC,CAAC,CAAC;AAJU,QAAA,UAAU,cAIpB;AAEI,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IAC7F,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;CACX,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAJ5B,QAAA,UAAU,cAIkB;AAElC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAI,EAAE,IAAI,GAAG,kBAAkB,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,IAAA,iBAAS,EAAC;IAChH,MAAM;IACN,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAL5B,QAAA,UAAU,cAKkB;AAElC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC;IAC3E,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,OAAO,CAAC,cAAc,EAAE;QACnC,MAAM;QACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;KACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAChB,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAChB,CAAC,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAPF,QAAA,iBAAiB,qBAOf;AAEf,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,UAAU,CAAC;IAC3E,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AAEP,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACnG,IAAA,eAAG,EAAC,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EACvC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,IAAY,CAAC,CAAC,EAClC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJvB,QAAA,YAAY,gBAIW;AAE7B,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACjG,IAAA,eAAG,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EACf,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,KAAK,CAAC,CAAC,EAC3B,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJrB,QAAA,UAAU,cAIW;AAE3B,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAC/F,IAAA,eAAG,EAAC,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,EACzC,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,SAAmB,CAAC,CAAC,EACzC,IAAA,gBAAI,EAAC,CAAC,CAAC,CACR,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAJnB,QAAA,QAAQ,YAIW;AAEzB,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAAC,gBAAQ,CAAC,CAAC,CAAC,IAAA,WAAI,EAAC,IAAA,aAAK,GAAE,CAAC,YAAY,CAAC;IACjG,MAAM;IACN,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;CACpB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAHjC,QAAA,YAAY,gBAGqB;AAEvC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,cAAM,EAAE,SAAkB,EAA4B,EAAE;IAC7G,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAA,aAAK,GAAE,CAAC,aAAa,CAAC;QACnF,MAAM;QACN,MAAM;QACN,iBAAiB,EAAE,SAAS;QAC5B,OAAO,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CAAC;IACb,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,IAAA,kBAAU,EAAC,MAAM,EAAE,cAAM,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxG,CAAC,CAAC;AARW,QAAA,UAAU,cAQrB;AAEF,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;AAErD,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,cAAM,EAAE,EAAE,CAC1D,IAAA,gBAAQ,EAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CACxB,IAAA,eAAG,EAAC,IAAI,CAAC,EAAE,CAAC,IAAA,WAAK,EAAC,qBAAqB,EAAE,IAAA,cAAQ,EAAC,IAAI,CAAC,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,EACrF,IAAA,eAAG,EAAC,IAAI,CAAC,EAAE,CAAC,IAAA,cAAQ,EAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAC5C,CAAC;AAJS,QAAA,WAAW,eAIpB"}
|
package/dist/sns.d.ts
CHANGED
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
import * as AWS from 'aws-sdk';
|
|
2
1
|
import { IFunctionParam } from './s3';
|
|
3
|
-
export declare const publish: (functionName: string, params: IFunctionParam, topicArn?: string) => import("rxjs").Observable<
|
|
2
|
+
export declare const publish: (functionName: string, params: IFunctionParam, topicArn?: string) => import("rxjs").Observable<any>;
|
package/dist/sns.js
CHANGED
|
@@ -5,18 +5,21 @@ const AWS = require("aws-sdk");
|
|
|
5
5
|
const rxjs_1 = require("rxjs");
|
|
6
6
|
const operators_1 = require("rxjs/operators");
|
|
7
7
|
const log_1 = require("./log");
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
const s3_1 = require("./s3");
|
|
9
|
+
const publish = (functionName, params, topicArn = process.env.SNS_TOPIC) => s3_1.isS3Mode
|
|
10
|
+
? (0, rxjs_1.of)(null)
|
|
11
|
+
: (0, rxjs_1.from)(new AWS.SNS().publish({
|
|
12
|
+
TopicArn: topicArn,
|
|
13
|
+
Message: JSON.stringify(params),
|
|
14
|
+
MessageAttributes: {
|
|
15
|
+
functionName: {
|
|
16
|
+
DataType: 'String',
|
|
17
|
+
StringValue: functionName
|
|
18
|
+
}
|
|
15
19
|
}
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}));
|
|
20
|
+
}).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
|
|
21
|
+
(0, log_1.debug)(`publish ${functionName}`, res);
|
|
22
|
+
return res;
|
|
23
|
+
}));
|
|
21
24
|
exports.publish = publish;
|
|
22
25
|
//# sourceMappingURL=sns.js.map
|
package/dist/sns.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+
|
|
1
|
+
{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAgD;AAEzC,MAAM,OAAO,GAAG,CAAC,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,aAAQ;IACjH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC;IACV,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;SACF;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,WAAW,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH,CAAC;AAnBS,QAAA,OAAO,WAmBhB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/pipeline-utils",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.5",
|
|
4
4
|
"description": "Utilities for running pipelines",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"typings": "dist/index.d.ts",
|
|
@@ -38,18 +38,18 @@
|
|
|
38
38
|
"rxjs": "^6.6.7"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
|
-
"@hestia-earth/api": ">=0.
|
|
42
|
-
"@hestia-earth/json-schema": ">=
|
|
43
|
-
"@hestia-earth/schema": ">=
|
|
41
|
+
"@hestia-earth/api": ">=0.19.0",
|
|
42
|
+
"@hestia-earth/json-schema": ">=20.0.0",
|
|
43
|
+
"@hestia-earth/schema": ">=20.0.0",
|
|
44
44
|
"@hestia-earth/utils": ">=0.11.2"
|
|
45
45
|
},
|
|
46
46
|
"devDependencies": {
|
|
47
47
|
"@commitlint/cli": "^17.4.2",
|
|
48
48
|
"@commitlint/config-conventional": "^17.4.2",
|
|
49
|
-
"@hestia-earth/api": "^0.
|
|
49
|
+
"@hestia-earth/api": "^0.19.0",
|
|
50
50
|
"@hestia-earth/eslint-config": "^0.0.5",
|
|
51
|
-
"@hestia-earth/json-schema": "^
|
|
52
|
-
"@hestia-earth/schema": "^
|
|
51
|
+
"@hestia-earth/json-schema": "^20.0.0",
|
|
52
|
+
"@hestia-earth/schema": "^20.0.0",
|
|
53
53
|
"@hestia-earth/utils": "^0.11.2",
|
|
54
54
|
"@types/aws-lambda": "^8.10.109",
|
|
55
55
|
"@types/aws-sdk": "^2.7.0",
|
package/src/s3.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import * as AWS from 'aws-sdk';
|
|
2
2
|
import { createReadStream } from 'fs';
|
|
3
3
|
import { Subject, Observable, from, of } from 'rxjs';
|
|
4
|
-
import { catchError, map, take } from 'rxjs/operators';
|
|
4
|
+
import { catchError, map, take, tap } from 'rxjs/operators';
|
|
5
|
+
import { sizeInMb } from '@hestia-earth/api';
|
|
5
6
|
|
|
7
|
+
import { debug } from './log';
|
|
6
8
|
import * as local from './local';
|
|
7
9
|
import { stripBOM } from './utils';
|
|
8
10
|
|
|
9
|
-
const isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
|
|
11
|
+
export const isS3Mode = (process.env.SOURCE_MODE || 's3') === 's3';
|
|
10
12
|
export const bucket = process.env.BUCKET;
|
|
11
13
|
|
|
12
14
|
export const filePath = (key: string) => `https://${bucket}.s3.amazonaws.com/${key || ''}`;
|
|
@@ -159,3 +161,11 @@ export const listFolder = async (Prefix = '', Bucket = bucket, nextToken?: strin
|
|
|
159
161
|
}).promise();
|
|
160
162
|
return [...Contents, ...(IsTruncated ? await listFolder(Prefix, bucket, NextContinuationToken) : [])];
|
|
161
163
|
};
|
|
164
|
+
|
|
165
|
+
const largeFileSize = +(process.env.LARGE_SIZE_MB || '100');
|
|
166
|
+
|
|
167
|
+
export const isLargeFile = (key: string, Bucket = bucket) =>
|
|
168
|
+
fileSize(key, Bucket).pipe(
|
|
169
|
+
tap(size => debug('Detected file size:', sizeInMb(size), 'max size:', largeFileSize)),
|
|
170
|
+
map(size => sizeInMb(size) > largeFileSize)
|
|
171
|
+
);
|
package/src/sns.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import * as AWS from 'aws-sdk';
|
|
2
|
-
import { from } from 'rxjs';
|
|
2
|
+
import { from, of } from 'rxjs';
|
|
3
3
|
import { take, map } from 'rxjs/operators';
|
|
4
4
|
|
|
5
5
|
import { debug } from './log';
|
|
6
|
-
import { IFunctionParam } from './s3';
|
|
6
|
+
import { IFunctionParam, isS3Mode } from './s3';
|
|
7
7
|
|
|
8
|
-
export const publish = (functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC) =>
|
|
9
|
-
|
|
8
|
+
export const publish = (functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC) => isS3Mode
|
|
9
|
+
? of(null)
|
|
10
|
+
: from(
|
|
10
11
|
new AWS.SNS().publish({
|
|
11
12
|
TopicArn: topicArn,
|
|
12
13
|
Message: JSON.stringify(params),
|