@hestia-earth/pipeline-utils 0.13.22 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mongodb.d.ts +7 -1
- package/dist/progress.js +2 -2
- package/dist/progress.js.map +1 -1
- package/dist/sns.d.ts +1 -1
- package/dist/sns.js +3 -3
- package/dist/sns.js.map +1 -1
- package/dist/sqs.d.ts +1 -1
- package/dist/sqs.js +3 -3
- package/dist/sqs.js.map +1 -1
- package/package.json +5 -5
- package/src/progress.ts +3 -3
- package/src/sns.ts +1 -1
- package/src/sqs.ts +1 -1
package/dist/mongodb.d.ts
CHANGED
|
@@ -3,12 +3,12 @@ import { Aggregation, File, User, FileStatus, Reconciliation, Setting, SettingKe
|
|
|
3
3
|
export declare const transaction: <T>(query: (db: Db) => Promise<T>, defaultValue?: T) => Promise<T>;
|
|
4
4
|
export declare const idsQuery: (ids: string[]) => any;
|
|
5
5
|
export declare const getAggregations: <U extends keyof Aggregation>(conditions: {
|
|
6
|
+
folder?: any;
|
|
6
7
|
filename?: any;
|
|
7
8
|
filepath?: any;
|
|
8
9
|
validatedAt?: any;
|
|
9
10
|
status?: any;
|
|
10
11
|
pipelineStatus?: any;
|
|
11
|
-
readonly nodeType?: any;
|
|
12
12
|
readonly productId?: any;
|
|
13
13
|
readonly countryId?: any;
|
|
14
14
|
readonly startYear?: any;
|
|
@@ -58,6 +58,10 @@ export declare const getFiles: <U extends keyof File>(conditions: {
|
|
|
58
58
|
readonly dataRecalculatedCsvPath?: any;
|
|
59
59
|
readonly logPath?: any;
|
|
60
60
|
readonly errorPath?: any;
|
|
61
|
+
readonly sources?: any;
|
|
62
|
+
readonly nbCycles?: any;
|
|
63
|
+
readonly nbImpactAssessments?: any;
|
|
64
|
+
readonly hestiaDataPath?: any;
|
|
61
65
|
id?: any;
|
|
62
66
|
createdAt?: any;
|
|
63
67
|
updatedAt?: any;
|
|
@@ -78,6 +82,8 @@ export declare const getUsers: <U extends keyof User>(conditions: {
|
|
|
78
82
|
maxApiCalls?: any;
|
|
79
83
|
dataPrivate?: any;
|
|
80
84
|
validFilesCount?: any;
|
|
85
|
+
canCommitHestiaData?: any;
|
|
86
|
+
permissions?: any;
|
|
81
87
|
emailNotificationsSuccess?: any;
|
|
82
88
|
emailNotificationsFailure?: any;
|
|
83
89
|
emailNotificationsFeedback?: any;
|
package/dist/progress.js
CHANGED
|
@@ -39,9 +39,9 @@ const uploadProgress = (step, path, Bucket, since, err, details) => (0, rxjs_1.o
|
|
|
39
39
|
details,
|
|
40
40
|
error: err ? err.toString() : null
|
|
41
41
|
}, 'application/json', Bucket)))));
|
|
42
|
-
const handleError = (step, params, since, Bucket = s3_1.bucket) => (err) => uploadProgress(step, params.key, Bucket, since, err).pipe((0, operators_1.mergeMap)(() => (0, sns_1.
|
|
42
|
+
const handleError = (step, params, since, Bucket = s3_1.bucket) => (err) => uploadProgress(step, params.key, Bucket, since, err).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publishSnsMessage)(`${step}${publishFunctionSuffix.Error}`, params)));
|
|
43
43
|
exports.handleError = handleError;
|
|
44
|
-
const handleSuccess = (step, params, since, Bucket = s3_1.bucket) => (details) => uploadProgress(step, params.key, Bucket, since, null, details).pipe((0, operators_1.mergeMap)(() => (0, sns_1.
|
|
44
|
+
const handleSuccess = (step, params, since, Bucket = s3_1.bucket) => (details) => uploadProgress(step, params.key, Bucket, since, null, details).pipe((0, operators_1.mergeMap)(() => (0, sns_1.publishSnsMessage)(`${step}${publishFunctionSuffix.Success}`, params)));
|
|
45
45
|
exports.handleSuccess = handleSuccess;
|
|
46
46
|
const defaultErrorHandler = (key) => (err) => (0, rxjs_1.throwError)(new HestiaError(err.message, key, err.stack));
|
|
47
47
|
exports.defaultErrorHandler = defaultErrorHandler;
|
package/dist/progress.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,8CAAsD;AACtD,+BAAsC;AAGtC,6BAAwE;AACxE,+
|
|
1
|
+
{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":";;;AAAA,8CAAsD;AACtD,+BAAsC;AAGtC,6BAAwE;AACxE,+BAA0C;AAY1C,IAAY,qBAGX;AAHD,WAAY,qBAAqB;IAC/B,yCAAgB,CAAA;IAChB,wCAAe,CAAA;AACjB,CAAC,EAHW,qBAAqB,qCAArB,qBAAqB,QAGhC;AAEM,MAAM,SAAS,GAAG,CAAC,KAAW,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;AAApE,QAAA,SAAS,aAA2D;AAEjF,MAAa,WAAY,SAAQ,KAAK;IAOpC,YACS,OAAO,EACP,GAAW,EACX,SAAc,EAAE;QAEvB,KAAK,CAAC,OAAO,CAAC,CAAC;QAJR,YAAO,GAAP,OAAO,CAAA;QACP,QAAG,GAAH,GAAG,CAAQ;QACX,WAAM,GAAN,MAAM,CAAU;IAGzB,CAAC;IAED,QAAQ;QACN,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAtBD,kCAsBC;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;AAE1E,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,MAAc,EAAE,KAAY,EAAE,GAAiB,EAAE,OAAa,EAAE,EAAE,CACpH,IAAA,SAAE,EAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAC3B,IAAA,oBAAQ,EAAC,QAAQ,CAAC,EAAE,CAClB,IAAA,iBAAY,EAAyB,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CACzD,IAAA,sBAAU,EAAC,GAAG,EAAE,CAAC,IAAA,SAAE,EAAC,EAA4B,CAAC,CAAC,EAClD,IAAA,oBAAQ,EAAC,IAAI,CAAC,EAAE,CACd,IAAA,eAAU,EACR,QAAQ,EACR;IACE,IAAI;IACJ,OAAO,EAAE,CAAC,GAAG;IACb,IAAI,EAAE;QACJ,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACpB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAA,iBAAS,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C;IACD,OAAO;IACP,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;CACnC,EACD,kBAAkB,EAClB,MAAM,CACP,CACF,CACF,CACF,CACF,CAAC;AAEG,MAAM,WAAW,GACtB,CAAC,IAAkB,EAAE,MAAsB,EAAE,KAAY,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CAC9E,CAAC,GAAgB,EAAE,EAAE,CACnB,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CACvD,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,uBAAiB,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CACnF,CAAC;AALO,QAAA,WAAW,eAKlB;AAEC,MAAM,aAAa,GACxB,CAAC,IAAkB,EAAE,MAAsB,EAAE,KAAY,EAAE,MAAM,GAAG,WAAM,EAAE,EAAE,CAC9E,CAAC,OAAa,EAAE,EAAE,CAChB,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CACjE,IAAA,oBAAQ,EAAC,GAAG,EAAE,CAAC,IAAA,uBAAiB,EAAC,GAAG,IAAI,GAAG,qBAAqB,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC,CACrF,CAAC;AALO,QAAA,aAAa,iBAKpB;AAEC,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAU,EAAE,EAAE,CACjE,IAAA,iBAAU,EAAC,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAD9C,QAAA,mBAAmB,uBAC2B"}
|
package/dist/sns.d.ts
CHANGED
|
@@ -2,5 +2,5 @@ import { IFunctionParam } from './s3';
|
|
|
2
2
|
type attributes = {
|
|
3
3
|
[key: string]: string;
|
|
4
4
|
};
|
|
5
|
-
export declare const
|
|
5
|
+
export declare const publishSnsMessage: (functionName: string, params: IFunctionParam | Record<string, any>, topicArn?: string, extras?: attributes) => import("rxjs").Observable<any>;
|
|
6
6
|
export {};
|
package/dist/sns.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.publishSnsMessage = void 0;
|
|
4
4
|
const AWS = require("aws-sdk");
|
|
5
5
|
const rxjs_1 = require("rxjs");
|
|
6
6
|
const operators_1 = require("rxjs/operators");
|
|
7
7
|
const log_1 = require("./log");
|
|
8
8
|
const s3_1 = require("./s3");
|
|
9
9
|
const utils_1 = require("./utils");
|
|
10
|
-
const
|
|
10
|
+
const publishSnsMessage = (functionName, params, topicArn = process.env.SNS_TOPIC, extras = {}) => s3_1.isS3Mode
|
|
11
11
|
? (0, rxjs_1.from)(new AWS.SNS()
|
|
12
12
|
.publish({
|
|
13
13
|
TopicArn: topicArn,
|
|
@@ -34,5 +34,5 @@ const publish = (functionName, params, topicArn = process.env.SNS_TOPIC, extras
|
|
|
34
34
|
return (0, rxjs_1.throwError)(err);
|
|
35
35
|
}))
|
|
36
36
|
: (0, rxjs_1.of)(null);
|
|
37
|
-
exports.
|
|
37
|
+
exports.publishSnsMessage = publishSnsMessage;
|
|
38
38
|
//# sourceMappingURL=sns.js.map
|
package/dist/sns.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAA4C;AAC5C,8CAAuD;AAEvD,+BAAoC;AACpC,6BAAgD;AAChD,mCAAmC;AAM5B,MAAM,
|
|
1
|
+
{"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAA4C;AAC5C,8CAAuD;AAEvD,+BAAoC;AACpC,6BAAgD;AAChD,mCAAmC;AAM5B,MAAM,iBAAiB,GAAG,CAC/B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAChC,SAAqB,EAAE,EACvB,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,GAAG,CAAC,GAAG,EAAE;SACV,OAAO,CAAC;QACP,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;KACF,CAAC;SACD,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9F,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,EACF,IAAA,sBAAU,EAAC,GAAG,CAAC,EAAE;QACf,IAAA,WAAK,EAAC,eAAe,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAA,gBAAQ,EAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/F,OAAO,IAAA,iBAAU,EAAC,GAAG,CAAC,CAAC;IACzB,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAxCF,QAAA,iBAAiB,qBAwCf"}
|
package/dist/sqs.d.ts
CHANGED
|
@@ -2,5 +2,5 @@ import { IFunctionParam } from './s3';
|
|
|
2
2
|
type attributes = {
|
|
3
3
|
[key: string]: string;
|
|
4
4
|
};
|
|
5
|
-
export declare const
|
|
5
|
+
export declare const sendSqsMessage: (functionName: string, params: IFunctionParam | Record<string, any>, queueUrl?: string, extras?: attributes) => import("rxjs").Observable<any>;
|
|
6
6
|
export {};
|
package/dist/sqs.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.sendSqsMessage = void 0;
|
|
4
4
|
const AWS = require("aws-sdk");
|
|
5
5
|
const rxjs_1 = require("rxjs");
|
|
6
6
|
const operators_1 = require("rxjs/operators");
|
|
7
7
|
const log_1 = require("./log");
|
|
8
8
|
const s3_1 = require("./s3");
|
|
9
|
-
const
|
|
9
|
+
const sendSqsMessage = (functionName, params, queueUrl = process.env.SQS_URL, extras = {}) => s3_1.isS3Mode
|
|
10
10
|
? (0, rxjs_1.from)(new AWS.SQS({ region: s3_1.region })
|
|
11
11
|
.sendMessage({
|
|
12
12
|
QueueUrl: queueUrl,
|
|
@@ -30,5 +30,5 @@ const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL, extra
|
|
|
30
30
|
return res;
|
|
31
31
|
}))
|
|
32
32
|
: (0, rxjs_1.of)(null);
|
|
33
|
-
exports.
|
|
33
|
+
exports.sendSqsMessage = sendSqsMessage;
|
|
34
34
|
//# sourceMappingURL=sqs.js.map
|
package/dist/sqs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA6B;AAC7B,6BAAwD;AAMjD,MAAM,
|
|
1
|
+
{"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA6B;AAC7B,6BAAwD;AAMjD,MAAM,cAAc,GAAG,CAC5B,YAAoB,EACpB,MAA4C,EAC5C,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAC9B,SAAqB,EAAE,EACvB,EAAE,CACF,aAAQ;IACN,CAAC,CAAC,IAAA,WAAI,EACF,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC;SACpB,WAAW,CAAC;QACX,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACjD,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CACH;SACF;KACF,CAAC;SACD,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,UAAI,EAAC,oBAAoB,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACH,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AApCF,QAAA,cAAc,kBAoCZ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/pipeline-utils",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Utilities for running pipelines",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"typings": "dist/index.d.ts",
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
"devDependencies": {
|
|
49
49
|
"@commitlint/cli": "^17.8.1",
|
|
50
50
|
"@commitlint/config-conventional": "^17.8.1",
|
|
51
|
-
"@hestia-earth/api": "^0.
|
|
51
|
+
"@hestia-earth/api": "^0.24.50",
|
|
52
52
|
"@hestia-earth/eslint-config": "^0.1.0",
|
|
53
|
-
"@hestia-earth/json-schema": "^
|
|
54
|
-
"@hestia-earth/schema": "^
|
|
55
|
-
"@hestia-earth/utils": "^0.
|
|
53
|
+
"@hestia-earth/json-schema": "^30.3.2",
|
|
54
|
+
"@hestia-earth/schema": "^30.3.2",
|
|
55
|
+
"@hestia-earth/utils": "^0.13.9",
|
|
56
56
|
"@types/aws-lambda": "^8.10.130",
|
|
57
57
|
"@types/chai": "^4.3.11",
|
|
58
58
|
"@types/lodash.get": "^4.4.9",
|
package/src/progress.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { of, throwError } from 'rxjs';
|
|
|
3
3
|
import { FileProgress } from '@hestia-earth/api';
|
|
4
4
|
|
|
5
5
|
import { IFunctionParam, uploadJSON, bucket, loadJSONFile } from './s3';
|
|
6
|
-
import {
|
|
6
|
+
import { publishSnsMessage } from './sns';
|
|
7
7
|
|
|
8
8
|
interface IFileProgress {
|
|
9
9
|
step: FileProgress;
|
|
@@ -78,14 +78,14 @@ export const handleError =
|
|
|
78
78
|
(step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
|
|
79
79
|
(err: HestiaError) =>
|
|
80
80
|
uploadProgress(step, params.key, Bucket, since, err).pipe(
|
|
81
|
-
mergeMap(() =>
|
|
81
|
+
mergeMap(() => publishSnsMessage(`${step}${publishFunctionSuffix.Error}`, params))
|
|
82
82
|
);
|
|
83
83
|
|
|
84
84
|
export const handleSuccess =
|
|
85
85
|
(step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
|
|
86
86
|
(details?: any) =>
|
|
87
87
|
uploadProgress(step, params.key, Bucket, since, null, details).pipe(
|
|
88
|
-
mergeMap(() =>
|
|
88
|
+
mergeMap(() => publishSnsMessage(`${step}${publishFunctionSuffix.Success}`, params))
|
|
89
89
|
);
|
|
90
90
|
|
|
91
91
|
export const defaultErrorHandler = (key: string) => (err: Error) =>
|
package/src/sns.ts
CHANGED