@hestia-earth/pipeline-utils 0.10.1 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/gitlab.d.ts +9 -0
  3. package/dist/gitlab.js +29 -0
  4. package/dist/gitlab.js.map +1 -0
  5. package/dist/sns.d.ts +5 -1
  6. package/dist/sns.js +9 -2
  7. package/dist/sns.js.map +1 -1
  8. package/dist/sqs.d.ts +5 -1
  9. package/dist/sqs.js +9 -2
  10. package/dist/sqs.js.map +1 -1
  11. package/package.json +4 -2
  12. package/src/gitlab.ts +34 -0
  13. package/src/sns.ts +15 -2
  14. package/src/sqs.ts +15 -2
package/CHANGELOG.md CHANGED
@@ -2,6 +2,22 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
4
4
 
5
+ ### [0.10.4](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.10.3...v0.10.4) (2023-06-15)
6
+
7
+ ### [0.10.3](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.10.2...v0.10.3) (2023-06-15)
8
+
9
+
10
+ ### Features
11
+
12
+ * **gitlab:** add `listFiles` and `readFile` functions ([76b4daa](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/76b4daa2eeb23d15b5a09637756fc0c7c3200aed))
13
+
14
+ ### [0.10.2](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.10.1...v0.10.2) (2023-06-13)
15
+
16
+
17
+ ### Features
18
+
19
+ * add extra attributes to SNS and SQS ([e981692](https://gitlab.com/hestia-earth/hestia-pipeline-utils/commit/e98169230a9c41ac8a2ad85104275f3c8913dc0c))
20
+
5
21
  ### [0.10.1](https://gitlab.com/hestia-earth/hestia-pipeline-utils/compare/v0.10.0...v0.10.1) (2023-06-08)
6
22
 
7
23
 
package/dist/gitlab.d.ts ADDED
@@ -0,0 +1,9 @@
1
+ /// <reference types="node" />
2
+ import { Readable } from 'stream';
3
+ export interface IGitlabFile {
4
+ id?: string;
5
+ name: string;
6
+ path: string;
7
+ }
8
+ export declare const listFiles: (path?: string, page?: number, limit?: number) => Promise<IGitlabFile[]>;
9
+ export declare const readFile: ({ id }: Partial<IGitlabFile>) => Promise<Readable>;
package/dist/gitlab.js ADDED
@@ -0,0 +1,29 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.readFile = exports.listFiles = void 0;
4
+ const axios_1 = require("axios");
5
+ const token = process.env.GITLAB_TOKEN;
6
+ const projectId = process.env.GITLAB_PROJECT_ID;
7
+ const branch = process.env.GITLAB_BRANCH;
8
+ const gitlabUrl = `https://gitlab.com/api/v4/projects/${projectId}`;
9
+ const headers = { 'PRIVATE-TOKEN': token };
10
+ const listFiles = async (path, page = 1, limit = 100) => {
11
+ const { data } = await axios_1.default.get(`${gitlabUrl}/repository/tree`, {
12
+ headers: { 'PRIVATE-TOKEN': token },
13
+ params: { page, per_page: limit, ref: branch, path }
14
+ });
15
+ return [
16
+ ...data,
17
+ ...(data.length === limit ? await (0, exports.listFiles)(path, page + 1) : [])
18
+ ];
19
+ };
20
+ exports.listFiles = listFiles;
21
+ const readFile = async ({ id }) => {
22
+ const { data } = await axios_1.default.get(`${gitlabUrl}/repository/blobs/${id}/raw`, {
23
+ headers,
24
+ responseType: 'stream'
25
+ });
26
+ return data;
27
+ };
28
+ exports.readFile = readFile;
29
+ //# sourceMappingURL=gitlab.js.map
package/dist/gitlab.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gitlab.js","sourceRoot":"","sources":["../src/gitlab.ts"],"names":[],"mappings":";;;AACA,iCAA0B;AAQ1B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AACvC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;AAChD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;AAEzC,MAAM,SAAS,GAAG,sCAAsC,SAAS,EAAE,CAAC;AACpE,MAAM,OAAO,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;AAEpC,MAAM,SAAS,GAAG,KAAK,EAAE,IAAa,EAAE,IAAI,GAAG,CAAC,EAAE,KAAK,GAAG,GAAG,EAA0B,EAAE;IAC9F,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAgB,GAAG,SAAS,kBAAkB,EAAE;QAC9E,OAAO,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE;QACnC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;KACrD,CAAC,CAAC;IACH,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,IAAA,iBAAS,EAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAClE,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,SAAS,aASpB;AAEK,MAAM,QAAQ,GAAG,KAAK,EAAE,EAAE,EAAE,EAAwB,EAAE,EAAE;IAC7D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAK,CAAC,GAAG,CAAW,GAAG,SAAS,qBAAqB,EAAE,MAAM,EAAE;QACpF,OAAO;QACP,YAAY,EAAE,QAAQ;KACvB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AANW,QAAA,QAAQ,YAMnB"}
package/dist/sns.d.ts CHANGED
@@ -1,2 +1,6 @@
1
1
  import { IFunctionParam } from './s3';
2
- export declare const publish: (functionName: string, params: IFunctionParam, topicArn?: string) => import("rxjs").Observable<any>;
2
+ declare type attributes = {
3
+ [key: string]: string;
4
+ };
5
+ export declare const publish: (functionName: string, params: IFunctionParam, topicArn?: string, extras?: attributes) => import("rxjs").Observable<any>;
6
+ export {};
package/dist/sns.js CHANGED
@@ -6,7 +6,7 @@ const rxjs_1 = require("rxjs");
6
6
  const operators_1 = require("rxjs/operators");
7
7
  const log_1 = require("./log");
8
8
  const s3_1 = require("./s3");
9
- const publish = (functionName, params, topicArn = process.env.SNS_TOPIC) => s3_1.isS3Mode
9
+ const publish = (functionName, params, topicArn = process.env.SNS_TOPIC, extras = {}) => s3_1.isS3Mode
10
10
  ? (0, rxjs_1.from)(new AWS.SNS().publish({
11
11
  TopicArn: topicArn,
12
12
  Message: JSON.stringify(params),
@@ -14,7 +14,14 @@ const publish = (functionName, params, topicArn = process.env.SNS_TOPIC) => s3_1
14
14
  functionName: {
15
15
  DataType: 'String',
16
16
  StringValue: functionName
17
- }
17
+ },
18
+ ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
19
+ key,
20
+ {
21
+ DataType: 'String',
22
+ StringValue: value
23
+ }
24
+ ])))
18
25
  }
19
26
  }).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
20
27
  (0, log_1.debug)(`SNS publish ${functionName}`, res);
package/dist/sns.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAgD;AAEzC,MAAM,OAAO,GAAG,CAAC,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,aAAQ;IACjH,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;SACF;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,eAAe,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACD,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAnBA,QAAA,OAAO,WAmBP"}
1
+ {"version":3,"file":"sns.js","sourceRoot":"","sources":["../src/sns.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAgD;AAMzC,MAAM,OAAO,GAAG,CACrB,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,SAAqB,EAAE,EACvG,EAAE,CAAC,aAAQ;IACX,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/B,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACxE,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CAAC,CAAC;SACL;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,eAAe,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACD,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AA5BA,QAAA,OAAO,WA4BP"}
package/dist/sqs.d.ts CHANGED
@@ -1,2 +1,6 @@
1
1
  import { IFunctionParam } from './s3';
2
- export declare const sendMessage: (functionName: string, params: IFunctionParam, queueUrl?: string) => import("rxjs").Observable<any>;
2
+ declare type attributes = {
3
+ [key: string]: string;
4
+ };
5
+ export declare const sendMessage: (functionName: string, params: IFunctionParam, queueUrl?: string, extras?: attributes) => import("rxjs").Observable<any>;
6
+ export {};
package/dist/sqs.js CHANGED
@@ -6,7 +6,7 @@ const rxjs_1 = require("rxjs");
6
6
  const operators_1 = require("rxjs/operators");
7
7
  const log_1 = require("./log");
8
8
  const s3_1 = require("./s3");
9
- const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL) => s3_1.isS3Mode
9
+ const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL, extras = {}) => s3_1.isS3Mode
10
10
  ? (0, rxjs_1.from)(new AWS.SQS({ region: s3_1.region }).sendMessage({
11
11
  QueueUrl: queueUrl,
12
12
  MessageBody: JSON.stringify(params),
@@ -14,7 +14,14 @@ const sendMessage = (functionName, params, queueUrl = process.env.SQS_URL) => s3
14
14
  functionName: {
15
15
  DataType: 'String',
16
16
  StringValue: functionName
17
- }
17
+ },
18
+ ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
19
+ key,
20
+ {
21
+ DataType: 'String',
22
+ StringValue: value
23
+ }
24
+ ])))
18
25
  }
19
26
  }).promise()).pipe((0, operators_1.take)(1), (0, operators_1.map)(res => {
20
27
  (0, log_1.debug)(`SQS send message ${functionName}`, res);
package/dist/sqs.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAwD;AAEjD,MAAM,WAAW,GAAG,CAAC,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,aAAQ;IACnH,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC,CAAC,WAAW,CAAC;QAClC,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;SACF;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,oBAAoB,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACD,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AAnBA,QAAA,WAAW,eAmBX"}
1
+ {"version":3,"file":"sqs.js","sourceRoot":"","sources":["../src/sqs.ts"],"names":[],"mappings":";;;AAAA,+BAA+B;AAC/B,+BAAgC;AAChC,8CAA2C;AAE3C,+BAA8B;AAC9B,6BAAwD;AAMjD,MAAM,WAAW,GAAG,CACzB,YAAoB,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,SAAqB,EAAE,EACrG,EAAE,CAAC,aAAQ;IACX,CAAC,CAAC,IAAA,WAAI,EACJ,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAN,WAAM,EAAE,CAAC,CAAC,WAAW,CAAC;QAClC,QAAQ,EAAE,QAAQ;QAClB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,iBAAiB,EAAE;YACjB,YAAY,EAAE;gBACZ,QAAQ,EAAE,QAAQ;gBAClB,WAAW,EAAE,YAAY;aAC1B;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;gBACxE,GAAG;gBACH;oBACE,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC,CAAC,CAAC;SACL;KACF,CAAC,CAAC,OAAO,EAAE,CACb,CAAC,IAAI,CACJ,IAAA,gBAAI,EAAC,CAAC,CAAC,EACP,IAAA,eAAG,EAAC,GAAG,CAAC,EAAE;QACR,IAAA,WAAK,EAAC,oBAAoB,YAAY,EAAE,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CACH;IACD,CAAC,CAAC,IAAA,SAAE,EAAC,IAAI,CAAC,CAAC;AA5BA,QAAA,WAAW,eA4BX"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hestia-earth/pipeline-utils",
3
- "version": "0.10.1",
3
+ "version": "0.10.4",
4
4
  "description": "Utilities for running pipelines",
5
5
  "main": "dist/index.js",
6
6
  "typings": "dist/index.d.ts",
@@ -41,7 +41,8 @@
41
41
  "@hestia-earth/api": ">=0.19.0",
42
42
  "@hestia-earth/json-schema": ">=20.0.0",
43
43
  "@hestia-earth/schema": ">=20.0.0",
44
- "@hestia-earth/utils": ">=0.11.2"
44
+ "@hestia-earth/utils": ">=0.11.2",
45
+ "axios": "^1.0.0"
45
46
  },
46
47
  "devDependencies": {
47
48
  "@commitlint/cli": "^17.4.2",
@@ -63,6 +64,7 @@
63
64
  "@types/sinon": "^9.0.11",
64
65
  "@typescript-eslint/eslint-plugin": "^5.27.1",
65
66
  "@typescript-eslint/parser": "^5.27.1",
67
+ "axios": "^1.4.0",
66
68
  "chai": "^4.3.7",
67
69
  "eslint": "^7.32.0",
68
70
  "eslint-plugin-jsdoc": "^30.7.13",
package/src/gitlab.ts ADDED
@@ -0,0 +1,34 @@
1
+ import { Readable } from 'stream';
2
+ import axios from 'axios';
3
+
4
+ export interface IGitlabFile {
5
+ id?: string;
6
+ name: string;
7
+ path: string;
8
+ }
9
+
10
+ const token = process.env.GITLAB_TOKEN;
11
+ const projectId = process.env.GITLAB_PROJECT_ID;
12
+ const branch = process.env.GITLAB_BRANCH;
13
+
14
+ const gitlabUrl = `https://gitlab.com/api/v4/projects/${projectId}`;
15
+ const headers = { 'PRIVATE-TOKEN': token };
16
+
17
+ export const listFiles = async (path?: string, page = 1, limit = 100): Promise<IGitlabFile[]> => {
18
+ const { data } = await axios.get<IGitlabFile[]>(`${gitlabUrl}/repository/tree`, {
19
+ headers: { 'PRIVATE-TOKEN': token },
20
+ params: { page, per_page: limit, ref: branch, path }
21
+ });
22
+ return [
23
+ ...data,
24
+ ...(data.length === limit ? await listFiles(path, page + 1) : [])
25
+ ];
26
+ };
27
+
28
+ export const readFile = async ({ id }: Partial<IGitlabFile>) => {
29
+ const { data } = await axios.get<Readable>(`${gitlabUrl}/repository/blobs/${id}/raw`, {
30
+ headers,
31
+ responseType: 'stream'
32
+ });
33
+ return data;
34
+ };
package/src/sns.ts CHANGED
@@ -5,7 +5,13 @@ import { take, map } from 'rxjs/operators';
5
5
  import { debug } from './log';
6
6
  import { IFunctionParam, isS3Mode } from './s3';
7
7
 
8
- export const publish = (functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC) => isS3Mode
8
+ type attributes = {
9
+ [key: string]: string;
10
+ };
11
+
12
+ export const publish = (
13
+ functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC, extras: attributes = {}
14
+ ) => isS3Mode
9
15
  ? from(
10
16
  new AWS.SNS().publish({
11
17
  TopicArn: topicArn,
@@ -14,7 +20,14 @@ export const publish = (functionName: string, params: IFunctionParam, topicArn =
14
20
  functionName: {
15
21
  DataType: 'String',
16
22
  StringValue: functionName
17
- }
23
+ },
24
+ ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
25
+ key,
26
+ {
27
+ DataType: 'String',
28
+ StringValue: value
29
+ }
30
+ ])))
18
31
  }
19
32
  }).promise()
20
33
  ).pipe(
package/src/sqs.ts CHANGED
@@ -5,7 +5,13 @@ import { take, map } from 'rxjs/operators';
5
5
  import { debug } from './log';
6
6
  import { IFunctionParam, isS3Mode, region } from './s3';
7
7
 
8
- export const sendMessage = (functionName: string, params: IFunctionParam, queueUrl = process.env.SQS_URL) => isS3Mode
8
+ type attributes = {
9
+ [key: string]: string;
10
+ };
11
+
12
+ export const sendMessage = (
13
+ functionName: string, params: IFunctionParam, queueUrl = process.env.SQS_URL, extras: attributes = {}
14
+ ) => isS3Mode
9
15
  ? from(
10
16
  new AWS.SQS({ region }).sendMessage({
11
17
  QueueUrl: queueUrl,
@@ -14,7 +20,14 @@ export const sendMessage = (functionName: string, params: IFunctionParam, queueU
14
20
  functionName: {
15
21
  DataType: 'String',
16
22
  StringValue: functionName
17
- }
23
+ },
24
+ ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
25
+ key,
26
+ {
27
+ DataType: 'String',
28
+ StringValue: value
29
+ }
30
+ ])))
18
31
  }
19
32
  }).promise()
20
33
  ).pipe(