@hestia-earth/pipeline-utils 0.12.1 → 0.13.1

package/src/local.ts

@@ -17,25 +17,22 @@ const currentDir = (key: string) => {
 };
 
 export const loadTextFile = (folder: string) => (file: string) =>
-  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true })
-    .pipe(
-      mergeMap(() => readFileAsObservable(join(folder, file), 'utf8')),
-      take(1),
-      map((res: any) => res as string)
-    );
+  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true }).pipe(
+    mergeMap(() => readFileAsObservable(join(folder, file), 'utf8')),
+    take(1),
+    map((res: any) => res as string)
+  );
 
-export const loadJSONFile = (folder: string) => <T>(file: string) =>
-  loadTextFile(folder)(file)
-    .pipe(
-      map(res => JSON.parse(stripBOM(res)) as T)
-    );
+export const loadJSONFile =
+  (folder: string) =>
+  <T>(file: string) =>
+    loadTextFile(folder)(file).pipe(map(res => JSON.parse(stripBOM(res)) as T));
 
 export const uploadText = (folder: string) => (file: string, data: string) =>
-  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true })
-    .pipe(
-      mergeMap(() => writeFileAsObservable(join(folder, file), data, 'utf8')),
-      map(() => data)
-    );
+  mkdirAsObservable(currentDir(join(folder, file)), { recursive: true }).pipe(
+    mergeMap(() => writeFileAsObservable(join(folder, file), data, 'utf8')),
+    map(() => data)
+  );
 
 export const uploadJSON = (folder: string) => (file: string, data) =>
   uploadText(folder)(file, JSON.stringify(data, null, 2));
@@ -43,8 +40,7 @@ export const uploadJSON = (folder: string) => (file: string, data) =>
 export const lastModified = (folder: string) => (file: string) => {
   try {
     return of(statSync(join(folder, file)).mtime);
-  }
-  catch (err) {
+  } catch (err) {
     return of(new Date());
   }
 };
@@ -53,8 +49,7 @@ export const fileExists = (folder: string) => (file: string) => {
   try {
     const success = statSync(join(folder, file)).isFile();
     return of(success);
-  }
-  catch (err) {
+  } catch (err) {
     return of(false);
   }
 };
@@ -62,11 +57,12 @@ export const fileExists = (folder: string) => (file: string) => {
 export const fileSize = (folder: string) => (file: string) => {
   try {
     return of(statSync(join(folder, file)).size);
-  }
-  catch (err) {
+  } catch (err) {
     return of(undefined as number);
   }
 };
 
-export const deleteFile = (folder: string) => (file: string): Observable<any> =>
-  unlinkAsObservable(join(folder, file));
+export const deleteFile =
+  (folder: string) =>
+  (file: string): Observable<any> =>
+    unlinkAsObservable(join(folder, file));

package/src/progress.ts

@@ -29,7 +29,11 @@ export class HestiaError extends Error {
    * @param key The file throwing this error
    * @param errors The errors
    */
-  constructor(public message, public key: string, public errors: any = '') {
+  constructor(
+    public message,
+    public key: string,
+    public errors: any = ''
+  ) {
     super(message);
   }
 
@@ -46,30 +50,43 @@ const fileToProgress = (path: string) => `${path.split('.')[0]}.progress`;
 
 const uploadProgress = (step: string, path: string, Bucket: string, since?: Date, err?: HestiaError, details?: any) =>
   of(fileToProgress(path)).pipe(
-    mergeMap(filepath => loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
-      catchError(() => of({} as Partial<IFileProgress>)),
-      mergeMap(data => uploadJSON(filepath, {
-        step,
-        success: !err,
-        time: {
-          ...(data.time || {}),
-          ...(since ? { [step]: timeSpent(since) } : {})
-        },
-        details,
-        error: err ? err.toString() : null
-      }, 'application/json', Bucket))
-    ))
+    mergeMap(filepath =>
+      loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
+        catchError(() => of({} as Partial<IFileProgress>)),
+        mergeMap(data =>
+          uploadJSON(
+            filepath,
+            {
+              step,
+              success: !err,
+              time: {
+                ...(data.time || {}),
+                ...(since ? { [step]: timeSpent(since) } : {})
+              },
+              details,
+              error: err ? err.toString() : null
+            },
+            'application/json',
+            Bucket
+          )
+        )
+      )
+    )
   );
 
-export const handleError = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
-  (err: HestiaError) => uploadProgress(step, params.key, Bucket, since, err).pipe(
-    mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
-  );
+export const handleError =
+  (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (err: HestiaError) =>
+    uploadProgress(step, params.key, Bucket, since, err).pipe(
+      mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
+    );
 
-export const handleSuccess = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
-  (details?: any) => uploadProgress(step, params.key, Bucket, since, null, details).pipe(
-    mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
-  );
+export const handleSuccess =
+  (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (details?: any) =>
+    uploadProgress(step, params.key, Bucket, since, null, details).pipe(
+      mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
+    );
 
 export const defaultErrorHandler = (key: string) => (err: Error) =>
   throwError(new HestiaError(err.message, key, err.stack));

package/src/s3.ts

@@ -32,43 +32,62 @@ export const getS3 = () => {
 };
 
 export const getObject = ({ Key, ...params }: AWS.S3.GetObjectRequest) =>
-  from(getS3().getObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
+  from(
+    getS3()
+      .getObject({ ...params, Key: decodeURI(Key) })
+      .promise()
+  ).pipe(
     take(1),
     map(({ Body }) => Body.toString())
   );
 
 export const putObject = ({ Key, ...params }: AWS.S3.PutObjectRequest) =>
-  from(getS3().putObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
+  from(
+    getS3()
+      .putObject({ ...params, Key: decodeURI(Key) })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
 export const deleteObjects = (Objects: AWS.S3.Object[], Bucket = bucket) =>
-  from(getS3().deleteObjects({
-    Bucket,
-    Delete: {
-      Objects: Objects.map(({ Key }) => ({ Key }))
-    }
-  }).promise()).pipe(
+  from(
+    getS3()
+      .deleteObjects({
+        Bucket,
+        Delete: {
+          Objects: Objects.map(({ Key }) => ({ Key }))
+        }
+      })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
 export const copyObject = (fromParams, toParams, MetadataDirective?: string) =>
-  from(getS3().copyObject({
-    Bucket: toParams.bucket,
-    Key: decodeURI(toParams.key),
-    CopySource: `${fromParams.bucket}/${fromParams.key}`,
-    MetadataDirective
-  }).promise()).pipe(
+  from(
+    getS3()
+      .copyObject({
+        Bucket: toParams.bucket,
+        Key: decodeURI(toParams.key),
+        CopySource: `${fromParams.bucket}/${fromParams.key}`,
+        MetadataDirective
+      })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
-export const loadStream = (key: string, Bucket = bucket) => getS3().getObject({
-  Bucket,
-  Key: decodeURI(key)
-}).createReadStream();
+export const loadStream = (key: string, Bucket = bucket) =>
+  getS3()
+    .getObject({
+      Bucket,
+      Key: decodeURI(key)
+    })
+    .createReadStream();
 
 export const loadFile = (key: string) => {
   const stream = loadStream(key);
@@ -86,80 +105,122 @@ export const loadFile = (key: string) => {
   return subject.asObservable();
 };
 
-export const loadTextFile = (key: string, Bucket = bucket) => isS3Mode ? getObject({
-  Bucket,
-  Key: key
-}).pipe(map(res => res.replace(/\"/g, ''))) : local.loadTextFile(Bucket)(key);
-
-export const loadJSONFile = <T>(key: string, Bucket = bucket): Observable<T> => isS3Mode ? getObject({
-  Bucket,
-  Key: key,
-  ResponseContentType: 'application/json'
-}).pipe(map(res => JSON.parse(stripBOM(res)) as T)) : local.loadJSONFile(Bucket)(key);
-
-export const uploadFile = (filepath: string, key: string, Bucket = bucket) => putObject({
-  Bucket,
-  Key: key,
-  Body: createReadStream(filepath, 'utf8')
-});
-
-export const uploadText = (key: string, data: string, Bucket = bucket) => isS3Mode ? putObject({
-  Bucket,
-  Key: key,
-  Body: data
-}) : local.uploadText(Bucket)(key, data);
-
-export const uploadJSON = (key: string, data, type = 'application/json', Bucket = bucket) => isS3Mode ? putObject({
-  Bucket,
-  Key: key,
-  Body: JSON.stringify(data, null, 2),
-  ContentType: type
-}) : local.uploadJSON(Bucket)(key, data);
-
-export const waitObjectCreated = (key: string, Bucket = bucket) => isS3Mode ?
-  from(getS3().waitFor('objectExists', {
+export const loadTextFile = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? getObject({
+        Bucket,
+        Key: key
+      }).pipe(map(res => res.replace(/\"/g, '')))
+    : local.loadTextFile(Bucket)(key);
+
+export const loadJSONFile = <T>(key: string, Bucket = bucket): Observable<T> =>
+  isS3Mode
+    ? getObject({
+        Bucket,
+        Key: key,
+        ResponseContentType: 'application/json'
+      }).pipe(map(res => JSON.parse(stripBOM(res)) as T))
+    : local.loadJSONFile(Bucket)(key);
+
+export const uploadFile = (filepath: string, key: string, Bucket = bucket) =>
+  putObject({
     Bucket,
-    Key: decodeURI(key)
-  }).promise()).pipe(
-    take(1),
-    map(() => true)
-  ) : of(true);
-
-const headObject = (key: string, Bucket = bucket) => from(getS3().headObject({
-  Bucket,
-  Key: decodeURI(key)
-}).promise());
-
-export const lastModified = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(({ LastModified }) => LastModified),
-  catchError(() => of(null as Date)),
-  take(1)
-) : local.lastModified(Bucket)(key);
-
-export const fileExists = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(() => true),
-  catchError(() => of(false)),
-  take(1)
-) : local.fileExists(Bucket)(key);
-
-export const fileSize = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(({ ContentLength }) => ContentLength),
-  catchError(() => of(undefined as number)),
-  take(1)
-) : local.fileSize(Bucket)(key);
-
-export const deleteObject = (key: string, Bucket = bucket) => isS3Mode ? from(getS3().deleteObject({
-  Bucket,
-  Key: decodeURI(key)
-}).promise()) : local.deleteFile(Bucket)(key);
+    Key: key,
+    Body: createReadStream(filepath, 'utf8')
+  });
+
+export const uploadText = (key: string, data: string, Bucket = bucket) =>
+  isS3Mode
+    ? putObject({
+        Bucket,
+        Key: key,
+        Body: data
+      })
+    : local.uploadText(Bucket)(key, data);
+
+export const uploadJSON = (key: string, data, type = 'application/json', Bucket = bucket) =>
+  isS3Mode
+    ? putObject({
+        Bucket,
+        Key: key,
+        Body: JSON.stringify(data, null, 2),
+        ContentType: type
+      })
+    : local.uploadJSON(Bucket)(key, data);
+
+export const waitObjectCreated = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? from(
+        getS3()
+          .waitFor('objectExists', {
+            Bucket,
+            Key: decodeURI(key)
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(() => true)
+      )
+    : of(true);
+
+const headObject = (key: string, Bucket = bucket) =>
+  from(
+    getS3()
+      .headObject({
+        Bucket,
+        Key: decodeURI(key)
+      })
+      .promise()
+  );
+
+export const lastModified = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(({ LastModified }) => LastModified),
+        catchError(() => of(null as Date)),
+        take(1)
+      )
+    : local.lastModified(Bucket)(key);
+
+export const fileExists = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(() => true),
+        catchError(() => of(false)),
+        take(1)
+      )
+    : local.fileExists(Bucket)(key);
+
+export const fileSize = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(({ ContentLength }) => ContentLength),
+        catchError(() => of(undefined as number)),
+        take(1)
+      )
+    : local.fileSize(Bucket)(key);
+
+export const deleteObject = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? from(
+        getS3()
+          .deleteObject({
+            Bucket,
+            Key: decodeURI(key)
+          })
+          .promise()
+      )
+    : local.deleteFile(Bucket)(key);
 
 export const listFolder = async (Prefix = '', Bucket = bucket, nextToken?: string): Promise<AWS.S3.Object[]> => {
-  const { Contents, IsTruncated, NextContinuationToken } = await getS3().listObjectsV2({
-    Bucket,
-    Prefix,
-    ContinuationToken: nextToken,
-    MaxKeys: 1000
-  }).promise();
+  const { Contents, IsTruncated, NextContinuationToken } = await getS3()
+    .listObjectsV2({
+      Bucket,
+      Prefix,
+      ContinuationToken: nextToken,
+      MaxKeys: 1000
+    })
+    .promise();
   return [...Contents, ...(IsTruncated ? await listFolder(Prefix, bucket, NextContinuationToken) : [])];
 };
 

package/src/sns.ts

@@ -10,31 +10,39 @@ type attributes = {
 };
 
 export const publish = (
-  functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC, extras: attributes = {}
-) => isS3Mode
-  ? from(
-    new AWS.SNS().publish({
-      TopicArn: topicArn,
-      Message: JSON.stringify(params),
-      MessageAttributes: {
-        functionName: {
-          DataType: 'String',
-          StringValue: functionName
-        },
-        ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
-          key,
-          {
-            DataType: 'String',
-            StringValue: value
-          }
-        ])))
-      }
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`SNS publish ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+  functionName: string,
+  params: IFunctionParam,
+  topicArn = process.env.SNS_TOPIC,
+  extras: attributes = {}
+) =>
+  isS3Mode
+    ? from(
+        new AWS.SNS()
+          .publish({
+            TopicArn: topicArn,
+            Message: JSON.stringify(params),
+            MessageAttributes: {
+              functionName: {
+                DataType: 'String',
+                StringValue: functionName
+              },
+              ...Object.fromEntries(
+                Object.entries(extras || {}).map(([key, value]) => [
+                  key,
+                  {
+                    DataType: 'String',
+                    StringValue: value
+                  }
+                ])
+              )
+            }
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`SNS publish ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);

package/src/sqs.ts

@@ -10,31 +10,39 @@ type attributes = {
 };
 
 export const sendMessage = (
-  functionName: string, params: IFunctionParam, queueUrl = process.env.SQS_URL, extras: attributes = {}
-) => isS3Mode
-  ? from(
-    new AWS.SQS({ region }).sendMessage({
-      QueueUrl: queueUrl,
-      MessageBody: JSON.stringify(params),
-      MessageAttributes: {
-        functionName: {
-          DataType: 'String',
-          StringValue: functionName
-        },
-        ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
-          key,
-          {
-            DataType: 'String',
-            StringValue: value
-          }
-        ])))
-      }
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`SQS send message ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+  functionName: string,
+  params: IFunctionParam,
+  queueUrl = process.env.SQS_URL,
+  extras: attributes = {}
+) =>
+  isS3Mode
+    ? from(
+        new AWS.SQS({ region })
+          .sendMessage({
+            QueueUrl: queueUrl,
+            MessageBody: JSON.stringify(params),
+            MessageAttributes: {
+              functionName: {
+                DataType: 'String',
+                StringValue: functionName
+              },
+              ...Object.fromEntries(
+                Object.entries(extras || {}).map(([key, value]) => [
+                  key,
+                  {
+                    DataType: 'String',
+                    StringValue: value
+                  }
+                ])
+              )
+            }
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`SQS send message ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);