@hestia-earth/pipeline-utils 0.13.0 → 0.13.1

package/src/progress.ts

@@ -29,7 +29,11 @@ export class HestiaError extends Error {
    * @param key The file throwing this error
    * @param errors The errors
    */
-  constructor(public message, public key: string, public errors: any = '') {
+  constructor(
+    public message,
+    public key: string,
+    public errors: any = ''
+  ) {
     super(message);
   }
 
@@ -46,30 +50,43 @@ const fileToProgress = (path: string) => `${path.split('.')[0]}.progress`;
 
 const uploadProgress = (step: string, path: string, Bucket: string, since?: Date, err?: HestiaError, details?: any) =>
   of(fileToProgress(path)).pipe(
-    mergeMap(filepath => loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
-      catchError(() => of({} as Partial<IFileProgress>)),
-      mergeMap(data => uploadJSON(filepath, {
-        step,
-        success: !err,
-        time: {
-          ...(data.time || {}),
-          ...(since ? { [step]: timeSpent(since) } : {})
-        },
-        details,
-        error: err ? err.toString() : null
-      }, 'application/json', Bucket))
-    ))
+    mergeMap(filepath =>
+      loadJSONFile<Partial<IFileProgress>>(filepath, Bucket).pipe(
+        catchError(() => of({} as Partial<IFileProgress>)),
+        mergeMap(data =>
+          uploadJSON(
+            filepath,
+            {
+              step,
+              success: !err,
+              time: {
+                ...(data.time || {}),
+                ...(since ? { [step]: timeSpent(since) } : {})
+              },
+              details,
+              error: err ? err.toString() : null
+            },
+            'application/json',
+            Bucket
+          )
+        )
+      )
+    )
   );
 
-export const handleError = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
-  (err: HestiaError) => uploadProgress(step, params.key, Bucket, since, err).pipe(
-    mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
-  );
+export const handleError =
+  (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (err: HestiaError) =>
+    uploadProgress(step, params.key, Bucket, since, err).pipe(
+      mergeMap(() => publish(`${step}${publishFunctionSuffix.Error}`, params))
+    );
 
-export const handleSuccess = (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
-  (details?: any) => uploadProgress(step, params.key, Bucket, since, null, details).pipe(
-    mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
-  );
+export const handleSuccess =
+  (step: FileProgress, params: IFunctionParam, since?: Date, Bucket = bucket) =>
+  (details?: any) =>
+    uploadProgress(step, params.key, Bucket, since, null, details).pipe(
+      mergeMap(() => publish(`${step}${publishFunctionSuffix.Success}`, params))
+    );
 
 export const defaultErrorHandler = (key: string) => (err: Error) =>
   throwError(new HestiaError(err.message, key, err.stack));

package/src/s3.ts

@@ -32,43 +32,62 @@ export const getS3 = () => {
 };
 
 export const getObject = ({ Key, ...params }: AWS.S3.GetObjectRequest) =>
-  from(getS3().getObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
+  from(
+    getS3()
+      .getObject({ ...params, Key: decodeURI(Key) })
+      .promise()
+  ).pipe(
     take(1),
     map(({ Body }) => Body.toString())
   );
 
 export const putObject = ({ Key, ...params }: AWS.S3.PutObjectRequest) =>
-  from(getS3().putObject({ ...params, Key: decodeURI(Key) }).promise()).pipe(
+  from(
+    getS3()
+      .putObject({ ...params, Key: decodeURI(Key) })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
 export const deleteObjects = (Objects: AWS.S3.Object[], Bucket = bucket) =>
-  from(getS3().deleteObjects({
-    Bucket,
-    Delete: {
-      Objects: Objects.map(({ Key }) => ({ Key }))
-    }
-  }).promise()).pipe(
+  from(
+    getS3()
+      .deleteObjects({
+        Bucket,
+        Delete: {
+          Objects: Objects.map(({ Key }) => ({ Key }))
+        }
+      })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
 export const copyObject = (fromParams, toParams, MetadataDirective?: string) =>
-  from(getS3().copyObject({
-    Bucket: toParams.bucket,
-    Key: decodeURI(toParams.key),
-    CopySource: `${fromParams.bucket}/${fromParams.key}`,
-    MetadataDirective
-  }).promise()).pipe(
+  from(
+    getS3()
+      .copyObject({
+        Bucket: toParams.bucket,
+        Key: decodeURI(toParams.key),
+        CopySource: `${fromParams.bucket}/${fromParams.key}`,
+        MetadataDirective
+      })
+      .promise()
+  ).pipe(
     take(1),
     map(res => (res.$response.data || '').toString())
   );
 
-export const loadStream = (key: string, Bucket = bucket) => getS3().getObject({
-  Bucket,
-  Key: decodeURI(key)
-}).createReadStream();
+export const loadStream = (key: string, Bucket = bucket) =>
+  getS3()
+    .getObject({
+      Bucket,
+      Key: decodeURI(key)
+    })
+    .createReadStream();
 
 export const loadFile = (key: string) => {
   const stream = loadStream(key);
@@ -86,80 +105,122 @@ export const loadFile = (key: string) => {
   return subject.asObservable();
 };
 
-export const loadTextFile = (key: string, Bucket = bucket) => isS3Mode ? getObject({
-  Bucket,
-  Key: key
-}).pipe(map(res => res.replace(/\"/g, ''))) : local.loadTextFile(Bucket)(key);
-
-export const loadJSONFile = <T>(key: string, Bucket = bucket): Observable<T> => isS3Mode ? getObject({
-  Bucket,
-  Key: key,
-  ResponseContentType: 'application/json'
-}).pipe(map(res => JSON.parse(stripBOM(res)) as T)) : local.loadJSONFile(Bucket)(key);
-
-export const uploadFile = (filepath: string, key: string, Bucket = bucket) => putObject({
-  Bucket,
-  Key: key,
-  Body: createReadStream(filepath, 'utf8')
-});
-
-export const uploadText = (key: string, data: string, Bucket = bucket) => isS3Mode ? putObject({
-  Bucket,
-  Key: key,
-  Body: data
-}) : local.uploadText(Bucket)(key, data);
-
-export const uploadJSON = (key: string, data, type = 'application/json', Bucket = bucket) => isS3Mode ? putObject({
-  Bucket,
-  Key: key,
-  Body: JSON.stringify(data, null, 2),
-  ContentType: type
-}) : local.uploadJSON(Bucket)(key, data);
-
-export const waitObjectCreated = (key: string, Bucket = bucket) => isS3Mode ?
-  from(getS3().waitFor('objectExists', {
+export const loadTextFile = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? getObject({
+        Bucket,
+        Key: key
+      }).pipe(map(res => res.replace(/\"/g, '')))
+    : local.loadTextFile(Bucket)(key);
+
+export const loadJSONFile = <T>(key: string, Bucket = bucket): Observable<T> =>
+  isS3Mode
+    ? getObject({
+        Bucket,
+        Key: key,
+        ResponseContentType: 'application/json'
+      }).pipe(map(res => JSON.parse(stripBOM(res)) as T))
+    : local.loadJSONFile(Bucket)(key);
+
+export const uploadFile = (filepath: string, key: string, Bucket = bucket) =>
+  putObject({
     Bucket,
-    Key: decodeURI(key)
-  }).promise()).pipe(
-    take(1),
-    map(() => true)
-  ) : of(true);
-
-const headObject = (key: string, Bucket = bucket) => from(getS3().headObject({
-  Bucket,
-  Key: decodeURI(key)
-}).promise());
-
-export const lastModified = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(({ LastModified }) => LastModified),
-  catchError(() => of(null as Date)),
-  take(1)
-) : local.lastModified(Bucket)(key);
-
-export const fileExists = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(() => true),
-  catchError(() => of(false)),
-  take(1)
-) : local.fileExists(Bucket)(key);
-
-export const fileSize = (key: string, Bucket = bucket) => isS3Mode ? headObject(key, Bucket).pipe(
-  map(({ ContentLength }) => ContentLength),
-  catchError(() => of(undefined as number)),
-  take(1)
-) : local.fileSize(Bucket)(key);
-
-export const deleteObject = (key: string, Bucket = bucket) => isS3Mode ? from(getS3().deleteObject({
-  Bucket,
-  Key: decodeURI(key)
-}).promise()) : local.deleteFile(Bucket)(key);
+    Key: key,
+    Body: createReadStream(filepath, 'utf8')
+  });
+
+export const uploadText = (key: string, data: string, Bucket = bucket) =>
+  isS3Mode
+    ? putObject({
+        Bucket,
+        Key: key,
+        Body: data
+      })
+    : local.uploadText(Bucket)(key, data);
+
+export const uploadJSON = (key: string, data, type = 'application/json', Bucket = bucket) =>
+  isS3Mode
+    ? putObject({
+        Bucket,
+        Key: key,
+        Body: JSON.stringify(data, null, 2),
+        ContentType: type
+      })
+    : local.uploadJSON(Bucket)(key, data);
+
+export const waitObjectCreated = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? from(
+        getS3()
+          .waitFor('objectExists', {
+            Bucket,
+            Key: decodeURI(key)
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(() => true)
+      )
+    : of(true);
+
+const headObject = (key: string, Bucket = bucket) =>
+  from(
+    getS3()
+      .headObject({
+        Bucket,
+        Key: decodeURI(key)
+      })
+      .promise()
+  );
+
+export const lastModified = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(({ LastModified }) => LastModified),
+        catchError(() => of(null as Date)),
+        take(1)
+      )
+    : local.lastModified(Bucket)(key);
+
+export const fileExists = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(() => true),
+        catchError(() => of(false)),
+        take(1)
+      )
+    : local.fileExists(Bucket)(key);
+
+export const fileSize = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? headObject(key, Bucket).pipe(
+        map(({ ContentLength }) => ContentLength),
+        catchError(() => of(undefined as number)),
+        take(1)
+      )
+    : local.fileSize(Bucket)(key);
+
+export const deleteObject = (key: string, Bucket = bucket) =>
+  isS3Mode
+    ? from(
+        getS3()
+          .deleteObject({
+            Bucket,
+            Key: decodeURI(key)
+          })
+          .promise()
+      )
+    : local.deleteFile(Bucket)(key);
 
 export const listFolder = async (Prefix = '', Bucket = bucket, nextToken?: string): Promise<AWS.S3.Object[]> => {
-  const { Contents, IsTruncated, NextContinuationToken } = await getS3().listObjectsV2({
-    Bucket,
-    Prefix,
-    ContinuationToken: nextToken,
-    MaxKeys: 1000
-  }).promise();
+  const { Contents, IsTruncated, NextContinuationToken } = await getS3()
+    .listObjectsV2({
+      Bucket,
+      Prefix,
+      ContinuationToken: nextToken,
+      MaxKeys: 1000
+    })
+    .promise();
   return [...Contents, ...(IsTruncated ? await listFolder(Prefix, bucket, NextContinuationToken) : [])];
 };
 

package/src/sns.ts

@@ -10,31 +10,39 @@ type attributes = {
 };
 
 export const publish = (
-  functionName: string, params: IFunctionParam, topicArn = process.env.SNS_TOPIC, extras: attributes = {}
-) => isS3Mode
-  ? from(
-    new AWS.SNS().publish({
-      TopicArn: topicArn,
-      Message: JSON.stringify(params),
-      MessageAttributes: {
-        functionName: {
-          DataType: 'String',
-          StringValue: functionName
-        },
-        ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
-          key,
-          {
-            DataType: 'String',
-            StringValue: value
-          }
-        ])))
-      }
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`SNS publish ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+  functionName: string,
+  params: IFunctionParam,
+  topicArn = process.env.SNS_TOPIC,
+  extras: attributes = {}
+) =>
+  isS3Mode
+    ? from(
+        new AWS.SNS()
+          .publish({
+            TopicArn: topicArn,
+            Message: JSON.stringify(params),
+            MessageAttributes: {
+              functionName: {
+                DataType: 'String',
+                StringValue: functionName
+              },
+              ...Object.fromEntries(
+                Object.entries(extras || {}).map(([key, value]) => [
+                  key,
+                  {
+                    DataType: 'String',
+                    StringValue: value
+                  }
+                ])
+              )
+            }
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`SNS publish ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);

package/src/sqs.ts

@@ -10,31 +10,39 @@ type attributes = {
 };
 
 export const sendMessage = (
-  functionName: string, params: IFunctionParam, queueUrl = process.env.SQS_URL, extras: attributes = {}
-) => isS3Mode
-  ? from(
-    new AWS.SQS({ region }).sendMessage({
-      QueueUrl: queueUrl,
-      MessageBody: JSON.stringify(params),
-      MessageAttributes: {
-        functionName: {
-          DataType: 'String',
-          StringValue: functionName
-        },
-        ...(Object.fromEntries(Object.entries(extras || {}).map(([key, value]) => [
-          key,
-          {
-            DataType: 'String',
-            StringValue: value
-          }
-        ])))
-      }
-    }).promise()
-  ).pipe(
-    take(1),
-    map(res => {
-      debug(`SQS send message ${functionName}`, res);
-      return res;
-    })
-  )
-  : of(null);
+  functionName: string,
+  params: IFunctionParam,
+  queueUrl = process.env.SQS_URL,
+  extras: attributes = {}
+) =>
+  isS3Mode
+    ? from(
+        new AWS.SQS({ region })
+          .sendMessage({
+            QueueUrl: queueUrl,
+            MessageBody: JSON.stringify(params),
+            MessageAttributes: {
+              functionName: {
+                DataType: 'String',
+                StringValue: functionName
+              },
+              ...Object.fromEntries(
+                Object.entries(extras || {}).map(([key, value]) => [
+                  key,
+                  {
+                    DataType: 'String',
+                    StringValue: value
+                  }
+                ])
+              )
+            }
+          })
+          .promise()
+      ).pipe(
+        take(1),
+        map(res => {
+          debug(`SQS send message ${functionName}`, res);
+          return res;
+        })
+      )
+    : of(null);

package/src/utils.ts

@@ -6,31 +6,35 @@ import { error } from './log';
 
 export const recordToFunctionParam = (event: SNSEventRecord | S3EventRecord): IFunctionParam =>
   reduceUndefinedValues(
-    'Sns' in event ? {
-      ...JSON.parse(event.Sns.Message),
-      timestamp: event.Sns.Timestamp ? new Date(event.Sns.Timestamp) : undefined
-    } : {
-      bucket: event.s3.bucket.name,
-      key: event.s3.object.key,
-      filepath: event.s3.object.key,
-      timestamp: event.eventTime ? new Date(event.eventTime) : undefined
-    }
+    'Sns' in event
+      ? {
+          ...JSON.parse(event.Sns.Message),
+          timestamp: event.Sns.Timestamp ? new Date(event.Sns.Timestamp) : undefined
+        }
+      : {
+          bucket: event.s3.bucket.name,
+          key: event.s3.object.key,
+          filepath: event.s3.object.key,
+          timestamp: event.eventTime ? new Date(event.eventTime) : undefined
+        }
   ) as IFunctionParam;
 
-export const functionParamToS3Record = ({ bucket, key, timestamp }: IFunctionParam) => ({
-  s3: {
-    bucket: { name: bucket },
-    object: { key }
-  },
-  ...(timestamp ? { eventTime: timestamp.toJSON() } : {})
-}) as Partial<S3EventRecord>;
+export const functionParamToS3Record = ({ bucket, key, timestamp }: IFunctionParam) =>
+  ({
+    s3: {
+      bucket: { name: bucket },
+      object: { key }
+    },
+    ...(timestamp ? { eventTime: timestamp.toJSON() } : {})
+  }) as Partial<S3EventRecord>;
 
-export const functionParamToSNSRecord = ({ bucket, key, timestamp }: IFunctionParam) => ({
-  Sns: {
-    Message: JSON.stringify({ bucket, key })
-  },
-  ...(timestamp ? { Timestamp: timestamp.toJSON() } : {})
-}) as Partial<SNSEventRecord>;
+export const functionParamToSNSRecord = ({ bucket, key, timestamp }: IFunctionParam) =>
+  ({
+    Sns: {
+      Message: JSON.stringify({ bucket, key })
+    },
+    ...(timestamp ? { Timestamp: timestamp.toJSON() } : {})
+  }) as Partial<SNSEventRecord>;
 
 export const returnResponse = (data: any, statusCode = 200) => {
   return {
@@ -45,36 +49,39 @@ export const returnResponse = (data: any, statusCode = 200) => {
 
 const defaultErrorStatus = 500;
 
-export const returnError = (err) => {
+export const returnError = err => {
   error('handleError', err, err?.stack, err?.meta);
-  const { error: errorContent, status = defaultErrorStatus } = typeof err === 'string'
-    ? { error: err }
-    : err.meta
+  const { error: errorContent, status = defaultErrorStatus } =
+    typeof err === 'string'
+      ? { error: err }
+      : err.meta
       ? err.meta.body
       : { error: err.message, status: defaultErrorStatus };
   return returnResponse({ error: errorContent }, status || defaultErrorStatus);
 };
 
 export const spliceAndProcess = <T, R>(docs: T[], func: (p: T[]) => Promise<R>, count = 1) =>
-  new Promise<R|void>(resolve => {
+  new Promise<R | void>(resolve => {
     if (!docs.length) {
       return resolve();
     }
     const remaining = docs.splice(docs.length - count);
-    void func(remaining).then(() => spliceAndProcess(docs, func, count)).then(resolve);
+
+    void func(remaining)
+      .then(() => spliceAndProcess(docs, func, count))
+      .then(resolve);
   });
 
-export const isEmpty = (value: any, minKeys = 1) => typeof value === 'object' ?
-  (
-    Array.isArray(value) ?
-      !value.length :
-      Object.keys(value).filter(key => key !== 'type').length < minKeys
-  ) :
-  typeof value === 'undefined' || value === null || value === '';
+export const isEmpty = (value: any, minKeys = 1) =>
+  typeof value === 'object'
+    ? Array.isArray(value)
+      ? !value.length
+      : Object.keys(value).filter(key => key !== 'type').length < minKeys
+    : typeof value === 'undefined' || value === null || value === '';
 
 export const refToSchemaType = ref => ref.substring(2).replace('.json#', '');
 
-export const prependWithKey = (data: any, key: string) => Object.keys(data)
-  .reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
+export const prependWithKey = (data: any, key: string) =>
+  Object.keys(data).reduce((prev, curr) => ({ ...prev, [`${key}.${curr}`]: data[curr] }), {});
 
-export const stripBOM = (content: string) => content.charCodeAt(0) === 0xFEFF ? content.slice(1) : content;
+export const stripBOM = (content: string) => (content.charCodeAt(0) === 0xfeff ? content.slice(1) : content);