@ellipticltd/aml-utils 0.7.0 → 0.8.0-EN-2570.1

package/lib/file-parser/__tests/file-parser.spec.js

@@ -0,0 +1,101 @@
+import sinon from 'sinon';
+import { expect } from 'chai';
+import parseFile from '../file-parser';
+import * as sanitizeRows from '../sanitzeRows';
+import * as parseRow from '../parse-row';
+
+describe('file-parser', () => {
+  const dependencies = {
+    processRow: sinon.spy(),
+  };
+
+  const parsedRows = [];
+  const mockParseRow = sinon.stub(parseRow, 'default').returns(parsedRows);
+  const mockSanitizeRows = sinon.stub(sanitizeRows, 'default').returns([{}]);
+  const mockSetTimeout = sinon.stub(global, 'setTimeout').callsFake((callback) => callback());
+
+  beforeEach(() => {
+    sinon.resetHistory();
+  });
+
+  after(() => {
+    sinon.restore();
+  });
+
+  it('should call parseRow once per input new line', async () => {
+    const input = '\n';
+
+    await parseFile(input, 50, dependencies);
+
+    expect(mockParseRow).to.be.calledTwice;
+  });
+
+  it('should call pass parseRow result into sanitizeRows', async () => {
+    const input = '\n';
+
+    await parseFile(input, 50, dependencies);
+
+    expect(mockSanitizeRows).to.be.calledOnce;
+    expect(mockSanitizeRows).to.be.deep.calledWith([[], []], dependencies.processRow);
+  });
+
+  it('should throw error when number of rows is more than max rows', async () => {
+    const input = '\n';
+
+    mockSanitizeRows.returns(new Array(20));
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.name).to.equal('TooManyRowsError');
+    }
+  });
+
+  it('should throw error when no rows returned from sanitizeRows', async () => {
+    const input = '\n';
+
+    mockSanitizeRows.returns([]);
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.name).to.equal('Error');
+    }
+  });
+
+  it('should throw error an error when content is "invalidFileType"', async () => {
+    const input = 'invalidFileType';
+
+    mockSanitizeRows.returns([]);
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.message).to.equal('Invalid file type');
+    }
+  });
+
+  it('should call process row once per row in single timeout', async () => {
+    const input = '\n';
+
+    const fakeRows = (new Array(30)).fill({});
+    mockSanitizeRows.returns(fakeRows);
+
+    await parseFile(input, 70, dependencies);
+
+    expect(dependencies.processRow).to.be.callCount(30);
+    expect(mockSetTimeout).to.be.calledOnce;
+  });
+
+  it('should call process row once per row in two timeouts when more than 30 rows', async () => {
+    const input = '\n';
+
+    const fakeRows = (new Array(50)).fill({});
+    mockSanitizeRows.returns(fakeRows);
+
+    await parseFile(input, 70, dependencies);
+
+    expect(dependencies.processRow).to.be.callCount(50);
+    expect(mockSetTimeout).to.be.calledTwice;
+  });
+});

package/lib/file-parser/__tests/parse-row.spec.js

@@ -0,0 +1,35 @@
+import { expect } from 'chai';
+import parseRow from '../parse-row';
+
+describe('parse-row', () => {
+  it('should handle tabs', () => {
+    const input = 'asdas \t asdar \t aaaaa \t bbb bb';
+
+    const result = parseRow(input);
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle comas', () => {
+    const input = 'asdas, asdar, aaaaa, bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle spaces', () => {
+    const input = 'asdas   asdar   aaaaa   bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle spaces', () => {
+    const input = 'asdas asdar aaaaa bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(5);
+  });
+});

package/lib/file-parser/__tests/sanitize-rows.spec.js

@@ -0,0 +1,88 @@
+import { expect } from 'chai';
+import sinon from 'sinon';
+import sanitizeRows from '../sanitzeRows';
+
+describe('sanitize-rows', () => {
+  it('should remove empty rows at the start of the array', () => {
+    const input = [
+      ['', '', ''],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove empty rows at the end of the array', () => {
+    const input = [
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove empty rows at the start and end of the array', () => {
+    const input = [
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove the first row if it is not empty and not valid', () => {
+    const input = [
+      ['', '', ''],
+      ['this', 'is', 'invalid'],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const returnsFalseOnce = sinon.stub();
+    returnsFalseOnce.returns(true);
+    returnsFalseOnce.onFirstCall().returns(false);
+
+    const result = sanitizeRows(input, returnsFalseOnce);
+    expect(result).to.have.length(2);
+  });
+
+  it('should allow rows with some empty values', () => {
+    const input = [
+      ['this', 'is', 'invalid'],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['valid', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(4);
+  });
+});

package/lib/file-parser/errors.ts

@@ -0,0 +1,7 @@
+// eslint-disable-next-line import/prefer-default-export
+export class TooManyRowsError extends Error {
+  constructor() {
+    super('More than configured number of rows imported');
+    this.name = 'TooManyRowsError';
+  }
+}

package/lib/file-parser/file-parser.ts

@@ -0,0 +1,84 @@
+import parseRow from './parse-row';
+import { TooManyRowsError } from './errors';
+import sanitizeRows from './sanitzeRows';
+
+export type ProcessingEntry = {
+  isValid: boolean,
+};
+
+type Dependencies<T extends ProcessingEntry> = {
+  processRow: () => T,
+};
+
+function chunk<T>(array: T[], size: number): T[][] {
+  if (!array.length) {
+    return [];
+  }
+  const head = array.slice(0, size);
+  const tail = array.slice(size);
+
+  return [head, ...chunk(tail, size)];
+}
+
+async function processChunk<T extends ProcessingEntry>(
+  parsedRows: string[][],
+  dependencies: Dependencies<T>,
+): Promise<T[]> {
+  const { processRow } = dependencies;
+  return new Promise(
+    (
+      resolve: (value: T[] | undefined) => void,
+      reject: (error: Error) => void,
+    ): void => {
+      setTimeout(
+        () => {
+          try {
+            const processedRows = parsedRows.map(processRow);
+            resolve(processedRows);
+          } catch (ex) {
+            reject(ex);
+          }
+        },
+        0,
+      );
+    },
+  );
+}
+
+async function parseFile<T extends ProcessingEntry>(
+  fileContent: string,
+  maxRows: number,
+  dependencies: Dependencies<T>,
+): Promise<T[]> {
+  const { processRow } = dependencies;
+
+  if (fileContent === 'invalidFileType') {
+    throw Error('Invalid file type');
+  }
+
+  const rows = fileContent.split('\n');
+  const parsedRows = rows.map(parseRow);
+
+  const sanitizedRows = sanitizeRows(parsedRows, processRow);
+
+  if (sanitizedRows.length > maxRows) {
+    throw new TooManyRowsError();
+  }
+
+  if (sanitizedRows.length === 0) {
+    throw new Error('No rows');
+  }
+
+  const parsedRowChunks = chunk(sanitizedRows, 30);
+  let results: T[] = [];
+  // eslint-disable-next-line no-restricted-syntax
+  for (const currentRowChunk of parsedRowChunks) {
+    // eslint-disable-next-line no-await-in-loop
+    const chunkResult = await processChunk(currentRowChunk, dependencies);
+    results = results.concat(chunkResult);
+  }
+
+  return results;
+}
+
+export default parseFile;

package/lib/file-parser/parse-row.ts

@@ -0,0 +1,52 @@
+import { pipe } from 'lodash/fp';
+
+const parseRow = (row: string): string[] => {
+  const trySplitByTab = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match('\t')) {
+      return line.split('\t');
+    }
+    return line;
+  };
+
+  const trySplitByComa = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(',')) {
+      return line.split(',');
+    }
+    return line;
+  };
+
+  const trySplitByMultipleSpaces = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(/ {3,}/)) {
+      return line.split(/ {3,}/);
+    }
+    return line;
+  };
+
+  const trySplitBySpace = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(' ')) {
+      return line.split(' ');
+    }
+    return line;
+  };
+
+  const trySplitByNothing = (line: string | string[]): string[] => {
+    if (typeof line === 'string') {
+      return [line];
+    }
+    return line;
+  };
+
+  const splitRow = pipe(
+    trySplitByTab,
+    trySplitByComa,
+    trySplitByMultipleSpaces,
+    trySplitBySpace,
+    trySplitByNothing,
+  );
+
+  const cells = splitRow(row);
+
+  return cells.map((c: string) => c.trim());
+};
+
+export default parseRow;

package/lib/file-parser/sanitzeRows.ts

@@ -0,0 +1,32 @@
+import { ProcessingEntry } from './file-parser';
+
+const sensicalRowCheck = (cells: string[]): boolean => (
+  cells.some((cell: string) => !!cell)
+);
+
+function sanitizeRows<T extends ProcessingEntry>(
+  parsedRows: string[][],
+  processRow: (row: string[]) => T,
+): string[][] {
+  const firstSensicalRowIndex = parsedRows
+    .findIndex(sensicalRowCheck);
+
+  const lastSensicalRowIndex = [...parsedRows]
+    .reverse()
+    .findIndex(sensicalRowCheck);
+
+  const sanitizedRows = parsedRows
+    .splice(
+      firstSensicalRowIndex,
+      parsedRows.length - firstSensicalRowIndex - lastSensicalRowIndex,
+    );
+
+  const firstRow = processRow(sanitizedRows[0]);
+
+  if (firstRow.isValid) {
+    return sanitizedRows;
+  }
+  return sanitizedRows.splice(1);
+}
+
+export default sanitizeRows;

package/lib/formatting/formatting.spec.js

@@ -0,0 +1,45 @@
+const chai = require('chai');
+const { sqlEscapeWildcard, rethrowError } = require('./formatting');
+
+const { expect } = chai;
+
+describe('formatting.sqlEscapeWildcard', () => {
+  describe('when passed an empty string', () => {
+    it('returns an empty string', () => sqlEscapeWildcard('').should.equal(''));
+  });
+  describe('when passed a string with no sql wildcards', () => {
+    it('returns the same string', () => {
+      const str = 'foo bar';
+      return sqlEscapeWildcard(str).should.equal(str);
+    });
+  });
+  return describe('when passed a string with sql wildcards', () => {
+    it('returns the string with \\ % _ escaped', () => {
+      const str = 'foo\\dfs_erwr%rewr';
+      return sqlEscapeWildcard(str).should.equal('foo\\\\dfs\\_erwr\\%rewr');
+    });
+  });
+});
+
+describe('rethrowError', () => {
+  it('returns an error when given a string', (
+
+  ) => rethrowError('myFunc', { error: 'Oh dear, myFunc is broken' })
+    .should.equal('myFunc - undefined: undefined - Oh dear, myFunc is broken'));
+
+  it('returns an error when passed an error object', () => rethrowError('myFunc', {
+    name: "I'm a teapot",
+    statusCode: 418,
+    error: "I'm a little teapot, short and stout",
+  }).should.equal("myFunc - I'm a teapot: 418 - I'm a little teapot, short and stout"));
+
+  it('accepts an array containing an error', () => {
+    const thrownError = rethrowError('myFunc', {
+      error: ["I'm a teapot", 418, "I'm a little teapot, short and stout"],
+    });
+
+    const expected = JSON.stringify('myFunc - undefined: undefined - \n[\n  "I\'m a teapot",\n  418,\n  "I\'m a little teapot, short and stout"\n]');
+
+    expect(JSON.stringify(thrownError)).to.equal(expected);
+  });
+});

package/lib/index.ts

@@ -0,0 +1,17 @@
+import types from './types/types';
+import validations from './validations/validations';
+import errors from './errors/errors';
+import formatting from './formatting/formatting';
+import middleware from './middleware/middleware';
+import ormHelpers from './orm-helpers/ormHelpers';
+import fileParser from './file-parser/file-parser';
+
+export {
+  types,
+  validations,
+  errors,
+  formatting,
+  fileParser,
+  middleware,
+  ormHelpers,
+};

package/lib/{middleware.js → middleware/middleware.js}

@@ -1,4 +1,4 @@
-const T = require('./types');
+const T = require('../types/types');
 
 const ensureType = (type) => (req, res, next, id, name) => {
   let error;

package/lib/orm-helpers/ormHelpers.spec.js

@@ -0,0 +1,41 @@
+const { includeNested } = require('./ormHelpers');
+
+describe('ormIncludeNested', () => {
+  const orm = {
+    a: 'a',
+    b: 'b',
+    c: 'c',
+    d: 'd',
+  };
+
+  describe('when passed an empty object', () => {
+    it('returns an empty array', () => includeNested(orm, {}).should.deep.equal([]));
+  });
+
+  describe('when passed an object with no inclusions', () => {
+    it('returns an array of just the key', () => includeNested(orm, {
+      a: {},
+    }).should.deep.equal(['a']));
+  });
+
+  describe('when passed a hierarchy of inclusions', () => {
+    it('should nest them in sequelize style', () => includeNested(orm, {
+      a: {
+        b: {
+          c: {},
+          d: {},
+        },
+      },
+    }).should.deep.equal([
+      {
+        model: 'a',
+        include: [
+          {
+            model: 'b',
+            include: ['c', 'd'],
+          },
+        ],
+      },
+    ]));
+  });
+});

package/lib/{types.js → types/types.js}

@@ -2,9 +2,9 @@ const TC = require('type-check');
 
 const _ = require('lodash');
 
-const E = require('./errors');
+const E = require('../errors/errors');
 
-const V = require('./validations');
+const V = require('../validations/validations');
 
 const customTypes = {
   Integer: {

package/lib/{validations.js → validations/validations.js}

@@ -12,8 +12,7 @@ const addressValidator = require('wallet-address-validator');
 const zilUtils = require('@zilliqa-js/util');
 const web3 = require('web3-utils');
 const stellarSDK = require('stellar-sdk');
-const E = require('./errors');
-
+const E = require('../errors/errors');
 
 const validations = {
   _validate(Err, pred, x, msg) {
@@ -115,6 +114,17 @@ const validations = {
       && (x != null ? x.length : undefined) <= 50
     );
   },
+  binanceCoin: {
+    isAddress(str) {
+      return /^(bnb)([a-z0-9]{39})$/.test(str);
+    },
+    isTxHash(str) {
+      if (!V.isHexadecimal(str)) {
+        return false;
+      }
+      return str.length === 64;
+    },
+  },
   bitcoin: {
     isAddress(str) {
       const network = process.env.BITCOIN_NETWORK === 'testnet' ? 'testnet' : 'prod';
@@ -219,6 +229,17 @@ const validations = {
       }
     },
   },
+  horizen: {
+    isAddress(str) {
+      return addressValidator.validate(str, 'ZEN');
+    },
+    isTxHash(str) {
+      if (!V.isHexadecimal(str)) {
+        return false;
+      }
+      return str.length === 64;
+    },
+  },
   litecoin: {
     isAddress(str) {
       return /^[3LM][a-km-zA-HJ-NP-Z1-9]{24,33}$|^ltc1[ac-hj-np-z02-9]{6,86}$|^LTC1[AC-HJ-NP-Z02-9]{6,86}$/.test(str);
@@ -241,17 +262,20 @@ const validations = {
       return str.length === 64;
     },
   },
-  zilliqa: {
+  stellar: {
     isAddress(str) {
-      return zilUtils.validation.isBech32(str);
+      return stellarSDK.StrKey.isValidEd25519PublicKey(str);
     },
     isTxHash(str) {
-      return str.length === 66 && web3.isHexStrict(str);
+      if (!V.isHexadecimal(str)) {
+        return false;
+      }
+      return str.length === 64;
     },
   },
-  stellar: {
+  zcash: {
     isAddress(str) {
-      return stellarSDK.StrKey.isValidEd25519PublicKey(str);
+      return addressValidator.validate(str, 'ZEC');
     },
     isTxHash(str) {
       if (!V.isHexadecimal(str)) {
@@ -260,6 +284,14 @@ const validations = {
       return str.length === 64;
     },
   },
+  zilliqa: {
+    isAddress(str) {
+      return zilUtils.validation.isBech32(str);
+    },
+    isTxHash(str) {
+      return str.length === 66 && web3.isHexStrict(str);
+    },
+  },
 };
 
 Object.keys(V).forEach((k) => {