@ellipticltd/aml-utils 0.16.26 → 0.16.27

package/lib/file-parser/__tests/file-parser.spec.js

@@ -0,0 +1,107 @@
+import sinon from 'sinon';
+import { expect } from 'chai';
+import parseFile from '../file-parser';
+import * as sanitizeRows from '../sanitzeRows';
+import * as parseRow from '../parse-row';
+
+describe('file-parser', () => {
+  const dependencies = {
+    processRow: sinon.spy(),
+  };
+
+  const parsedRows = [];
+  let mockParseRow;
+  let mockSanitizeRows;
+  let mockSetTimeout;
+
+  before(() => {
+    mockParseRow = sinon.stub(parseRow, 'default').returns(parsedRows);
+    mockSanitizeRows = sinon.stub(sanitizeRows, 'default').returns([{}]);
+    mockSetTimeout = sinon.stub(global, 'setTimeout').callsFake((callback) => callback());
+  });
+
+  beforeEach(() => {
+    sinon.resetHistory();
+  });
+
+  after(() => {
+    sinon.restore();
+  });
+
+  it('should call parseRow once per input new line', async () => {
+    const input = '\n';
+
+    await parseFile(input, 50, dependencies);
+
+    expect(mockParseRow).to.be.calledTwice;
+  });
+
+  it('should call pass parseRow result into sanitizeRows', async () => {
+    const input = '\n';
+
+    await parseFile(input, 50, dependencies);
+
+    expect(mockSanitizeRows).to.be.calledOnce;
+    expect(mockSanitizeRows).to.be.deep.calledWith([[], []], dependencies.processRow);
+  });
+
+  it('should throw error when number of rows is more than max rows', async () => {
+    const input = '\n';
+
+    mockSanitizeRows.returns(new Array(20));
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.name).to.equal('TooManyRowsError');
+    }
+  });
+
+  it('should throw error when no rows returned from sanitizeRows', async () => {
+    const input = '\n';
+
+    mockSanitizeRows.returns([]);
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.name).to.equal('Error');
+    }
+  });
+
+  it('should throw error an error when content is "invalidFileType"', async () => {
+    const input = 'invalidFileType';
+
+    mockSanitizeRows.returns([]);
+    try {
+      await parseFile(input, 10, dependencies);
+      expect(1).to.equal(2);
+    } catch (ex) {
+      expect(ex.message).to.equal('Invalid file type');
+    }
+  });
+
+  it('should call process row once per row in single timeout', async () => {
+    const input = '\n';
+
+    const fakeRows = (new Array(30)).fill({});
+    mockSanitizeRows.returns(fakeRows);
+
+    await parseFile(input, 70, dependencies);
+
+    expect(dependencies.processRow).to.be.callCount(30);
+    expect(mockSetTimeout).to.be.calledOnce;
+  });
+
+  it('should call process row once per row in two timeouts when more than 30 rows', async () => {
+    const input = '\n';
+
+    const fakeRows = (new Array(50)).fill({});
+    mockSanitizeRows.returns(fakeRows);
+
+    await parseFile(input, 70, dependencies);
+
+    expect(dependencies.processRow).to.be.callCount(50);
+    expect(mockSetTimeout).to.be.calledTwice;
+  });
+});

package/lib/file-parser/__tests/parse-row.spec.js

@@ -0,0 +1,35 @@
+import { expect } from 'chai';
+import parseRow from '../parse-row';
+
+describe('parse-row', () => {
+  it('should handle tabs', () => {
+    const input = 'asdas \t asdar \t aaaaa \t bbb bb';
+
+    const result = parseRow(input);
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle comas', () => {
+    const input = 'asdas, asdar, aaaaa, bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle spaces', () => {
+    const input = 'asdas   asdar   aaaaa   bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(4);
+  });
+
+  it('should handle spaces', () => {
+    const input = 'asdas asdar aaaaa bbb bb';
+
+    const result = parseRow(input);
+
+    expect(result).to.have.length(5);
+  });
+});

package/lib/file-parser/__tests/sanitize-rows.spec.js

@@ -0,0 +1,88 @@
+import { expect } from 'chai';
+import sinon from 'sinon';
+import sanitizeRows from '../sanitzeRows';
+
+describe('sanitize-rows', () => {
+  it('should remove empty rows at the start of the array', () => {
+    const input = [
+      ['', '', ''],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove empty rows at the end of the array', () => {
+    const input = [
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove empty rows at the start and end of the array', () => {
+    const input = [
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(2);
+  });
+
+  it('should remove the first row if it is not empty and not valid', () => {
+    const input = [
+      ['', '', ''],
+      ['this', 'is', 'invalid'],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const returnsFalseOnce = sinon.stub();
+    returnsFalseOnce.returns(true);
+    returnsFalseOnce.onFirstCall().returns(false);
+
+    const result = sanitizeRows(input, returnsFalseOnce);
+    expect(result).to.have.length(2);
+  });
+
+  it('should allow rows with some empty values', () => {
+    const input = [
+      ['this', 'is', 'invalid'],
+      ['this', 'is', 'valid'],
+      ['this', 'is', 'valid'],
+      ['valid', '', ''],
+      ['', '', ''],
+      ['', '', ''],
+    ];
+
+    const result = sanitizeRows(input, () => ({
+      isValid: true,
+    }));
+
+    expect(result).to.have.length(4);
+  });
+});

package/lib/file-parser/errors.ts

@@ -0,0 +1,7 @@
+// eslint-disable-next-line import/prefer-default-export
+export class TooManyRowsError extends Error {
+  constructor() {
+    super('More than configured number of rows imported');
+    this.name = 'TooManyRowsError';
+  }
+}

package/lib/file-parser/file-parser.ts

@@ -0,0 +1,84 @@
+import parseRow from './parse-row';
+import { TooManyRowsError } from './errors';
+import sanitizeRows from './sanitzeRows';
+
+export type ProcessingEntry = {
+  isValid: boolean,
+};
+
+type Dependencies<T extends ProcessingEntry> = {
+  processRow: (cells: string[]) => T,
+};
+
+function chunk<T>(array: T[], size: number): T[][] {
+  if (!array.length) {
+    return [];
+  }
+  const head = array.slice(0, size);
+  const tail = array.slice(size);
+
+  return [head, ...chunk(tail, size)];
+}
+
+async function processChunk<T extends ProcessingEntry>(
+  parsedRows: string[][],
+  dependencies: Dependencies<T>,
+): Promise<T[]> {
+  const { processRow } = dependencies;
+  return new Promise(
+    (
+      resolve: (value: T[] | undefined) => void,
+      reject: (error: Error) => void,
+    ): void => {
+      setTimeout(
+        () => {
+          try {
+            const processedRows = parsedRows.map(processRow);
+            resolve(processedRows);
+          } catch (ex) {
+            reject(ex);
+          }
+        },
+        0,
+      );
+    },
+  );
+}
+
+async function parseFile<T extends ProcessingEntry>(
+  fileContent: string,
+  maxRows: number,
+  dependencies: Dependencies<T>,
+): Promise<T[]> {
+  const { processRow } = dependencies;
+
+  if (fileContent === 'invalidFileType') {
+    throw Error('Invalid file type');
+  }
+
+  const rows = fileContent.split('\n');
+  const parsedRows = rows.map(parseRow);
+
+  const sanitizedRows = sanitizeRows(parsedRows, processRow);
+
+  if (sanitizedRows.length > maxRows) {
+    throw new TooManyRowsError();
+  }
+
+  if (sanitizedRows.length === 0) {
+    throw new Error('No rows');
+  }
+
+  const parsedRowChunks = chunk(sanitizedRows, 30);
+  let results: T[] = [];
+  // eslint-disable-next-line no-restricted-syntax
+  for (const currentRowChunk of parsedRowChunks) {
+    // eslint-disable-next-line no-await-in-loop
+    const chunkResult = await processChunk(currentRowChunk, dependencies);
+    results = results.concat(chunkResult);
+  }
+
+  return results;
+}
+
+export default parseFile;

package/lib/file-parser/parse-row.ts

@@ -0,0 +1,52 @@
+import { pipe } from 'lodash/fp';
+
+const parseRow = (row: string): string[] => {
+  const trySplitByTab = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match('\t')) {
+      return line.split('\t');
+    }
+    return line;
+  };
+
+  const trySplitByComa = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(',')) {
+      return line.split(',');
+    }
+    return line;
+  };
+
+  const trySplitByMultipleSpaces = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(/ {3,}/)) {
+      return line.split(/ {3,}/);
+    }
+    return line;
+  };
+
+  const trySplitBySpace = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(' ')) {
+      return line.split(' ');
+    }
+    return line;
+  };
+
+  const trySplitByNothing = (line: string | string[]): string[] => {
+    if (typeof line === 'string') {
+      return [line];
+    }
+    return line;
+  };
+
+  const splitRow = pipe(
+    trySplitByTab,
+    trySplitByComa,
+    trySplitByMultipleSpaces,
+    trySplitBySpace,
+    trySplitByNothing,
+  );
+
+  const cells = splitRow(row);
+
+  return cells.map((c: string) => c.trim());
+};
+
+export default parseRow;

package/lib/file-parser/sanitzeRows.ts

@@ -0,0 +1,32 @@
+import { ProcessingEntry } from './file-parser';
+
+const sensicalRowCheck = (cells: string[]): boolean => (
+  cells.some((cell: string) => !!cell)
+);
+
+function sanitizeRows<T extends ProcessingEntry>(
+  parsedRows: string[][],
+  processRow: (row: string[]) => T,
+): string[][] {
+  const firstSensicalRowIndex = parsedRows
+    .findIndex(sensicalRowCheck);
+
+  const lastSensicalRowIndex = [...parsedRows]
+    .reverse()
+    .findIndex(sensicalRowCheck);
+
+  const sanitizedRows = parsedRows
+    .splice(
+      firstSensicalRowIndex,
+      parsedRows.length - firstSensicalRowIndex - lastSensicalRowIndex,
+    );
+
+  const firstRow = processRow(sanitizedRows[0]);
+
+  if (firstRow.isValid) {
+    return sanitizedRows;
+  }
+  return sanitizedRows.splice(1);
+}
+
+export default sanitizeRows;

package/lib/formatting/formatting.js

@@ -1,17 +1,18 @@
 const _ = require('lodash');
+
 const rethrowError = (fnName, e) => {
-    let formattedError;
-    if (_.isPlainObject(e.error) || _.isArray(e.error)) {
-        formattedError = `\n${JSON.stringify(e.error, null, 2)}`;
-    }
-    else {
-        formattedError = e.error;
-    }
-    return `${fnName} - ${e.name}: ${e.statusCode} - ${formattedError}`;
+  let formattedError;
+  if (_.isPlainObject(e.error) || _.isArray(e.error)) {
+    formattedError = `\n${JSON.stringify(e.error, null, 2)}`;
+  } else {
+    formattedError = e.error;
+  }
+  return `${fnName} - ${e.name}: ${e.statusCode} - ${formattedError}`;
 };
+
 const sqlEscapeWildcard = (str) => str.replace(/_|%|\\/g, (x) => `\\${x}`);
+
 module.exports = {
-    rethrowError,
-    sqlEscapeWildcard,
+  rethrowError,
+  sqlEscapeWildcard,
 };
-//# sourceMappingURL=formatting.js.map

package/lib/formatting/formatting.spec.js

@@ -0,0 +1,45 @@
+const chai = require('chai');
+const { sqlEscapeWildcard, rethrowError } = require('./formatting');
+
+const { expect } = chai;
+
+describe('formatting.sqlEscapeWildcard', () => {
+  describe('when passed an empty string', () => {
+    it('returns an empty string', () => sqlEscapeWildcard('').should.equal(''));
+  });
+  describe('when passed a string with no sql wildcards', () => {
+    it('returns the same string', () => {
+      const str = 'foo bar';
+      return sqlEscapeWildcard(str).should.equal(str);
+    });
+  });
+  return describe('when passed a string with sql wildcards', () => {
+    it('returns the string with \\ % _ escaped', () => {
+      const str = 'foo\\dfs_erwr%rewr';
+      return sqlEscapeWildcard(str).should.equal('foo\\\\dfs\\_erwr\\%rewr');
+    });
+  });
+});
+
+describe('rethrowError', () => {
+  it('returns an error when given a string', (
+
+  ) => rethrowError('myFunc', { error: 'Oh dear, myFunc is broken' })
+    .should.equal('myFunc - undefined: undefined - Oh dear, myFunc is broken'));
+
+  it('returns an error when passed an error object', () => rethrowError('myFunc', {
+    name: "I'm a teapot",
+    statusCode: 418,
+    error: "I'm a little teapot, short and stout",
+  }).should.equal("myFunc - I'm a teapot: 418 - I'm a little teapot, short and stout"));
+
+  it('accepts an array containing an error', () => {
+    const thrownError = rethrowError('myFunc', {
+      error: ["I'm a teapot", 418, "I'm a little teapot, short and stout"],
+    });
+
+    const expected = JSON.stringify('myFunc - undefined: undefined - \n[\n  "I\'m a teapot",\n  418,\n  "I\'m a little teapot, short and stout"\n]');
+
+    expect(JSON.stringify(thrownError)).to.equal(expected);
+  });
+});

package/lib/index.ts

@@ -0,0 +1,19 @@
+import types from './types/types';
+import validations from './validations/validations';
+import errors from './errors/errors';
+import formatting from './formatting/formatting';
+import middleware from './middleware/middleware';
+import ormHelpers from './orm-helpers/ormHelpers';
+import fileParser from './file-parser/file-parser';
+import structuredFileParser from './structured-file-parser/structured-file-parser';
+
+export {
+  types,
+  validations,
+  errors,
+  formatting,
+  fileParser,
+  structuredFileParser,
+  middleware,
+  ormHelpers,
+};

package/lib/middleware/middleware.js

@@ -1,22 +1,25 @@
 const T = require('../types/types');
+
 const ensureType = (type) => (req, res, next, id, name) => {
-    let error;
-    try {
-        T.ensureType(type, id, `Invalid ${name}`);
-        return next();
-    }
-    catch (error1) {
-        error = error1;
-        return next(error);
-    }
+  let error;
+  try {
+    T.ensureType(type, id, `Invalid ${name}`);
+    return next();
+  } catch (error1) {
+    error = error1;
+    return next(error);
+  }
 };
+
 const ensureUUID = ensureType('UUID');
+
 const ensureHex32 = ensureType('Hex32');
+
 const ensureInteger = ensureType('IntString');
+
 module.exports = {
-    ensureType,
-    ensureUUID,
-    ensureHex32,
-    ensureInteger,
+  ensureType,
+  ensureUUID,
+  ensureHex32,
+  ensureInteger,
 };
-//# sourceMappingURL=middleware.js.map

package/lib/orm-helpers/ormHelpers.js

@@ -1,17 +1,18 @@
 const _ = require('lodash');
+
 const includeNested = (orm, models) => _.map(_.toPairs(models), (arg) => {
-    const [modelName, inclusion] = arg;
-    const include = includeNested(orm, inclusion);
-    const model = orm[modelName];
-    if (_.isEmpty(include)) {
-        return model;
-    }
-    return {
-        model,
-        include,
-    };
+  const [modelName, inclusion] = arg;
+  const include = includeNested(orm, inclusion);
+  const model = orm[modelName];
+  if (_.isEmpty(include)) {
+    return model;
+  }
+  return {
+    model,
+    include,
+  };
 });
+
 module.exports = {
-    includeNested,
+  includeNested,
 };
-//# sourceMappingURL=ormHelpers.js.map

package/lib/orm-helpers/ormHelpers.spec.js

@@ -0,0 +1,41 @@
+const { includeNested } = require('./ormHelpers');
+
+describe('ormIncludeNested', () => {
+  const orm = {
+    a: 'a',
+    b: 'b',
+    c: 'c',
+    d: 'd',
+  };
+
+  describe('when passed an empty object', () => {
+    it('returns an empty array', () => includeNested(orm, {}).should.deep.equal([]));
+  });
+
+  describe('when passed an object with no inclusions', () => {
+    it('returns an array of just the key', () => includeNested(orm, {
+      a: {},
+    }).should.deep.equal(['a']));
+  });
+
+  describe('when passed a hierarchy of inclusions', () => {
+    it('should nest them in sequelize style', () => includeNested(orm, {
+      a: {
+        b: {
+          c: {},
+          d: {},
+        },
+      },
+    }).should.deep.equal([
+      {
+        model: 'a',
+        include: [
+          {
+            model: 'b',
+            include: ['c', 'd'],
+          },
+        ],
+      },
+    ]));
+  });
+});

package/lib/structured-file-parser/errors.ts

@@ -0,0 +1,25 @@
+/* eslint-disable max-classes-per-file */
+export class InvalidFileError extends Error {
+  constructor() {
+    super('Invalid file type');
+    this.name = 'InvalidFileError';
+  }
+}
+export class TooManyRowsError extends Error {
+  constructor() {
+    super('More than configured number of rows imported');
+    this.name = 'TooManyRowsError';
+  }
+}
+export class NoRowsError extends Error {
+  constructor() {
+    super('No rows');
+    this.name = 'NoRowsError';
+  }
+}
+export class InvalidHeadersError extends Error {
+  constructor() {
+    super('Invalid or incomplete headers list');
+    this.name = 'InvalidHeadersError';
+  }
+}