@ellipticltd/aml-utils 0.16.26 → 0.16.27

package/lib/structured-file-parser/parse-row.ts

@@ -0,0 +1,52 @@
+import { pipe } from 'lodash/fp';
+
+const parseRow = (row: string): string[] => {
+  const trySplitByTab = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match('\t')) {
+      return line.split('\t');
+    }
+    return line;
+  };
+
+  const trySplitByComa = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(',')) {
+      return line.split(',');
+    }
+    return line;
+  };
+
+  const trySplitByMultipleSpaces = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(/ {3,}/)) {
+      return line.split(/ {3,}/);
+    }
+    return line;
+  };
+
+  const trySplitBySpace = (line: string | string[]): string | string[] => {
+    if (typeof line === 'string' && line.match(' ')) {
+      return line.split(' ');
+    }
+    return line;
+  };
+
+  const trySplitByNothing = (line: string | string[]): string[] => {
+    if (typeof line === 'string') {
+      return [line];
+    }
+    return line;
+  };
+
+  const splitRow = pipe(
+    trySplitByTab,
+    trySplitByComa,
+    trySplitByMultipleSpaces,
+    trySplitBySpace,
+    trySplitByNothing,
+  );
+
+  const cells = splitRow(row);
+
+  return cells.map((c: string) => c.trim());
+};
+
+export default parseRow;

package/lib/structured-file-parser/sanitize-rows.ts

@@ -0,0 +1,24 @@
+const sensicalRowCheck = (cells: string[]): boolean => (
+  cells.some((cell: string) => !!cell)
+);
+
+function sanitizeRows(parsedRows: string[][]): string[][] {
+  const firstSensicalRowIndex = parsedRows.findIndex(sensicalRowCheck);
+
+  const lastSensicalRowIndex = [...parsedRows]
+    .reverse()
+    .findIndex(sensicalRowCheck);
+
+  if (firstSensicalRowIndex < 0) {
+    return [];
+  }
+
+  const sanitizedRows = [...parsedRows].splice(
+    firstSensicalRowIndex,
+    parsedRows.length - firstSensicalRowIndex - lastSensicalRowIndex,
+  );
+
+  return sanitizedRows;
+}
+
+export default sanitizeRows;

package/lib/structured-file-parser/structured-file-parser.ts

@@ -0,0 +1,155 @@
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-restricted-syntax */
+import parseRow from './parse-row';
+import {
+  InvalidFileError,
+  TooManyRowsError,
+  NoRowsError,
+  InvalidHeadersError,
+} from './errors';
+import sanitizeRows from './sanitize-rows';
+
+export type ProcessingEntry = {
+  isValid: boolean;
+};
+
+type Dependencies<T extends ProcessingEntry> = {
+  processRow: (cells: string[], headers: string[]) => T;
+};
+
+type CompleteDependencies<T extends ProcessingEntry, U extends string[]> = {
+  processRow: (cells: string[], headers: U) => T;
+  headers: U;
+};
+
+function chunk<T>(array: T[], size: number): T[][] {
+  if (!array.length) {
+    return [];
+  }
+  const head = array.slice(0, size);
+  const tail = array.slice(size);
+
+  return [head, ...chunk(tail, size)];
+}
+
+async function processChunk<T extends ProcessingEntry, U extends string[]>(
+  parsedRows: string[][],
+  dependencies: CompleteDependencies<T, U>,
+): Promise<T[]> {
+  const { processRow, headers } = dependencies;
+  // @ts-ignore
+  return new Promise(
+    (
+      resolve: (value: T[] | undefined) => void,
+      reject: (error: Error) => void,
+    ): void => {
+      setTimeout(() => {
+        try {
+          const processedRows = parsedRows.map((row) => processRow(row, headers));
+          resolve(processedRows);
+        } catch (ex) {
+          // @ts-ignore
+          reject(ex);
+        }
+      }, 0);
+    },
+  );
+}
+
+// a row is said to contain headers if two valid header names are present
+export const hasHeaderRow = (
+  rows: string[][],
+  requiredHeaderNames: string[],
+): boolean => {
+  const firstRow = rows[0];
+  if (!firstRow) {
+    return false;
+  }
+  const foundIndex = firstRow.findIndex(
+    (value) => requiredHeaderNames.includes(value.toLowerCase()),
+  );
+  if (foundIndex < 0) {
+    return false;
+  }
+  const containsSecondHeader = firstRow
+    .filter((_val, i) => i !== foundIndex)
+    .some((value) => requiredHeaderNames.includes(value.toLowerCase()));
+  return containsSecondHeader;
+};
+
+export const isHeaderRowValid = (
+  rows: string[][],
+  requiredHeaderNames: string[],
+): boolean => {
+  const firstRow = rows[0];
+  if (!firstRow) {
+    return false;
+  }
+  return (
+    requiredHeaderNames.every(
+      (header) => firstRow.map((value) => value.toLowerCase()).includes(header),
+    ) && requiredHeaderNames.length === firstRow.length
+  );
+};
+
+async function parseFile<T extends ProcessingEntry>(
+  fileContent: string,
+  maxRows: number,
+  requiredHeaderNames: string[],
+  dependencies: Dependencies<T>,
+): Promise<T[]> {
+  if (fileContent === 'invalidFileType') {
+    throw new InvalidFileError();
+  }
+
+  const rows = fileContent.split('\n');
+  const parsedRows = rows.map(parseRow);
+
+  const sanitizedRows = sanitizeRows(parsedRows);
+
+  if (sanitizedRows.length > maxRows) {
+    throw new TooManyRowsError();
+  }
+
+  if (sanitizedRows.length === 0) {
+    throw new NoRowsError();
+  }
+
+  const containsHeaders = hasHeaderRow(sanitizedRows, requiredHeaderNames);
+  if (
+    containsHeaders
+    && !isHeaderRowValid(sanitizedRows, requiredHeaderNames)
+  ) {
+    throw new InvalidHeadersError();
+  }
+
+  const firstSanitizedRow = sanitizedRows[0];
+
+  if (!firstSanitizedRow) {
+    throw new NoRowsError();
+  }
+
+  const headers = containsHeaders
+    ? firstSanitizedRow.map((value) => value.toLowerCase())
+    : requiredHeaderNames;
+
+  const dataRows = containsHeaders ? sanitizedRows.splice(1) : sanitizedRows;
+
+  if (dataRows.length === 0) {
+    throw new NoRowsError();
+  }
+
+  const parsedRowChunks = chunk(dataRows, 30);
+  let results: T[] = [];
+  for (const currentRowChunk of parsedRowChunks) {
+    const chunkResult = await processChunk(currentRowChunk, {
+      ...dependencies,
+      headers,
+    });
+    results = results.concat(chunkResult);
+  }
+
+  return results;
+}
+
+export default parseFile;

package/lib/types/types.js

@@ -1,201 +1,213 @@
 const TC = require('type-check');
+
 const _ = require('lodash');
+
 const E = require('../errors/errors');
+
 const V = require('../validations/validations');
+
 const customTypes = {
-    Integer: {
-        typeOf: 'Number',
-        validate(x) {
-            return x % 1 === 0;
-        },
-    },
-    IntString: {
-        typeOf: 'String',
-        validate: V.isInt,
-    },
-    BoolString: {
-        typeOf: 'String',
-        validate: V.isBoolean,
-    },
-    Hex32: {
-        typeOf: 'String',
-        validate(x) {
-            return V.isHexadecimal(x) && x.length === 32;
-        },
-    },
-    Email: {
-        typeOf: 'String',
-        validate: V.isEmail,
-    },
-    UUID: {
-        typeOf: 'String',
-        validate: V.isUUID,
-    },
-    UUIDv4: {
-        typeOf: 'String',
-        validate(x) {
-            return V.isUUID(x, 4);
-        },
-    },
-    JSON: {
-        typeOf: 'String',
-        validate: V.isJSON,
-    },
-    Url: {
-        typeOf: 'String',
-        validate: V.isURL,
-    },
-    DateString: {
-        typeOf: 'String',
-        validate: V.isDate,
-    },
-    NonEmptyArray: {
-        typeOf: 'Array',
-        validate: V.nonEmpty,
-    },
-    NonNullString: {
-        typeOf: 'String',
-        validate: V.nonEmpty,
-    },
-    CustomerReference: {
-        typeOf: 'String',
-        validate: V.isCustomerReference,
-    },
-    CustomerLabelName: {
-        typeOf: 'String',
-        validate: V.isCustomerLabelName,
-    },
-    BitcoinAddress: {
-        typeOf: 'String',
-        validate: V.bitcoin.isAddress,
-    },
-    EthereumAddress: {
-        typeOf: 'String',
-        validate: V.ethereum.isAddress,
-    },
-    EthereumTx: {
-        typeOf: 'String',
-        validate: V.ethereum.isTxHash,
-    },
-    EthereumBlockHash: {
-        typeOf: 'String',
-        validate: V.ethereum.isBlockHash,
-    },
-    EthereumWeiAmount: {
-        typeOf: 'String',
-        validate: V.ethereum.isValidWeiAmount,
-    },
-    EthereumAddressCode: {
-        typeOf: 'String',
-        validate: V.ethereum.isAddressCode,
-    },
-    MineId: {
-        typeOf: 'String',
-        validate(x) {
-            return x === 'mine';
-        },
-    },
-    BitcoinTxHash: {
-        typeOf: 'String',
-        validate: V.bitcoin.isTxHash,
-    },
-    BitcoinAddressArray: {
-        typeOf: 'Array',
-        validate(as) {
-            return _.every(as, V.bitcoin.isAddress);
-        },
-    },
-    BitcoinTxHashArray: {
-        typeOf: 'Array',
-        validate(as) {
-            return _.every(as, V.bitcoin.isTxHash);
-        },
-    },
-    BitcoinTxHex: {
-        typeOf: 'String',
-        validate: V.bitcoin.isTxHex,
-    },
-    BitcoinScriptHex: {
-        typeOf: 'String',
-        validate: V.bitcoin.isScriptHex,
-    },
-    BitcoinHDPath: {
-        typeOf: 'String',
-        validate: V.bitcoin.isHDPath,
-    },
-    BitcoinPublicKey: {
-        typeOf: 'String',
-        validate: V.bitcoin.isPublicKey,
-    },
-    BitcoinXpub: {
-        typeOf: 'String',
-        validate: V.bitcoin.isHDPublicKey,
-    },
-    BitcoinTxSignature: {
-        typeOf: 'String',
-        validate: V.bitcoin.isTxSignature,
-    },
-    BitcoinBlockHeight: {
-        typeOf: 'Number',
-        validate: V.bitcoin.isBlockHeight,
-    },
+  Integer: {
+    typeOf: 'Number',
+    validate(x) {
+      return x % 1 === 0;
+    },
+  },
+  IntString: {
+    typeOf: 'String',
+    validate: V.isInt,
+  },
+  BoolString: {
+    typeOf: 'String',
+    validate: V.isBoolean,
+  },
+  Hex32: {
+    typeOf: 'String',
+    validate(x) {
+      return V.isHexadecimal(x) && x.length === 32;
+    },
+  },
+  Email: {
+    typeOf: 'String',
+    validate: V.isEmail,
+  },
+  UUID: {
+    typeOf: 'String',
+    validate: V.isUUID,
+  },
+  UUIDv4: {
+    typeOf: 'String',
+    validate(x) {
+      return V.isUUID(x, 4);
+    },
+  },
+  JSON: {
+    typeOf: 'String',
+    validate: V.isJSON,
+  },
+  Url: {
+    typeOf: 'String',
+    validate: V.isURL,
+  },
+  DateString: {
+    typeOf: 'String',
+    validate: V.isDate,
+  },
+  NonEmptyArray: {
+    typeOf: 'Array',
+    validate: V.nonEmpty,
+  },
+  NonNullString: {
+    typeOf: 'String',
+    validate: V.nonEmpty,
+  },
+  CustomerReference: {
+    typeOf: 'String',
+    validate: V.isCustomerReference,
+  },
+  CustomerLabelName: {
+    typeOf: 'String',
+    validate: V.isCustomerLabelName,
+  },
+  BitcoinAddress: {
+    typeOf: 'String',
+    validate: V.bitcoin.isAddress,
+  },
+  EthereumAddress: {
+    typeOf: 'String',
+    validate: V.ethereum.isAddress,
+  },
+  EthereumTx: {
+    typeOf: 'String',
+    validate: V.ethereum.isTxHash,
+  },
+  EthereumBlockHash: {
+    typeOf: 'String',
+    validate: V.ethereum.isBlockHash,
+  },
+  EthereumWeiAmount: {
+    typeOf: 'String',
+    validate: V.ethereum.isValidWeiAmount,
+  },
+  EthereumAddressCode: {
+    typeOf: 'String',
+    validate: V.ethereum.isAddressCode,
+  },
+  MineId: {
+    typeOf: 'String',
+    validate(x) {
+      return x === 'mine';
+    },
+  },
+  BitcoinTxHash: {
+    typeOf: 'String',
+    validate: V.bitcoin.isTxHash,
+  },
+  BitcoinAddressArray: {
+    typeOf: 'Array',
+    validate(as) {
+      return _.every(as, V.bitcoin.isAddress);
+    },
+  },
+  BitcoinTxHashArray: {
+    typeOf: 'Array',
+    validate(as) {
+      return _.every(as, V.bitcoin.isTxHash);
+    },
+  },
+  BitcoinTxHex: {
+    typeOf: 'String',
+    validate: V.bitcoin.isTxHex,
+  },
+  BitcoinScriptHex: {
+    typeOf: 'String',
+    validate: V.bitcoin.isScriptHex,
+  },
+  BitcoinHDPath: {
+    typeOf: 'String',
+    validate: V.bitcoin.isHDPath,
+  },
+  BitcoinPublicKey: {
+    typeOf: 'String',
+    validate: V.bitcoin.isPublicKey,
+  },
+  BitcoinXpub: {
+    typeOf: 'String',
+    validate: V.bitcoin.isHDPublicKey,
+  },
+  BitcoinTxSignature: {
+    typeOf: 'String',
+    validate: V.bitcoin.isTxSignature,
+  },
+  BitcoinBlockHeight: {
+    typeOf: 'Number',
+    validate: V.bitcoin.isBlockHeight,
+  },
 };
+
 // add support for nullable properties
-const withNullable = Object.keys(customTypes).reduce((acc, type) => {
-    acc[type] = customTypes[type];
-    const { typeOf, validate } = customTypes[type];
-    // for every type, add Nullable+type. A nullable string can also be empty
-    acc[`Nullable${type}`] = Object.assign(Object.assign({}, customTypes[type]), { validate: (x) => x == null || (typeOf === 'String' && x === '') || validate(x) });
-    return acc;
+const withNullable = Object.keys(customTypes).reduce((acc, _type) => {
+  acc[_type] = customTypes[_type];
+  const { typeOf, validate } = customTypes[_type];
+
+  // for every type, add Nullable+type. A nullable string can also be empty
+  acc[`Nullable${_type}`] = Object.assign({}, customTypes[_type], {
+    validate: (x) => x == null || (typeOf === 'String' && x === '') || validate(x),
+  });
+
+  return acc;
 }, {});
+
 module.exports = {
-    parseType(str) {
-        return TC.parseType(str);
-    },
-    parsedTypeCheck(type, obj) {
-        return TC.parsedTypeCheck(type, obj, this.opts);
-    },
-    typeCheck(type, obj) {
-        return TC.typeCheck(type, obj, this.opts);
-    },
-    opts: {
-        customTypes: withNullable,
-    },
-    ensureType(type, obj, msg) {
-        return V.ensure(this.typeCheck(type, obj), msg);
-    },
-    checkArg(obj, prop, type, msg) {
-        let message;
-        if (msg == null) {
-            message = `Invalid ${prop}`;
-        }
-        else {
-            message = msg;
-        }
-        this.ensureType(type, obj[prop], message);
-        return obj[prop];
-    },
-    checkArgs(input, types) {
-        const inputKeys = {};
-        const [typ] = types;
-        const fields = typ.of;
-        let numInputKeys = 0;
-        let numKeys;
-        Object.keys(input).forEach((k) => {
-            inputKeys[k] = true;
-            numInputKeys += 1;
-        });
-        numKeys = 0;
-        Object.keys(fields).forEach((key) => {
-            const objTypes = fields[key];
-            V.ensure(this.parsedTypeCheck(objTypes, input[key]), `Invalid ${key}`);
-            if (inputKeys[key]) {
-                numKeys += 1;
-            }
-        });
-        if (!(typ.subset || numInputKeys === numKeys)) {
-            throw new E.BadRequest('invalid extra arguments are present');
-        }
-    },
+  parseType(str) {
+    return TC.parseType(str);
+  },
+  parsedTypeCheck(type, obj) {
+    return TC.parsedTypeCheck(type, obj, this.opts);
+  },
+  typeCheck(type, obj) {
+    return TC.typeCheck(type, obj, this.opts);
+  },
+  opts: {
+    customTypes: withNullable,
+  },
+  ensureType(type, obj, msg) {
+    return V.ensure(this.typeCheck(type, obj), msg);
+  },
+  checkArg(obj, prop, type, msg) {
+    let message;
+    if (msg == null) {
+      message = `Invalid ${prop}`;
+    } else {
+      message = msg;
+    }
+    this.ensureType(type, obj[prop], message);
+    return obj[prop];
+  },
+  checkArgs(input, types) {
+    const inputKeys = {};
+    const [typ] = types;
+    const fields = typ.of;
+    let numInputKeys = 0;
+    let numKeys;
+
+    Object.keys(input).forEach((k) => {
+      inputKeys[k] = true;
+      numInputKeys += 1;
+    });
+
+    numKeys = 0;
+
+    Object.keys(fields).forEach((key) => {
+      const objTypes = fields[key];
+      V.ensure(this.parsedTypeCheck(objTypes, input[key]), `Invalid ${key}`);
+      if (inputKeys[key]) {
+        numKeys += 1;
+      }
+    });
+
+    if (!(typ.subset || numInputKeys === numKeys)) {
+      throw new E.BadRequest('invalid extra arguments are present');
+    }
+  },
 };
-//# sourceMappingURL=types.js.map