@loadmill/core 0.3.50 → 0.3.51

package/src/multipart-form-data/multipart-text-to-post-form-data.ts

@@ -1,26 +1,25 @@
+import { extension } from 'mime-types';
+import { isBase64 } from 'validator';
+
+import { HarParam } from '../har';
 import { PostFormData, PostFormDataEntry } from '../request';
+import { isBinaryFile } from './is-binary-file';
 
-export const multipartFormDataTextToPostFormData = (multipartFormDataText: string): PostFormData => {
-  const result: PostFormData = [];
+export const multipartFormDataTextToPostFormData = (multipartFormDataText: string = ''): PostFormData =>
+  getLines(multipartFormDataText)
+    .map(line => extractPostFormDataEntry(line));
 
-  const lines = getLines(multipartFormDataText);
-  for (const line of lines) {
-    const postFormDataEntry: PostFormDataEntry = extractPostFormDataEntry(line);
-    result.push(postFormDataEntry);
-  }
-  return result;
-};
-
-const getLines = (multipartFormDataText: string) => {
-  const boundary = getBoundary(multipartFormDataText);
-  const boundaryRegex = splitByBoundaryRegex(boundary);
-  const linesSplittedByBoundary = multipartFormDataText.split(boundaryRegex);
-  return removeEmptyLines(linesSplittedByBoundary);
-};
+const getLines = (multipartFormDataText: string = ''): string[] =>
+  removeEmptyLines(
+    multipartFormDataText.split(
+      splitByBoundaryRegex(
+        getBoundary(multipartFormDataText)
+      )
+    )
+  );
 
-const getBoundary = (multipartFormDataText: string): string => {
-  return multipartFormDataText.substring(0, multipartFormDataText.indexOf('\r\n'));
-};
+const getBoundary = (multipartFormDataText: string): string =>
+  multipartFormDataText.substring(0, multipartFormDataText.indexOf('\r\n'));
 
 const splitByBoundaryRegex = (boundary: string) => {
   const normalBoundary = boundary + '\r\n';
@@ -28,7 +27,7 @@ const splitByBoundaryRegex = (boundary: string) => {
   return new RegExp(`${normalBoundary}|${lastBoundary}`);
 };
 
-const removeEmptyLines = (lines: string[]): string[] => lines.filter(Boolean);
+const removeEmptyLines = (lines: string[] = []): string[] => lines.filter(Boolean);
 
 const extractPostFormDataEntry = (line: string): PostFormDataEntry => {
   const indexOfFirst2Newlines = line.indexOf('\r\n\r\n');
@@ -38,52 +37,113 @@ const extractPostFormDataEntry = (line: string): PostFormDataEntry => {
 
   const postFormDataEntry: PostFormDataEntry = { name: fields.name, value };
   fields.fileName && (postFormDataEntry.fileName = fields.fileName);
+  fields.contentType && (postFormDataEntry.contentType = fields.contentType);
 
   return postFormDataEntry;
 };
 
-type Fields = Pick<PostFormDataEntry, 'name' | 'fileName'>;
+type Fields = HarParam;
 
 const extractFields = (line: string, indexOfFirst2Newlines: number): Fields => {
   const rawFields = extractRawFields(line, indexOfFirst2Newlines);
   const res: Fields = { name: '' };
-  const onlyRawNameFields = rawFields.split('\r\n')[0];
-  const tokens = onlyRawNameFields.split(';').map(t => t.trim());
-  for (const token of tokens) {
-    const [k, v] = token.split('=');
-    if (k === 'name') {
-      res.name = removeDoubleQuotes(v);
-    } else if (k === 'filename') {
-      res.fileName = removeDoubleQuotes(v);
-    }
-  }
+  const fieldsLines = rawFields.split('\r\n');
+  addNameFields(fieldsLines[0], res);
+  fieldsLines.length > 1 && addContentType(fieldsLines[1], res);
   return res;
 };
 
-const extractRawFields = (line: string, indexOfFieldsValueSeperator: number): string => {
-  return line.substring(0, indexOfFieldsValueSeperator);
+const extractRawFields = (line: string, indexOfFieldsValueSeperator: number): string =>
+  line.substring(0, indexOfFieldsValueSeperator);
+
+const removeDoubleQuotes = (v: string): string =>
+  v.replace(/"/g, '');
+
+const calculateValue = ({ fileName, contentType }: Fields, line: string, indexOfFirst2Newlines: number): string | '' => {
+  const value = extractValue(line, indexOfFirst2Newlines);
+  return getBinaryOrTextValue({ value, fileName, contentType });
+};
+
+export const getBinaryOrTextValue = (fields: Omit<Fields, 'name'>): string => {
+  const { value = '', fileName } = fields;
+  if (fileName) {
+    const isBass64 = isBase64(value);
+    const toBinary = isToBinary(fields);
+    if (isBass64 || toBinary) {
+      return myAtob(value, isBass64, toBinary);
+    }
+  }
+  return value;
 };
 
-const removeDoubleQuotes = (v: string): string => {
-  return v.replace(/"/g, '');
+const myAtob = (target = '', fromBase64?: boolean, toBinary?: boolean) => {
+  const defaultEncoding = 'utf8';
+  const encodingFrom = fromBase64 ? 'base64' : defaultEncoding;
+  const encodingTo = toBinary ? 'binary' : defaultEncoding;
+  return Buffer.from(target, encodingFrom).toString(encodingTo);
 };
 
-function calculateValue(fields: Fields, line: string, indexOfFirst2Newlines: number): string | '' {
-  return fields.fileName ? '' : extractValue(line, indexOfFirst2Newlines);
-}
+const isToBinary = (fields: Omit<Fields, 'name'>): boolean => {
+  const { value = '', fileName, contentType } = fields;
+  if (isAlreadyBinary(value)) {
+    return false;
+  }
+  if (fileName) {
+    const ext = getExtension(fileName);
+    if (ext) {
+      return isBinaryFileExtension(ext);
+    }
+  }
+  if (contentType) {
+    return isBinaryContentType(contentType);
+  }
+  return false;
+};
 
-const extractValue = (line: string, indexOfFieldsValueSeperator: number): string => {
-  const value = line.substring(indexOfFieldsValueSeperator + 4);
-  return removeEndingNewline(value);
+const isAlreadyBinary = (target = '') => {
+  const bytes = Buffer.from(target);
+  const size = target.length;
+  return isBinaryFile(bytes, size);
 };
 
-const removeEndingNewline = (s: string): string => {
-  if (s.endsWith('\r\n')) {
-    s = removeLast2Chars(s);
+const getExtension = (fileName: string): string | undefined =>
+  fileName.split('.').pop();
+
+const isBinaryFileExtension = (ext: string): boolean =>
+  ['docx', 'pdf', 'doc', 'jpg', 'jpeg', 'png'].includes(ext);
+
+const isBinaryContentType = (contentType: string): boolean =>
+  ['docx', 'pdf', 'bin'].includes(extension(contentType) || '');
+
+const extractValue = (line: string, indexOfFieldsValueSeperator: number): string =>
+  removeEndingNewline(
+    line.substring(indexOfFieldsValueSeperator + 4)
+  );
+
+const removeEndingNewline = (s: string): string =>
+  s.endsWith('\r\n') ? removeLast2Chars(s) : s;
+
+const removeLast2Chars = (s: string): string =>
+  s.substring(0, s.length - 2);
+
+const addNameFields = (nameFieldsLine: string, res: HarParam) => {
+  const tokens = extractTokens(nameFieldsLine);
+  for (const token of tokens) {
+    const [k, v] = token.split('=');
+    if (k === 'name') {
+      res.name = removeDoubleQuotes(v);
+    } else if (k === 'filename') {
+      res.fileName = removeDoubleQuotes(v);
+    }
   }
-  return s;
 };
 
-const removeLast2Chars = (s: string) => {
-  return s.substring(0, s.length - 2);
+const extractTokens = (nameFieldsLine: string): string[] =>
+  nameFieldsLine.split(';').map(t => t.trim());
+
+const addContentType = (contentTypeLine: string, res: HarParam) => {
+  const [k, v] = contentTypeLine.split(': ');
+  if (k === 'content-type') {
+    res.contentType = v;
+  }
 };

package/src/parameters/extractions.ts

@@ -17,14 +17,7 @@ const { getParameterName } = parameterUtils;
  * @param ext The extraction object to take a part
  * @returns The spreaded parts of the given extraction object
  */
-export function getExtractionParts(ext: Extractions): {
-  name: string,
-  value: string | {
-    query: string;
-    attr?: string;
-  }
-  type?: string,
-} {
+export function getExtractionParts(ext: Extractions): ExtractionParts {
   const name = getParameterName(ext);
   const extContent = ext[name];
   if (typeof extContent === 'string') {
@@ -35,6 +28,15 @@ export function getExtractionParts(ext: Extractions): {
   return { name, value, type };
 }
 
+export type ExtractionParts = {
+  name: string;
+  value: string | {
+    query: string;
+    attr?: string;
+  };
+  type?: string;
+};
+
 export function requestsExtractionsArr(requests, toIndex: number) {
   const extractionParamNames = [] as string[];
   for (let i = 0; i <= toIndex; i++) {

package/src/parameters/index.ts

@@ -177,6 +177,9 @@ export const parameterUtils = {
   },
 
   findIndex(key, arr) {
+    if (!arr) {
+      return -1;
+    }
     return arr.findIndex((obj) => {
       return Object.keys(obj)[0] === key;
     });
@@ -244,7 +247,7 @@ export const parameterUtils = {
     return this.getUsedConfParams({ requests: [request] }, parameters);
   },
 
-  getValueByKeyFromArr(key: string, parameters: Parameters[], returnArray: boolean = false) {
+  getValueByKeyFromArr(key: string, parameters: Parameters[] | undefined = [], returnArray: boolean = false) {
     let res;
     const param = getParameterByKey(key, parameters);
     if (param) {

package/src/request/index.ts

@@ -398,6 +398,7 @@ export type PostFormDataEntry = {
   name: string;
   value: string;
   fileName?: string;
+  contentType?: string;
 };
 
 export type RequestRawFileData = {

package/test/multipart-form-data/form-data-utils.spec.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-len */
 const fs = require('fs');
 const path = require('path');
 import { suite, describe, it } from 'mocha';
@@ -34,7 +35,6 @@ suite('form-data-utils - multipart form data', () => {
   describe('Form Class - manages and manipulates multipart form data', () => {
     it('to string', () => {
       const boundary = '------WebKitFormBoundary12345678ASDFGHJ';
-      // eslint-disable-next-line max-len
       const expectedText = '--------WebKitFormBoundary12345678ASDFGHJ\r\nContent-Disposition: form-data; name="the_name_of_the_thing"; filename="some file name.ext"\r\ncontent-type: application/vnd.mspowerpoint-sniffing.x-rar-compressed\r\n\r\nthe_contents_of_the_file\r\n--------WebKitFormBoundary12345678ASDFGHJ\r\nContent-Disposition: form-data; name="entityId"\r\n\r\n123456789123456789123456789\r\n--------WebKitFormBoundary12345678ASDFGHJ\r\nContent-Disposition: form-data; name="shouldDoStuff"\r\n\r\ntrue\r\n--------WebKitFormBoundary12345678ASDFGHJ\r\nContent-Disposition: form-data; name="content-transfer-encoding"\r\n\r\nquoted-printable\r\n--------WebKitFormBoundary12345678ASDFGHJ--\r\n';
       const form = new Form(params, boundary);
       const result = form.toString();
@@ -49,7 +49,7 @@ suite('form-data-utils - multipart form data', () => {
         postFormData: [
           {
             name: 'the_name_of_the_thing',
-            value: '',
+            value: 'the_contents_of_the_file',
             fileName: 'some file name.ext',
           },
           {
@@ -75,7 +75,7 @@ suite('form-data-utils - multipart form data', () => {
         postFormData: [
           {
             name: 'file',
-            value: '',
+            value: 'ÿØÿà\u0000\u0010JFIF\u0000\u0001\u0001\u0000\u0000\u0001\u0000\u0001\u0000\u0000ÿÛ\u0000„\u0000\u0005\u0003\u0004\t\t\b\t\t\t\t\t\t\t\t\t\u0005u001c\u0017\t\b\u001a\t\t\u0006\u0018!\u0018\u001a\u001d\u001d\u001f\u001f\u001f\u0007\u000b"\u0018"\u001e\u0018\u001c\u001e\u001f\u001e\u0001\u0005\u0005\u0005\b\u0007\b\u000e\b\b\r\u0012\r\u000e\r\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012\u0012R#B?\u00005Ù',
             fileName: 'silk_sonic.jpeg',
           },
         ]
@@ -98,24 +98,45 @@ suite('form-data-utils - multipart form data', () => {
     });
 
     it('should return 2 name-values', () => {
-      // eslint-disable-next-line max-len
       const text = '----10453916712\r\nContent-Disposition: form-data; name="arnon"\r\n\r\npeleg\r\n----10453916712\r\nContent-Disposition: form-data; name="rivi"\r\n\r\nshimi\r\n----10453916712--\r\n';
       const result = multipartFormDataTextToPostFormData(text);
       expect(result).to.have.deep.ordered.members([{ name: 'arnon', value: 'peleg' }, { name: 'rivi', value: 'shimi' }]);
     });
 
     it('should return name-value-filename', () => {
-      // eslint-disable-next-line max-len
       const text = '-----------------------------10453916712809342873442003634\r\nContent-Disposition: form-data; name="file"; filename="silk_sonic.jpeg"\r\nContent-Type: image/jpeg\r\n\r\nXCXCXCXCXCXCXCXCXCXCXCXCXCXC\t\r\n\t \n\n\nXXC101010101010101101010101010\r\n-----------------------------10453916712809342873442003634--\r\n';
       const result = multipartFormDataTextToPostFormData(text);
       expect(result).to.have.deep.ordered.members([
         {
           name: 'file',
-          value: '',
+          value: 'XCXCXCXCXCXCXCXCXCXCXCXCXCXC\t\r\n\t \n\n\nXXC101010101010101101010101010',
           fileName: 'silk_sonic.jpeg'
         }
       ]);
     });
-  });
 
+    it('should return also the value of a simple file', () => {
+      const text = '------WebKitFormBoundaryntAaM2MokVD4YVOO\r\nContent-Disposition: form-data; name="file"; filename="Cars.csv"\r\nContent-Type: text/csv\r\n\r\nID,First_Name,Last_Name,Car_Make,Car_Model,Car_Color,Car_Model Year,Car_VIN\n1,Lenora,Hegdonne,Lincoln,Town Car,Blue,1994,WAUUL98E68A703943\n2,Jermaine,Bradane,Ford,GT,Pink,2005,1N6AA0EC1FN193806\r\n------WebKitFormBoundaryntAaM2MokVD4YVOO--\r\n';
+      const result = multipartFormDataTextToPostFormData(text);
+      expect(result).to.have.deep.ordered.members([
+        {
+          name: 'file',
+          value: 'ID,First_Name,Last_Name,Car_Make,Car_Model,Car_Color,Car_Model Year,Car_VIN\n1,Lenora,Hegdonne,Lincoln,Town Car,Blue,1994,WAUUL98E68A703943\n2,Jermaine,Bradane,Ford,GT,Pink,2005,1N6AA0EC1FN193806',
+          fileName: 'Cars.csv'
+        }
+      ]);
+    });
+    it('should detect content-type', () => {
+      const text = '----------------------------148649485734387205386089\r\nContent-Disposition: form-data; name="file"; filename="Bell data engineer.docx"\r\ncontent-type: application/vnd.openxmlformats-officedocument.wordprocessingml.document\r\n\r\nUEsDBBQABgAIAAAAIQCTkoLJhQEAACkHAAATAAgCW0NvbnRlbnRfVHlwZXNdLnhtbCCiBAIo\r\n----------------------------148649485734387205386089--\r\n';
+      const result = multipartFormDataTextToPostFormData(text);
+      expect(result).to.have.deep.ordered.members([
+        {
+          name: 'file',
+          value: 'PK\u0003\u0004\u0014\u0000\u0006\u0000\b\u0000\u0000\u0000!\u0000“’‚É…\u0001\u0000\u0000)\u0007\u0000\u0000\u0013\u0000\b\u0002[Content_Types].xml ¢\u0004\u0002(',
+          fileName: 'Bell data engineer.docx',
+          contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        }
+      ]);
+    });
+  });
 });