convert-csv-to-json 1.4.0 → 2.0.0

package/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"

package/.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

package/.github/workflows/nodejs.yml

@@ -9,12 +9,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [12.x, 14.x, 16.x, 18.x]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
         node-version: ${{ matrix.node-version }}
     - name: npm install, build, and test

package/.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

package/.idea/csv-plugin.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CsvFileAttributes">
+    <option name="attributeMap">
+      <map>
+        <entry key="/test/resource/input.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value=";" />
+            </Attribute>
+          </value>
+        </entry>
+        <entry key="/test/resource/input_example.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value=";" />
+            </Attribute>
+          </value>
+        </entry>
+        <entry key="/test/resource/input_example_sub_array.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value=";" />
+            </Attribute>
+          </value>
+        </entry>
+        <entry key="/test/resource/input_quoted_fields.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value="," />
+            </Attribute>
+          </value>
+        </entry>
+        <entry key="/test/resource/input_quoted_fields_with_subarray.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value="," />
+            </Attribute>
+          </value>
+        </entry>
+        <entry key="/test/resource/input_tilde_delimiter.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value="," />
+            </Attribute>
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+</project>

package/.idea/csvToJson.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

package/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

package/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/csvToJson.iml" filepath="$PROJECT_DIR$/.idea/csvToJson.iml" />
+    </modules>
+  </component>
+</project>

package/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

package/README.md

@@ -1,31 +1,40 @@
 # CSVtoJSON 
-![NodeJS Version](https://img.shields.io/badge/nodeJS-%3E=12.x-brightgreen.svg)
-[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/iuccio/csvToJson/Node%20CI?label=Build)](https://github.com/iuccio/csvToJson/actions?query=workflow%3A%22Node+CI%22)
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/iuccio/csvToJson/CodeQL/master?label=CodeQL)](https://github.com/iuccio/csvToJson/actions?query=workflow%3ACodeQL)
-[![Code Climate](https://codeclimate.com/github/iuccio/csvToJson/badges/gpa.svg)](https://codeclimate.com/github/iuccio/csvToJson)
+![Build](https://github.com/iuccio/csvToJson/actions/workflows/nodejs.yml/badge.svg)
+![CodeQL](https://github.com/iuccio/csvToJson/actions/workflows/codeql-analysis.yml/badge.svg)
 [![Known Vulnerabilities](https://snyk.io/test/github/iuccio/csvToJson/badge.svg)](https://snyk.io/test/github/iuccio/csvToJson)
+[![Code Climate](https://codeclimate.com/github/iuccio/csvToJson/badges/gpa.svg)](https://codeclimate.com/github/iuccio/csvToJson)
 [![NPM Version](https://img.shields.io/npm/v/convert-csv-to-json.svg)](https://npmjs.org/package/convert-csv-to-json)
+![NodeJS Version](https://img.shields.io/badge/nodeJS-%3E=12.x-brightgreen.svg)
 [![Downloads](https://img.shields.io/npm/dm/convert-csv-to-json.svg)](https://npmjs.org/package/convert-csv-to-json)
 
 **This project is not dependent on others packages or libraries.**
 
+
 ## Table of Contents
-1. [Description](#description)
-1. [Prerequisites](#prerequisites)
-1. [Install npm *convert-csv-to-json package*](#install-npm-convert-csv-to-json-package)
-1. [Usage](#usage)
-    * [Generate JSON file](#generate-json-file)
-    * [Generate Array of Object in JSON format](#generate-array-of-object-in-json-format)
-    * [Generate Object with sub array](#generate-object-with-sub-array)
-    * [Define field delimiter](#define-field-delimiter)
-    * [Format property value by type](#format-property-value-by-type)
-        * [Number](#number)
-        * [Boolean](#boolean)
-    * [Encoding](#encoding)
-1. [Development](#development)
-1. [License](#license)
-1. [Buy me a Coffee](#buy-me-a-coffee)
 
+<!-- toc -->
+
+- [Description](#description)
+- [Prerequisites](#prerequisites)
+- [Install npm *convert-csv-to-json package*](#install-npm-convert-csv-to-json-package)
+  * [Install](#install)
+  * [Usage](#usage)
+    + [Generate JSON file](#generate-json-file)
+    + [Generate Array of Object in JSON format](#generate-array-of-object-in-json-format)
+    + [Generate Object with sub array](#generate-object-with-sub-array)
+    + [Define field delimiter](#define-field-delimiter)
+    + [Support Quoted Fields](#support-quoted-fields)
+    + [Index header](#index-header)
+    + [Empty rows](#empty-rows)
+    + [Format property value by type](#format-property-value-by-type)
+      - [Number](#number)
+      - [Boolean](#boolean)
+    + [Encoding](#encoding)
+- [Development](#development)
+- [License](#license)
+- [Buy me a Coffee](#buy-me-a-coffee)
+
+<!-- tocstop -->
 
 ## Description
 Converts *csv* files to *JSON* files with Node.js. 
@@ -142,6 +151,40 @@ E.g. if your field delimiter is the comma **,** then:
  csvToJson.fieldDelimiter(',').getJsonFromCsv(fileInputName);
 ```
 
+#### Support Quoted Fields
+To be able to parse correctly fields wrapped in quote, like the **last_name** in the first row in the following example:
+
+|first_name|         last_name          |email|
+|:----------:|:--------------------------:|:---:|
+|Constantin| "Langsdon,Nandson,Gangson" |clangsdon0@hc360.com|
+
+you need to activate the support quoted fields feature: 
+
+```js
+ csvToJson.supportQuotedField(true).getJsonFromCsv(fileInputName);
+```
+
+The result will be:
+```json
+[
+  {
+      "firstName": "Constantin",
+      "lastName": "Langsdon,Nandson,Gangson",
+      "email": "clangsdon0@hc360.com"
+    }
+]
+``` 
+
+#### Index header
+If the header is not on the first line you can define the header index like: 
+
+```js
+ csvToJson.indexHeader(3).getJsonFromCsv(fileInputName);
+```
+
+#### Empty rows
+Empty rows are ignored and not parsed.
+
 #### Format property value by type
 If you want that a number will be printed as a Number type, and values *true* or *false* is printed as a boolean Type, use:
 ```js

package/SECURITY.md

@@ -14,9 +14,18 @@ A generated Github Depedabot PR is normally quickly merged and a new version is
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 1.4.0   | :white_check_mark: |
 | 1.3.3   | :white_check_mark: |
 | 1.3.2   | :white_check_mark: |      
 
+### Dev Dependencies
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 1.4.0   | :white_check_mark: |
+| 1.3.3   | :warning: moderate dev dependencies vulnerability|
+| 1.3.2   | :warning: moderate dev dependencies vulnerability|  
+
 ## Reporting a Vulnerability
 
 To report a Security Vulnerability please add an [Vulnerabilty Issue](https://github.com/iuccio/csvToJson/labels/vulnerabilty).  

package/index.js

@@ -20,6 +20,13 @@ exports.formatValueByType = function (active = true) {
   return this;
 };
 
+/**
+ *
+ */
+exports.supportQuotedField = function (active = false) {
+  csvToJson.supportQuotedField(active);
+  return this;
+};
 /**
  * Defines the field delimiter which will be used to split the fields
  */
@@ -28,6 +35,14 @@ exports.fieldDelimiter = function (delimiter) {
   return this;
 };
 
+/**
+ * Defines the index where the header is defined
+ */
+exports.indexHeader = function (index) {
+  csvToJson.indexHeader(index);
+  return this;
+};
+
 /**
  * Defines how to match and parse a sub array
  */

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "convert-csv-to-json",
-  "version": "1.4.0",
+  "version": "2.0.0",
   "description": "Convert CSV to JSON",
   "main": "index.js",
   "scripts": {
-    "test": "./node_modules/.bin/mocha --reporter spec",
+    "test": "./node_modules/.bin/mocha --parallel --reporter spec",
     "test-watch": "./node_modules/.bin/mocha -w"
   },
   "repository": {
@@ -37,6 +37,6 @@
   "devDependencies": {
     "chai": "^4.3.7",
     "chai-fs": "^2.0.0",
-    "mocha": "^10.1.0"
+    "mocha": "^10.2.0"
   }
 }

package/src/csvToJson.js

@@ -14,11 +14,25 @@ class CsvToJson {
     return this;
   }
 
-  fieldDelimiter(delimieter) {
-    this.delimiter = delimieter;
+  supportQuotedField(active) {
+    this.isSupportQuotedField = active;
     return this;
   }
 
+  fieldDelimiter(delimiter) {
+    this.delimiter = delimiter;
+    return this;
+  }
+
+  indexHeader(indexHeader) {
+    if(isNaN(indexHeader)){
+        throw new Error('The index Header must be a Number!');
+    }
+    this.indexHeader = indexHeader;
+    return this;
+  }
+
+
   parseSubArray(delimiter = '*',separator = ',') {
     this.parseSubArrayDelimiter = delimiter;
     this.parseSubArraySeparator = separator;
@@ -51,17 +65,30 @@ class CsvToJson {
   }
 
   csvToJson(parsedCsv) {
+  	this.validateInputConfig();
     let lines = parsedCsv.split(newLine);
     let fieldDelimiter = this.getFieldDelimiter();
-    let headers = lines[0].split(fieldDelimiter);
+    let index = this.getIndexHeader();
+    let headers = lines[index].split(fieldDelimiter);
 
-    let jsonResult = [];
-    for (let i = 1; i < lines.length; i++) {
-      let currentLine = lines[i].split(fieldDelimiter);
-      if (stringUtils.hasContent(currentLine)) {
-        jsonResult.push(this.buildJsonResult(headers, currentLine));
-      }
+    while(!stringUtils.hasContent(headers) && index <= lines.length){
+        index = index + 1;
+        headers = lines[index].split(fieldDelimiter);
     }
+
+    let jsonResult = [];
+    for (let i = (index + 1); i < lines.length; i++) {
+        let currentLine;
+        if(this.isSupportQuotedField){
+            currentLine = this.split(lines[i]);
+        }
+        else{
+            currentLine = lines[i].split(fieldDelimiter);
+        }
+        if (stringUtils.hasContent(currentLine)) {
+            jsonResult.push(this.buildJsonResult(headers, currentLine));
+        }
+       }
     return jsonResult;
   }
 
@@ -72,6 +99,13 @@ class CsvToJson {
     return defaultFieldDelimiter;
   }
 
+  getIndexHeader(){
+    if(this.indexHeader !== null && !isNaN(this.indexHeader)){
+        return this.indexHeader;
+    }
+    return 0;
+  }
+
   buildJsonResult(headers, currentLine) {
     let jsonObject = {};
     for (let j = 0; j < headers.length; j++) {
@@ -115,6 +149,76 @@ class CsvToJson {
     return false;
   }
 
+  validateInputConfig(){
+  	if(this.isSupportQuotedField) {
+  	 	if(this.getFieldDelimiter() === '"'){
+  	 		throw new Error('When SupportQuotedFields is enabled you cannot defined the field delimiter as quote -> ["]');
+  	 	}
+  	 	if(this.parseSubArraySeparator === '"'){
+  	 		throw new Error('When SupportQuotedFields is enabled you cannot defined the field parseSubArraySeparator as quote -> ["]');
+  	 	}
+  	 	if(this.parseSubArrayDelimiter === '"'){
+  	 		throw new Error('When SupportQuotedFields is enabled you cannot defined the field parseSubArrayDelimiter as quote -> ["]');
+  	 	}
+  	}
+  }
+
+  hasQuotes(line) {
+    return line.includes('"');
+  }
+
+  split(line) {
+    if(line.length == 0){
+      return [];
+    }
+    let delim = this.getFieldDelimiter();
+    let subSplits = [''];
+    if (this.hasQuotes(line)) {
+        let chars = line.split('');
+
+        let subIndex = 0;
+        let startQuote = false;
+        let isDouble = false;
+        chars.forEach((c, i, arr) => {
+            if (isDouble) { //when run into double just pop it into current and move on
+                subSplits[subIndex] += c;
+                isDouble = false;
+                return;
+            }
+
+            if (c != '"' && c != delim ) {
+                subSplits[subIndex] += c;
+            } else if(c == delim && startQuote){
+                subSplits[subIndex] += c;
+            } else if( c == delim ){
+                subIndex++
+                subSplits[subIndex] = '';
+                return;
+            } else {
+                if (arr[i + 1] === '"') {
+                    //Double quote
+                    isDouble = true;
+                    //subSplits[subIndex] += c; //Skip because this is escaped quote
+                } else {
+                    if (!startQuote) {
+                        startQuote = true;
+                        //subSplits[subIndex] += c; //Skip because we don't want quotes wrapping value
+                    } else {
+                        //end
+                        startQuote = false;
+                        //subSplits[subIndex] += c; //Skip because we don't want quotes wrapping value
+                    }
+                }
+            }
+        });
+        if(startQuote){
+            throw new Error('Row contains mismatched quotes!');
+        }
+        return subSplits;
+    } else {
+        return line.split(delim);
+    }
+  }
 }
 
 module.exports = new CsvToJson();