csv-import-analyzer 0.0.1

data/lib/csv-import-analyzer/sampleTab.csv

@@ -0,0 +1,5 @@
+Year ID,Make ID,Model ID,Description ID,Price ID
+1997,Ford,,"ac, abs, moon","3000"
+1999,Chevy,"Venture ""Extended Edition""","",4900.00
+1999,"Chevy","Venture ""Extended Edition, Very Large""",,5000.00
+1996,Jeep,Grand Che'rokee,"MUST SELL!air, moon roof, loaded",4799.00

data/lib/csv-import-analyzer/sql_query_builder.rb

@@ -0,0 +1,125 @@
+require 'pry'
+require_relative "query_builder/mysql_query_helper"
+require_relative "query_builder/pg_query_helper"
+require_relative "export/metadata_analysis"
+module CsvImportAnalyzer
+  class SqlQueryBuilder
+    # include CsvImportAnalyzer::mysql_query_helper
+    attr_accessor :create_query, :import_query, :csv_column_datatypes, :min_max_bounds, :nullable, :sql_helper_options
+
+    # Since Building SQL is dependent on multiple things,
+    # decided to go with an arguments hash that gets passed when creating an object for the class
+    def initialize(args)
+      @options = args
+      @create_query = {}
+      @import_query = {}
+      @csv_column_datatypes = args[:csv_column_datatypes]
+      @nullable = args[:nullable]
+      @sql_helper_options = {:tablename => tablename, :filename => @options[:filename], :delimiter => @options[:delimiter]}
+      @mysql_helper.extend(CsvImportAnalyzer::MysqlQueryHelper)
+      @pg_helper.extend(CsvImportAnalyzer::PgQueryHelper)
+    end
+
+    def options
+      @options
+    end
+
+    def databases
+      options[:database]
+    end
+
+    def filename
+      return options[:filename]
+    end
+
+    def tablename
+      # May be optimize this, not run all three operations everytime filename method is called
+      # May be creating filename as instance variable and using a double pipe will relive it from running everytime doesn't it?
+      tablename = File.basename(options[:filename])
+      tablename.gsub!(" ", "_")
+      tablename.downcase!
+      return tablename
+    end
+
+    def delimiter
+      options[:delimiter]
+    end
+
+    def mysql_helper
+      @mysql_helper
+    end
+
+    def pg_helper
+      @pg_helper
+    end
+
+    def generate_query
+      databases.each do |db|
+        create_query[db] = ["create table #{tablename} ("]
+      end
+      csv_column_datatypes.each do |header, datatype|
+        append_to_query = build_query_for_datatype(header, datatype)
+        append_to_query.each do |key, value|
+          create_query[key].push(value)
+        end
+      end
+      prepare_sql_statements
+      prepare_import_csv
+      print_metadata_analysis
+    end
+
+    private
+
+    def build_query_for_datatype(header, datatype)
+      query = {}
+      databases.each do |db|
+        if db == :mysql
+          query[db] = mysql_helper.form_query_for_datatype(header: header, datatype: datatype)
+        else
+          query[db] = pg_helper.form_query_for_datatype(header: header, datatype: datatype)
+        end
+      end
+      unless nullable.include?(header)
+        query.keys.each do |db|
+          query[db] << " not null"
+        end
+      end
+      return query
+    end
+
+    def prepare_import_csv
+      databases.each do |db|
+        if db == :mysql
+          import_query[db] = mysql_helper.import_csv(tablename: tablename, filename: filename, delimiter: delimiter)
+        elsif db == :pg
+          import_query[db] = pg_helper.import_csv(tablename: tablename, filename: filename, delimiter: delimiter)
+        end
+      end
+    end
+
+    def prepare_sql_statements
+      databases.each do |db|
+        create_query[db][0] = create_query[db].first + " " + create_query[db][1]
+        create_query[db].delete_at(1)
+        create_query[db] = create_query[db].join(", ")
+        create_query[db] << ");"
+      end
+    end
+
+    def print_metadata_analysis
+      options[:create_query] = create_query
+      options[:import_query] = import_query
+      export = CsvImportAnalyzer::MetadataAnalysis.new(options)
+      export.metadata_print
+    end
+
+  end
+end
+
+#Testing
+# args = {}
+# args[:options] = {:delimiter => ",", :chunk => 20, :filename => "/home/avinash/Desktop/csv-import-analyzer/lib/csv-import-analyzer/sampleTab"}
+# args[:column_datatypes] = {:year_id=>:int, :make_id=>:string, :model_id=>:string, :description_id=>:string, :price_id=>:float}
+# args[:nullable] = [:description_id]
+# query = CsvImportAnalyzer::SqlQueryBuilder.new(args)
+# puts query.generate_query

data/lib/csv-import-analyzer/version.rb

@@ -0,0 +1,5 @@
+module CsvImportAnalyzer
+  module Version
+    VERSION = "0.0.1"
+  end
+end

data/lib/metadata_output.json

@@ -0,0 +1,70 @@
+{
+  "csv_file": {
+    "filename": "sampleTab.csv",
+    "file_size": 276,
+    "record_delimiter": ","
+  },
+  "data_manipulations": {
+    "replace_nulls": true,
+    "replace_quotes": true
+  },
+  "csv_headers": {
+    "year_id": {
+      "datatype": "int",
+      "datatype_analysis": {
+        "int": 4
+      },
+      "distinct_values": [
+        1997,
+        1999,
+        1996
+      ]
+    },
+    "make_id": {
+      "datatype": "string",
+      "datatype_analysis": {
+        "string": 4
+      },
+      "distinct_values": [
+        "Ford",
+        "Chevy",
+        "Jeep"
+      ]
+    },
+    "model_id": {
+      "datatype": "string",
+      "datatype_analysis": {
+        "string": 4
+      },
+      "distinct_values": "3+"
+    },
+    "description_id": {
+      "datatype": "string",
+      "datatype_analysis": {
+        "string": 2
+      },
+      "distinct_values": [
+        "ac, abs, moon",
+        "MUST SELL!air, moon roof, loaded"
+      ],
+      "nullable": true
+    },
+    "price_id": {
+      "datatype": "float",
+      "datatype_analysis": {
+        "float": 4
+      },
+      "distinct_values": "3+"
+    }
+  },
+  "sql": {
+    "pg": {
+      "create_query": "create table sampletab.csv ( year_id int not null, make_id varchar(255) not null, model_id varchar(255) not null, description_id varchar(255), price_id float not null);",
+      "import_query": "COPY sampletab.csv FROM '/home/avinash/Desktop/csv-import-analyzer/lib/sampleTab.csv' HEADER DELIMITER ',' CSV NULL AS 'NULL';"
+    },
+    "mysql": {
+      "create_query": "create table sampletab.csv ( year_id int not null, make_id varchar(255) not null, model_id varchar(255) not null, description_id varchar(255), price_id float not null);",
+      "import_query": "COPY sampletab.csv FROM '/home/avinash/Desktop/csv-import-analyzer/lib/sampleTab.csv' HEADER DELIMITER ',' CSV NULL AS 'NULL';"
+    }
+  }
+}

data/lib/sampleTab.csv

@@ -0,0 +1,5 @@
+Year ID,Make ID,Model ID,Description ID,Price ID
+1997,Ford,E350,"ac, abs, moon","3000.00"
+1999,Chevy,"Venture ""Extended Edition""",,4900.00
+1999,"Chevy","Venture ""Extended Edition, Very Large""","",5000.00
+1996,Jeep,Grand Che'rokee,"MUST SELL!air, moon roof, loaded",4799.00

data/spec/csv-import-analyzer/analyzer/csv_check_bounds_spec.rb

@@ -0,0 +1,43 @@
+#require spec_helper.rb
+
+describe CsvImportAnalyzer::CsvCheckBounds do
+
+  describe '#get_min_max_values' do
+    context 'when not initialized right' do
+      let(:options) {Hash[filename: "sample", chunk_size: 200, delimiter: ",", unique: 2]}
+      
+      it 'will fail gracefully if filename is nil' do
+        @csv_check_bounds = CsvImportAnalyzer::CsvCheckBounds.new
+        expect(@csv_check_bounds.get_min_max_values).to be_instance_of(MissingRequiredArguments)
+      end
+      
+      it 'returns FileNotFound error if file is not found' do
+        @csv_check_bounds = CsvImportAnalyzer::CsvCheckBounds.new(options)
+        expect(@csv_check_bounds.get_min_max_values).to be_instance_of(FileNotFound)
+      end
+    end
+
+    context 'when initialized right' do
+      let(:options) {Hash[filename: $sample_csv_path, chunk_size: 200, delimiter: ",", unique: 2, csv_column_datatypes: {:year_id => :int, :make_id => :string, :model_id => :string, :description_id => :string, :price_id => :float}]}
+      before(:each) do
+        @csv_check_bounds = CsvImportAnalyzer::CsvCheckBounds.new(options)
+      end
+    
+      it 'returns a Hash' do
+        expect(@csv_check_bounds.get_min_max_values).to be_an_instance_of(Hash)
+      end
+    
+      it 'returns correct min & max values for integer type' do
+        result = @csv_check_bounds.get_min_max_values
+        expect(result[:min_max][:year_id][:min]).to eq(1996)
+        expect(result[:min_max][:year_id][:max]).to eq(1999)
+      end
+    
+      it 'returns correct min & max lengths for string type' do
+        result = @csv_check_bounds.get_min_max_values
+        expect(result[:min_max][:make_id][:min]).to eq(4)
+        expect(result[:min_max][:make_id][:max]).to eq(7)
+      end
+    end
+  end
+end

data/spec/csv-import-analyzer/analyzer/delimiter_identifier_spec.rb

@@ -0,0 +1,61 @@
+# require 'spec_helper'
+
+class DummyClass
+end
+
+describe '#identify_delimiter' do
+  
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::DelimiterIdentifier)
+  end
+
+  context 'unable to determine the delimiter' do
+  
+    it 'return invalid input when the input is neither string nor array' do
+      expect(@dummy_class.identify_delimiter(3)).to be_instance_of(InvalidInput)
+    end
+
+    it 'returns file not found when the input string is not a valid file' do
+      expect(@dummy_class.identify_delimiter("test")).to be_instance_of(FileNotFound)
+    end
+
+  end
+
+  context 'finds the delimiter when the input is a file' do
+    
+    it 'returns a comma as the delimiter for sample_csv file' do
+      expect(@dummy_class.identify_delimiter($sample_csv_path)).to eq(",")
+    end
+    
+    it 'returns a semicolon as the the delimiter for sample_ssv file' do
+      expect(@dummy_class.identify_delimiter($sample_ssv_path)).to eq(";")
+    end
+
+  end
+
+  context 'finds the delimiter when the input is an array' do
+    let(:sample) {['1999;Chevy;"Venture ""Extended Edition""";"";4900.00','1999;\'Chevy\';"Venture ""Extended Edition; Very Large""";;5000.00']}
+    it 'returns a semicolon as the delimiter for sample array input' do
+      expect(@dummy_class.identify_delimiter(sample)).to eq(";")
+    end
+  end
+end
+
+describe '#return_plausible_delimiter' do
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::DelimiterIdentifier)
+  end
+
+  context 'identifies delimiter' do
+    it 'returns comma as the delimiter by default' do
+      expect(@dummy_class.return_plausible_delimiter).to eq(",")
+    end
+
+    it 'returns semicolon as the delimiter for sample delimiter_count' do
+      @dummy_class.stub(:delimiter_count).and_return(Hash[","=>15, ";"=>16, "\t"=>0, "|"=>0])
+      expect(@dummy_class.return_plausible_delimiter).to eq(";")
+    end
+  end  
+end

data/spec/csv-import-analyzer/csv_datatype_analysis_spec.rb

@@ -0,0 +1 @@
+# CsvImportAnalyzer::CsvDatatypeAnalysis.new({:delimiter => ",", :chunk => 20, :filename => "sampleTab.csv"}).datatype_analysis

data/spec/csv-import-analyzer/csv_sanitizer_spec.rb

@@ -0,0 +1,24 @@
+# require 'spec_helper'
+
+require 'pry'
+describe CsvImportAnalyzer::CsvSanitizer do
+  let (:csv_sanitizer) { CsvImportAnalyzer::CsvSanitizer.new }
+  it 'should handle file not found issue' do
+    expect(csv_sanitizer.process("sample.csv", options = {})).to be_instance_of(FileNotFound)
+  end
+  #Testing private methods - Although one should really have to test private methods, it's 
+  context 'testing private methods' do
+    let (:test) {"\"t1\", 't2', \"t3\""}
+    let (:res) {"\"t1\", \"t2\", \"t3\""}
+    xit 'should replace single quotes to double' do
+      binding.pry
+      expect(csv_sanitizer.send(:replace_line_single_quotes, test, ",")).to eq(res)
+    end
+    let (:test) {["t1","t2","",nil,"t3"]}
+    let (:res) {["t1","t2","NULL","NULL","t3"]}
+    it 'should replace null values' do
+      expect(csv_sanitizer.send(:replace_null_values, test)).to eq(res)
+    end
+  end
+
+end

data/spec/csv-import-analyzer/helpers/common_functions_spec.rb

@@ -0,0 +1,31 @@
+# require 'spec_helper'
+
+class DummyClass
+end
+
+describe 'null_like?' do
+  
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::Helper)
+  end
+
+  context 'when called on null like objects' do
+    it 'returns NULL as null type' do
+      expect(@dummy_class.null_like?('NULL')).to eq(true)
+    end
+
+    it 'returns \\N as null type' do
+      expect(@dummy_class.null_like?('\N')).to eq(true)
+    end
+  end
+
+  context 'when called on non-null objects' do
+    it 'returns hello as not null' do
+      expect(@dummy_class.null_like?('Hello')).to eq(false)
+    end
+    it 'returns Fixnum(3) as not null' do
+      expect(@dummy_class.null_like?(3)).to eq(false)
+    end
+  end
+end

data/spec/csv-import-analyzer/helpers/csv_check_bounds_spec.rb

@@ -0,0 +1,3 @@
+# check_bounds = CsvImportAnalyzer::Helper::CsvCheckBounds.new({:year_id=>:int, :make_id=>:string, :model_id=>:string, :description_id=>:string, :price_id=>:float}, 
+#   {:delimiter => ",", :chunk => 20, :filename => "/home/avinash/Desktop/csv-import-analyzer/lib/csv-import-analyzer/sampleTab.csv"})
+# check_bounds.get_min_max_values

data/spec/csv-import-analyzer/helpers/datatype_validation_spec.rb

@@ -0,0 +1,75 @@
+# require 'spec_helper'
+require 'date'
+class DummyClass
+end
+
+describe '#validate_field' do
+  
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::DatatypeValidator)
+  end
+
+  context 'knows what an integer looks like' do
+    
+    it 'returns Fixnum type as integer' do
+      expect(@dummy_class.validate_field(10)).to eq("int")
+    end
+    it 'returns Fixnum type with spaces as integer' do
+      expect(@dummy_class.validate_field(' 10 ')).to eq("int")
+    end
+    it 'returns Fixnum type with comma as integer' do
+      expect(@dummy_class.validate_field('1,000')).to eq("int")
+    end
+    it 'returns Fixnum type negative number as integer' do
+      expect(@dummy_class.validate_field(-3)).to eq("int")
+    end
+
+  end
+
+  context 'knows what an Float looks like' do
+    
+    it 'returns Float type as float' do
+      expect(@dummy_class.validate_field(10.0)).to eq("float")
+    end
+    it 'returns Float type with spaces as float' do
+      expect(@dummy_class.validate_field(' 10.01 ')).to eq("float")
+    end
+    it 'returns Float type with comma as float' do
+      expect(@dummy_class.validate_field('1,000.01')).to eq("float")
+    end
+    it 'returns Float type negative number as float' do
+      expect(@dummy_class.validate_field(-3.3)).to eq("float")
+    end
+
+  end
+  
+  context 'it knows what a date looks like' do
+    it 'return true for a valid date type - dd/mm/yyyy' do
+      expect(@dummy_class.validate_field('31/12/2014')).to eq("date")
+    end
+    it 'return true for a valid date type - mm/dd/yyyy' do
+      expect(@dummy_class.validate_field('12/31/2014')).to eq("date")
+    end
+    it 'return true for a valid date type - mm-dd-yyyy' do
+      expect(@dummy_class.validate_field('12-31-2014')).to eq("date")
+    end
+    it 'return true for a valid date type - mm dd yyyy' do
+      expect(@dummy_class.validate_field('12 31 2014')).to eq("date")
+    end
+  end 
+
+  context 'it knows what a String looks like' do
+    it 'default to String type' do
+      expect(@dummy_class.validate_field("100 testingNow:)")).to eq("string")
+    end
+    it 'returns String type as string' do
+      expect(@dummy_class.validate_field("Hello")).to eq("string")
+    end
+    it 'returns String type of dates as string' do
+      expect(@dummy_class.validate_field("12 31 2014312")).to eq("string")
+      expect(@dummy_class.validate_field("12-31-2014312")).to eq("string")
+      expect(@dummy_class.validate_field("12/31/2014312")).to eq("string")
+    end
+  end
+end