csv-import-analyzer 0.0.1

data/lib/csv-import-analyzer/csv_sanitizer.rb

@@ -0,0 +1,86 @@
+require "smarter_csv"
+require "tempfile"
+require_relative "analyzer/delimiter_identifier"
+require_relative "helpers/string_class_extensions"
+require_relative "helpers/common_functions"
+require_relative "helpers/errors"
+require_relative "csv_datatype_analysis"
+
+module CsvImportAnalyzer
+  class CsvSanitizer
+    include CsvImportAnalyzer::Helper
+    include CsvImportAnalyzer::DelimiterIdentifier
+
+    def process(filename, options)
+
+      options = defaults.merge(options)
+      if File.exist?(filename)
+        options[:filename] = filename
+        #first thing to do - find the delimiter of the file.
+        delimiter = identify_delimiter(filename)
+        options[:delimiter] = delimiter
+        File.foreach(filename) do |line|
+          #Check if the line is empty - no point in processing empty lines
+          if line.length > 1
+            line = replace_line_single_quotes(line,delimiter)
+            begin
+              line = CSV.parse_line(line, {:col_sep => delimiter})
+            rescue CSV::MalformedCSVError => error
+              line = "#{line}\""
+              line = CSV.parse_line(line, {:col_sep => delimiter})
+            end
+            line = replace_null_values(line)
+          end
+        end
+        # Cleaned the file - Now analyze for datatypes
+        CsvImportAnalyzer::CsvDatatypeAnalysis.new(options).datatype_analysis
+      else
+        FileNotFound.new
+      end
+    end
+
+    private
+
+    def defaults
+      {
+        :metadata_output => nil,
+        :processed_input => nil,
+        :unique => 10,
+        :check_bounds => true,
+        :datatype_analysis => 200,
+        :chunk => 20,
+        :database => [:pg, :mysql], 
+        :quote_convert => true, 
+        :replace_nulls => true,
+        :out_format => :json
+      }
+    end
+
+    def replace_line_single_quotes(line, delimiter)
+      delimiter = "\\|" if delimiter == "|"
+      pattern = "#{delimiter}'.*?'#{delimiter}" # set the pattern to opening and closing single quote found between delimiters
+      res = line.gsub(/#{pattern}/)
+      result = res.each { |match|
+        replace = "#{delimiter}\""
+        replace = "\|\"" if delimiter == "\\|"
+        match = match.gsub(/^#{delimiter}'/,replace)
+        replace = "\"#{delimiter}"
+        replace = "\"\|" if delimiter == "\\|"
+        match = match.gsub(/'#{delimiter}$/,replace)
+      }
+      result = result.gsub(/''/,'\'') #replace any single quote that might have been used twice to escape single quote before 
+      return result
+    end
+
+    # Replace all nil, "NAN", empty values with NULL for maintaining consistency during database import
+    def replace_null_values(line)
+      line.each do |value|
+        if null_like?(value)
+          replace_index = line.index(value)
+          line[replace_index] = "NULL"
+        end
+      end
+      return line
+    end
+  end
+end

data/lib/csv-import-analyzer/export/metadata_analysis.rb

@@ -0,0 +1,156 @@
+require 'pry'
+require 'json'
+module CsvImportAnalyzer
+  class MetadataAnalysis
+    attr_accessor :metadata, :max_distinct_values
+    def initialize(options)
+      @options = options
+      @metadata = {}
+    end
+
+    def options
+      @options
+    end
+
+    def metadata
+      @metadata
+    end
+
+    def header_datatypes
+      @options[:csv_column_datatypes]
+    end
+
+    def header_datatype_analysis
+      @options[:csv_datatype_analysis]
+    end
+
+    def nullable_columns
+      @options[:nullable]
+    end
+
+    def databases
+      @options[:database]
+    end
+
+    def create_queries
+      @options[:create_query]
+    end
+
+    def import_queries
+      @options[:import_query]
+    end
+
+    def unique_values
+      @options[:uniques]
+    end
+
+    def max_distinct_values
+      @max_distinct_values ||= Integer(options[:unique]) + 1
+    end
+
+    def metadata_print
+      build_metadata_output
+      if options[:metadata_output]
+        if options[:out_format] == :json
+          json_print_to_file
+        end
+        if options[:out_format] == :csv
+          csv_print_to_file
+        end
+      end
+      return JSON.pretty_generate(metadata)
+    end
+
+
+    private
+
+    def json_print_to_file
+      outfile = File.open("metadata_output.json", "w")
+      outfile << JSON.pretty_generate(metadata)
+      outfile.close
+    end
+
+    # Priniting that csv from json is a mess - How to make pretty print ?
+    def csv_print_to_file
+      CSV.open("metadata_output.csv", "w") do |csv|
+        binding.pry
+        metadata.each do |key, value|
+          if value.class == Hash
+            csv << [key]
+            print_hash_to_csv(value, csv)
+          else
+            csv << [key, value]
+          end
+        end
+      end
+    end
+
+    def print_hash_to_csv(hash, csv_handler)
+      if hash.class == Hash
+        hash.each do |key, value|
+          csv_handler << [key]
+          print_hash_to_csv(value, csv_handler)
+        end
+      else
+        csv_handler << [hash]
+      end
+    end
+
+    def build_metadata_output
+      metadata[:csv_file] = add_file_metadata
+      metadata[:data_manipulations] = add_data_manipulations
+      metadata[:csv_headers] = add_header_metadata
+      metadata[:sql] = add_sql_data
+    end
+
+    def add_file_metadata
+      file_data = {}
+      file_data[:filename] = File.basename(options[:filename])
+      file_data[:file_size] = File.size(options[:filename])
+      # file_data[:rows] = options[:rows]
+      # file_data[:columns] = options[:columns]
+      file_data[:record_delimiter] = options[:delimiter]
+      return file_data
+    end
+
+    def add_data_manipulations
+      data_manipulations = {}
+      data_manipulations[:replace_nulls] = options[:replace_nulls]
+      data_manipulations[:replace_quotes] = options[:quote_convert]
+      return data_manipulations
+    end
+
+    def add_header_metadata
+      columns = {}
+      header_datatypes.keys.each do |column_name|
+        begin
+          columns[column_name] = {}
+          columns[column_name][:datatype] = header_datatypes[column_name]
+          columns[column_name][:datatype_analysis] = header_datatype_analysis[column_name]
+          if unique_values[column_name].size > max_distinct_values
+            columns[column_name][:distinct_values] = "#{max_distinct_values}+"
+          else
+            columns[column_name][:distinct_values] = unique_values[column_name]
+          end
+          if nullable_columns.include?(column_name)
+            columns[column_name][:nullable] = true
+          end
+        rescue Exception => e
+          puts e
+        end
+      end
+      return columns
+    end
+
+    def add_sql_data
+      sql = {}
+      databases.each do |db|
+        sql[db] = {}
+        sql[db][:create_query] = create_queries[db]
+        sql[db][:import_query] = import_queries[db]
+      end
+      return sql
+    end
+
+  end
+end

data/lib/csv-import-analyzer/helpers/common_functions.rb

@@ -0,0 +1,11 @@
+module CsvImportAnalyzer
+  module Helper
+    def null_like?(value)
+      if ["NULL", "Null", "NUll", "NULl", "null", nil, "", "NAN", "\\N"].include?(value)
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/csv-import-analyzer/helpers/datatype_validation.rb

@@ -0,0 +1,85 @@
+require 'pry'
+module CsvImportAnalyzer
+  module DatatypeValidator
+
+    def validate_field(content)
+      return get_datatype(content)
+    end
+
+    private
+    ###
+    # Date.parse("12/31/20145234", "%m/%d/%Y") => true which is not supposed to be true (although technically its true)
+    # Validate year part has only 4 numbers in it
+    ###
+
+    def validate_year_date(field)
+      date = nil
+      formats = ["%d/%m/%Y","%d-%m-%Y","%d %m %Y","%m/%d/%Y","%m-%d-%Y","%m %d %Y"]
+      formats.each do |format|
+        if(Date.strptime(field, format) rescue false)
+          date = Date.strptime(field, format)
+          break
+        end
+      end
+      unless date.nil?
+        field = date.to_s.scan(/\d*/) # Return an array with patterns matching with only numbers in it
+        if field[0].length == 4
+          return true
+        else
+          return false
+        end
+      else
+        return false
+      end
+    end
+
+    ###
+    # To check for pattern of Date format after Date.parse is successfull
+    # Date.parse(3000) => true which is not supposed to be true
+    ###
+    def datetime_pattern(field)
+      pattern1 = field.scan(/[0-9]\//)
+      pattern2 = field.scan(/[0-9]\-/)
+      pattern3 = field.scan(/[0-9] /)
+      pattern4 = field.scan(/[0-9] [A-Z][a-z][a-z] [0-9]|[0-9]-[A-Z][a-z][a-z]-[0-9]|[0-9] [a-z][a-z][a-z] [0-9]|[0-9]-[a-z][a-z][a-z]-[0-9]/)
+      if(pattern1.size == 2||pattern2.size == 2||pattern3.size == 2||pattern4.size != 0)
+        if(validate_year_date(field))
+          return true
+        else
+          return false
+        end
+      else
+        return false
+      end
+    end
+    
+    ###
+    #To determine the data-type of an input field
+    ###
+    def get_datatype(field)
+      #Remove if field has any comma's for int and float rep
+      if field != nil && field.class == String
+        num = field.gsub(/,/,'')
+      else
+        num = field
+      end
+      if(Integer(num) rescue false)
+        if num.class == Float
+          return "float"
+        end
+        return "int"
+      elsif(Float(num) rescue false)
+        return "float"
+      elsif(Date.parse(field) rescue false || Date.strptime(field, '%m/%d/%Y') rescue false || Date.strptime(field, '%m-%d-%Y') rescue false || Date.strptime(field, '%m %d %Y') rescue false)
+        if datetime_pattern(field)
+          if field =~ /:/ # To check if the field contains any pattern for Hours:minutes
+            return "datetime"
+          else
+            return "date"
+          end
+        end
+      end
+      return "string"
+    end
+  end
+end

data/lib/csv-import-analyzer/helpers/errors.rb

@@ -0,0 +1,3 @@
+class FileNotFound < StandardError; end
+class InvalidInput < StandardError; end
+class MissingRequiredArguments < StandardError; end

data/lib/csv-import-analyzer/helpers/string_class_extensions.rb

@@ -0,0 +1,8 @@
+require 'pry'
+class String
+  #Extending string class to return the count of substr inside a string
+  def substr_count(needle)
+    needle = "\\#{needle}" if(needle == '|') # To escape inside regex
+    self.scan(/(#{needle})/).size
+  end
+end

data/lib/csv-import-analyzer/query_builder/mysql_query_helper.rb

@@ -0,0 +1,31 @@
+require_relative "../helpers/errors"
+require 'pry'
+module CsvImportAnalyzer
+  module MysqlQueryHelper
+  
+    def form_query_for_datatype(args)
+      unless args[:datatype].nil? || args[:header].nil?
+        if args[:datatype] == :string
+          return args[:header].to_s + " varchar(255)"
+        else
+          return args[:header].to_s + " " + args[:datatype].to_s
+        end
+      else
+        MissingRequiredArguments.new("Required arguments missing for form_query_for_datatype")
+      end
+    end
+
+    def import_csv(args)
+      unless args[:tablename].nil? || args[:filename].nil? || args[:delimiter].nil?
+        import_statement = "LOAD DATA INFILE #{args[:filename]} INTO TABLE #{args[:tablename]} "+
+              "FIELDS TERMINATED BY '#{args[:delimiter]}' "+
+              "ENCLOSED BY '\"' "+
+              "LINES TERMINATED BY '\\n' "+
+              "IGNORE 1 LINES;"
+      else
+        MissingRequiredArguments.new("Required arguments missing for import_csv")
+      end
+    end
+
+  end
+end

data/lib/csv-import-analyzer/query_builder/pg_query_helper.rb

@@ -0,0 +1,27 @@
+require_relative "../helpers/errors"
+require "pry"
+module CsvImportAnalyzer
+  module PgQueryHelper
+
+    def form_query_for_datatype(args)
+      unless args[:datatype].nil? || args[:header].nil?
+        if args[:datatype] == :string
+          return args[:header].to_s + " varchar(255)"
+        else
+          return args[:header].to_s + " " + args[:datatype].to_s
+        end
+      else
+        MissingRequiredArguments.new("Required arguments missing for form_query_for_datatype")
+      end
+    end
+
+    def import_csv(args)
+      unless args[:tablename].nil? || args[:filename].nil? || args[:delimiter].nil?
+        pg_import_statement = "COPY #{args[:tablename]} FROM '#{args[:filename]}' HEADER DELIMITER '#{args[:delimiter]}' CSV NULL AS 'NULL';"
+      else
+        MissingRequiredArguments.new("Required arguments missing for import_csv")
+      end
+    end
+
+  end
+end

data/lib/csv-import-analyzer/query_builder/query_helper.rb

@@ -0,0 +1,27 @@
+require_relative "../helpers/errors"
+require "pry"
+module CsvImportAnalyzer
+  module QueryHelper
+
+    def form_query_for_datatype(args)
+      unless args[:datatype].nil? || args[:header].nil?
+        if args[:datatype] == :string
+          return args[:header].to_s + " varchar(255)"
+        else
+          return args[:header].to_s + " " + args[:datatype].to_s
+        end
+      else
+        MissingRequiredArgumentsError.new("Required arguments missing for form_query_for_datatype")
+      end
+    end
+
+    def import_csv(args)
+      unless args[:tablename].nil? || args[:filename].nil? || args[:delimiter].nil?
+        pg_import_statement = "COPY #{args[:tablename]} FROM '#{args[:filename]}' HEADER DELIMITER '#{args[:delimiter]}' CSV NULL AS 'NULL';"
+      else
+        MissingRequiredArgumentsError.new("Required arguments missing for import_csv")
+      end
+    end
+
+  end
+end