kuppayam 0.1.5 → 0.1.6dev2

data/config/initializers/importer.rb

@@ -0,0 +1,303 @@
+# require 'axlsx'
+require 'colorized_string'
+# require 'roo'
+require 'benchmark'
+require 'iconv'
+
+module Kuppayam
+	
+	module Importer
+
+		class ErrorHash
+
+			attr_accessor :warnings,
+		                :errors
+
+		  def initialize
+		  	@warnings = []
+		    @errors = []
+		  end
+
+		  def warnings?
+		  	@warnings.any?
+		  end
+
+		  def errors?
+		  	@errors.any?
+		  end
+
+		  def print_dot
+		  	if self.warnings?
+	        print ".".yellow
+	      elsif self.errors?
+	        print ".".red
+	      else
+	        print ".".green
+	      end
+		  end
+
+		  def print_all
+		  	self.warnings.each do |item|
+	        puts "Summary: #{item[:summary]}".yellow
+	        puts "Details: #{item[:details]}".yellow if item[:details]
+	        puts "Stack Trace: #{item[:stack_trace]}".yellow if item[:stack_trace]
+	        puts ""
+		  	end
+		  	self.errors.each do |item|
+	        puts "Summary: #{item[:summary]}".red
+	        puts "Details: #{item[:details]}".red if item[:details]
+	        puts "Stack Trace: #{item[:stack_trace]}".red if item[:stack_trace]
+	        puts ""
+		  	end
+		  end
+		end
+
+		class DataError
+			attr_accessor :errors, :columns
+
+			def initialize
+				@errors = {}
+				@columns = []
+			end
+
+			def add_column_error(name, value, errors, row_number)
+				if @errors[row_number].blank?
+					@errors[row_number] = {
+						name => { value: value, errors: errors }
+					}
+				else
+					@errors[row_number][name] = { value: value, errors: errors }
+				end
+			end
+
+			def generate_error_file
+
+		  	axlsx_package = Axlsx::Package.new
+				wb = axlsx_package.workbook
+
+				normal_cell = s.add_style :bg_color => "FFFFFF", :fg_color => "4B4B4B", :sz => 14, :alignment => { :horizontal=> :center }
+				error_cell = s.add_style :bg_color => "FFE2E2", :fg_color => "4B4B4B", :sz => 14, :alignment => { :horizontal=> :center }
+
+				wb.add_worksheet(:name => "Errors") do |sheet|
+			    
+			    sheet.add_row self.columns
+
+			    self.errors.each do |row_number, values|
+			    	row_data = self.columns.map{|col| values[col][:value] }
+						sheet.add_row row_data
+			    end
+
+			  end
+		  	axlsx_package.serialize("tmp/example.xlsx")
+		  end
+		end
+
+		def clean_string(str, label=nil)
+			begin
+				cleaned_str = str.gsub(/\r/, "").gsub(/\n/, "").squeeze(" ")
+				cleaned_str = Iconv.conv('UTF-8', 'ISO-8859-1//TRANSLIT//IGNORE', cleaned_str)
+			rescue Exception => e
+				puts "Error while cleaning #{label ? label : 'string'} '#{str}' - #{e.message}".red
+			end
+		end
+
+		def check_file_type_and_import(path, single_transaction=true, verbose=true)
+	  	if File.exists?(path)
+	    	if File.extname(path) == ".csv"
+	      	puts "CSV file found at '#{path.to_s}'.".green if verbose
+	      	self.import_from_csv(path.to_s, single_transaction, verbose)
+	      elsif File.extname(path) == ".xlsx"
+	      	puts "XSLX file found at '#{path.to_s}'.".green if verbose
+	      	self.import_from_xslx(path.to_s, single_transaction, verbose)
+	      else
+	      	puts "Unsupported File encountered'#{path.to_s}'.".red if verbose
+	      	return
+	      end
+	    else
+	      puts "Import File not found at '#{path.to_s}'.".red if verbose
+	    end
+	  end
+
+		def import_from_sql(sql_path, verbose=true)
+
+			errors = []
+
+			print_memory_usage do
+			  print_time_spent do
+			    sum = 0
+
+			    CSV.foreach(csv_path, headers: true, header_converters: :symbol) do |row|
+			    	obj, error_object = save_row_data(row)
+			      errors << error_object if error_object
+			      error_object.print_dot if error_object && verbose
+			      sum += 1
+			    end
+
+			    puts "Sum: #{sum}"
+			  end
+			end
+
+			if verbose
+	      puts ""
+	      errors.each do |error_object|
+	        error_object.print_all if error_object
+	      end
+	    end
+	  end
+
+		def import_from_csv(csv_path, single_transaction=true, verbose=true)
+			errors = []
+	    sum = 0
+
+	    # , encoding: 'windows-1251:utf-8', :row_sep => :auto
+	    if single_transaction
+		    ActiveRecord::Base.transaction do 
+			    CSV.foreach(csv_path, headers: true, header_converters: :symbol, skip_blanks: true) do |row|
+			    	obj, error_object = save_row_data(row)
+			      errors << error_object if error_object
+			      error_object.print_dot if error_object && verbose
+			      sum += 1
+			    end
+		    end
+		  else
+		  	CSV.foreach(csv_path, headers: true, header_converters: :symbol, skip_blanks: true) do |row|
+		    	obj, error_object = save_row_data(row)
+		      errors << error_object if error_object
+		      error_object.print_dot if error_object && verbose
+		      sum += 1
+		    end
+	    end
+	    puts "\tScanned #{sum} rows".yellow
+
+			if verbose
+	      puts ""
+	      errors.each do |error_object|
+	        error_object.print_all if error_object
+	      end
+	    end
+	  end
+
+	  def import_from_xslx(xlsx_path, single_transaction = true, verbose=true)
+	  	xlsx = Roo::Spreadsheet.open(xlsx_path, extension: :xlsx)
+    	sheet = xlsx.sheet(0)
+			headers = sheet.row(1)
+
+			errors = []
+			if single_transaction
+		    ActiveRecord::Base.transaction do 
+					2.upto(xlsx.last_row) do |line|
+						row_hash = ActiveSupport::HashWithIndifferentAccess[headers.zip(xlsx.row(line))]
+						obj, error_object = save_row_data(row_hash)
+			      errors << error_object if error_object
+			      error_object.print_dot if error_object && verbose
+					end
+				end
+			else
+				2.upto(xlsx.last_row) do |line|
+					row_hash = ActiveSupport::HashWithIndifferentAccess[headers.zip(xlsx.row(line))]
+					obj, error_object = save_row_data(row_hash)
+		      errors << error_object if error_object
+		      error_object.print_dot if error_object && verbose
+				end
+			end
+			
+			if verbose
+	      puts ""
+	      errors.each do |error_object|
+	        error_object.print_all if error_object
+	      end
+	    end
+	  end
+
+	  def walk_and_import(start, single_transaction=true, verbose=true)
+	  	puts "Importing Files from the Folder '#{start.to_s}'".yellow if verbose
+		  Dir.foreach(start) do |x|
+		  	next if x.starts_with?("master")
+		  	next unless x.ends_with?(".csv")
+		    path = File.join(start, x)
+		    if x == "." or x == ".."
+		      next
+		    elsif File.directory?(path)
+		    	self.walk_and_import(path, single_transaction, verbose)
+		    else
+		      self.check_file_type_and_import(path, single_transaction, verbose)
+		    end
+		  end
+		end
+
+	  def import_data_recursively(path, single_transaction=true, verbose=true)
+	  	print_memory_usage do
+			  print_time_spent do
+			  	if Dir.exists?(path)
+			  		self.walk_and_import(path, single_transaction, verbose)
+			  	else
+			      puts "Import Folder not found: '#{path.to_s}'.".red if verbose
+			    end
+			  end
+			end
+	  end
+
+	  def import_data_file(path, single_transaction=true, verbose=true)
+	  	print_memory_usage do
+			  print_time_spent do
+			  	check_file_type_and_import(path, single_transaction, verbose)
+			  end
+			end
+	  end
+
+	  def print_memory_usage
+		  memory_before = `ps -o rss= -p #{Process.pid}`.to_i
+		  yield
+		  memory_after = `ps -o rss= -p #{Process.pid}`.to_i
+
+		  puts "Memory: #{((memory_after - memory_before) / 1024.0).round(2)} MB"
+		end
+
+		def print_time_spent
+		  time = Benchmark.realtime do
+		    yield
+		  end
+
+		  puts "Time: #{time.round(2)}"
+		end
+
+		def csv_foreach_backup_code
+			#   batch_size = 1000
+
+			  #   File.open(csv_path) do |file|
+					#   file.lazy.drop(header_lines).each_slice(batch_size) do |lines|
+					#     puts lines
+					#   end
+					# end
+
+			  #   File.open(csv_path) do |file|
+			  #   	binding.pry
+					#   headers = file.first
+					#   file.lazy.each_slice(batch_size) do |lines|
+					#     csv_rows = CSV.parse(lines.join, write_headers: true, headers: headers)
+					#     csv_rows.each do |row|
+				 #    		error_object = save_row_data(row)
+					#       errors << error_object if error_object
+					#       error_object.print_dot if error_object
+					#       sum += 1
+					#     end
+					#   end
+					# end
+
+					# rows = CSV.new(File.open(csv_path,'r'), headers: true, header_converters: :symbol).lazy.select do |row|
+					# end
+
+					# binding.pry
+
+					# rows.each do |row|
+
+					# 	error_object = save_row_data(row)
+			  #     errors << error_object if error_object
+			  #     error_object.print_dot if error_object
+			  #     sum += 1
+			  #   end
+		end
+
+	end
+
+end

data/config/initializers/validators.rb

@@ -0,0 +1,78 @@
+module Kuppayam
+  module Validators
+
+    def generate_validation_options(attribute, options)
+      {a: 1}
+      reg_exp = /\A[a-zA-Z1-9\-\ \(\)\.+]*\z/i
+      options.reverse_merge!(
+        presence: true,
+        min_length: 3,
+        max_length: 256,
+        format: reg_exp,
+        uniqueness: false,
+        mandatory: false
+      )
+
+      if options.has_key?(:condition_method) && !options[:condition_method].blank?
+        condition_proc = proc {|obj| obj.send(options[:condition_method])}
+      elsif options.has_key?(:mandatory) && options[:mandatory] == false
+        condition_proc = proc {|obj| !obj.send(attribute).blank?}
+      else
+        condition_proc = nil
+      end
+
+      voptions = {
+        presence: options[:presence],
+        length: { minimum: options[:min_length], maximum: options[:max_length]},
+        format: {:with => options[:format]}
+      }
+      voptions.merge!(uniqueness: options[:uniqueness]) if options[:uniqueness]
+      voptions.merge!(if: condition_proc) if condition_proc
+
+      voptions
+    end
+
+    def validate_string(attribute, **options)
+      voptions = generate_validation_options(attribute, options)
+      validates attribute, **voptions
+    end
+
+    def validate_username(attribute, **options)
+      reg_exp = /\A[a-zA-Z0-9\_\.]*\z/
+      options.merge!(mandatory: true, min_length: 3, max_length: 128, format: reg_exp, :uniqueness => {:case_sensitive => false})
+      voptions = generate_validation_options(attribute, options)
+      validates attribute, **voptions
+    end
+
+    def validate_email(attribute, **options)
+      reg_exp = /\A(|(([A-Za-z0-9]+_+)|([A-Za-z0-9]+\-+)|([A-Za-z0-9]+\.+)|([A-Za-z0-9]+\++))*[A-Za-z0-9]+@((\w+\-+)|(\w+\.))*\w{1,63}\.[a-zA-Z]{2,6})\z/i
+      options.merge!(mandatory: true, format: reg_exp, :uniqueness => {:case_sensitive => false})
+      voptions = generate_validation_options(attribute, options)
+      validates attribute, **voptions
+    end
+
+    # format can be either :us or :indian
+    def validate_phone(attribute, **options)
+      options.reverse_merge!(phone_format: :india)
+      if options[:phone_format] == :us
+        reg_exp = /\A[0-9]{3}[-][0-9]{3}[-][0-9]{4}\z/
+        length = 12
+      else
+        reg_exp = /\A[0-9]{10}\z/
+        length = 10
+      end
+      options.merge!(format: reg_exp, min_length: length, max_length: length)
+      options.reverse_merge!(numericality: true, mandatory: true, uniqueness: true)
+      voptions = generate_validation_options(attribute, options)
+      validates attribute, **voptions
+    end
+
+    def validate_password(attribute, **options)
+      reg_exp = /\A(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9!@$#&*_\.,;:])/
+      options.merge!(mandatory: true, format: reg_exp, min_length: 7, max_length: 256)
+      voptions = generate_validation_options(attribute, options)
+      validates attribute, **voptions
+    end
+
+  end
+end

data/config/locales/kuppayam/general.yml

@@ -23,6 +23,7 @@ en:
     updated: "%{item} has been updated successfully."
     deleted: "%{item} has been deleted successfully."
     removed: "%{item} has been removed successfully."
+    uploaded: "%{item} has been uploaded successfully."
     no_results_found: "No results found."
   state:
     changed: "%{item} has been %{new_state}."

data/config/routes.rb

@@ -1,6 +1,6 @@
 Rails.application.routes.draw do
 
-	resources :images, controller: "kuppayam/images" do
+  resources :images, controller: "kuppayam/images" do
     member do
       put :crop
     end

data/db/migrate/20170000000000_create_images.rb

@@ -8,6 +8,7 @@ class CreateImages < ActiveRecord::Migration[5.0]
       t.timestamps
     end
 
+    add_index(:images, :image_type)
     add_index(:images, [ :imageable_id, :imageable_type ])
   end
 end

data/db/migrate/20170000000001_create_documents.rb

@@ -1,4 +1,4 @@
-class CreateDocuments < ActiveRecord::Migration
+class CreateDocuments < ActiveRecord::Migration[5.0]
   def change
     create_table :documents do |t|
       t.string  :document
@@ -8,6 +8,7 @@ class CreateDocuments < ActiveRecord::Migration
       t.timestamps
     end
 
+    add_index(:documents, :document_type)
     add_index(:documents, [ :documentable_id, :documentable_type ])
   end
 end

data/db/migrate/20170000000002_create_import_data.rb

@@ -1,9 +1,9 @@
-class CreateImportData < ActiveRecord::Migration
+class CreateImportData < ActiveRecord::Migration[5.0]
   def change
     create_table :import_data do |t|
     	t.integer :importable_id
       t.string  :importable_type
-      t.string  :data_type
+      t.string  :data_type # CSV, XLSX, XLS
       t.string :status, :null => false, :default=>"pending", :limit=>16
       t.timestamps
     end

data/lib/kuppayam/action_view/form_helper.rb

@@ -145,7 +145,7 @@ module Kuppayam
           when :text, :email, :search, :password, :date, :time, :tel, :url, :month, :number
             text_field_tag(options[:param_name], object.send(field_name.to_s), **options[:html_options])
           when :textarea
-            options[:html_options].merge!(style: "height: 140px;")
+            options[:html_options].reverse_merge!(style: "height: 140px;")
             text_area_tag(options[:param_name], object.send(field_name.to_s), **options[:html_options])
           when :file
             file_field_tag(options[:param_name], **options[:html_options])

data/lib/kuppayam/action_view/theme_helper.rb

@@ -6,6 +6,8 @@ module Kuppayam
       # <i class='fa fa-plus'></i>
       # theme_fa_icon('plus', 'lg')
       # <i class='fa fa-plus fa-lg'></i>
+
+      # TBR: To Be Removed
       def theme_fa_icon(icon_text, size="")
         size_class = %w{lg 2x 3x 4x 5x}.include?(size.strip) ? " fa-#{size.strip}" : ""
         "<i class='fa fa-#{icon_text}#{size_class}'></i>"
@@ -13,8 +15,10 @@ module Kuppayam
 
       # theme_button_text('New Project')
       # <span class='btn-text'> New Project</span>
+
+      # TBR: To Be Removed
       def theme_button_text(text)
-        "<span class='btn-text'> #{text}</span>"
+        "<span class='btn-text hidden-sm hidden-xs'> #{text}</span>"
       end
 
       # theme_button is used to create buttons which has built in classes and icons
@@ -24,6 +28,8 @@ module Kuppayam
       #   link_to raw("<i class='fa fa-plus mr-10'></i><span class='btn-text'> New Project</span>"), new_admin_project_path, :class=>"btn btn-primary pull-right ml-5", :remote=>true
       # And produces the following html
       #   <a class="btn btn-primary pull-right ml-5" data-remote="true" href="/admin/projects/new"><i class="fa fa-plus mr-10"></i><span class="btn-text"> New Project</span></a>
+
+      # TBR: To Be Removed
       def theme_button(text, icon, url, options={})
         options.reverse_merge!(
           method: :get,
@@ -130,7 +136,7 @@ module Kuppayam
       #   "No Results found"
       #  </div>
       def theme_panel_message(message)
-        content_tag(:div, class: "panel panel-default panel-message text-color-grey p-80 text-align-center", style: "height:200px;") do
+        content_tag(:div, class: "panel panel-default panel-message text-color-grey p-80 text-align-center", style: "height:200px;width:100%;") do
           raw(message)
         end
       end