active_record_importer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43724c0cedd35917d14e3439f96482c8f2357943
-  data.tar.gz: 7db97349cd0eafa87a2296906f56e6543c58395f
+  metadata.gz: 07cd0e7debf64daad8259915aa82f993c1c86f1c
+  data.tar.gz: 98df0543814ef29f1bbe2ec031c9f02394db723e
 SHA512:
-  metadata.gz: 22f7c0e2eb463b49e0a717d1ee4da685c111424681f20cbc2f578ebe5bbbab57277384f05715aca3abf22ffb25fe957efe8820ee916d6add08cc8ba82fa82e84
-  data.tar.gz: e90f8ab8ce49228e47441a3df8adcd214547fec041583912760e0792b1b6eff631f8378723d9cc73af23fbcc4605b8c58ee59578ca433a1c8a6070f413026502
+  metadata.gz: 521db635648cfe9da9d5ce938f58c519db6e6e16c7428df3efa25a251fce72f655c17ee98d61f9d98ac949b377107f07be5377b531e0ac1597edf4e9238aa5b2
+  data.tar.gz: 407647a49e620e64cc47c125f106017b132e24c44fb039d4e882f6ceea7f5b3ec9ed90ec65d8ed0e638ef79590c3bf660f50bf8f94a3557f765ed260facd09c6

data/README.md

@@ -1,11 +1,12 @@
 # ActiveRecordImporter
 
-Supports only Rails 4... (I'll release soon an update to support Rails 5)
+Supports only Rails 4 and 5
 
 This gem helps you insert/update records easily. For now, it only accepts CSV file.
 This also helps you monitor how many rows are imported, and how many rows failed.
+This gem also allows you to easily import to any model with few configurations.
+
 I'll release an update to enable this on background job.
-This gem allows you to easily import to any model with few configurations.
 
 ## Installation
 
@@ -46,7 +47,7 @@ class ActiveRecordImporterMigration < ActiveRecord::Migration
 end
 ```
 
-#### Import Model:
+#### Add Import Model:
 ```ruby
 class Import < ActiveRecord::Base
   extend Enumerize
@@ -57,6 +58,7 @@ class Import < ActiveRecord::Base
             default: :upsert
 
   has_attached_file :file
+  has_attached_file :failed_file
 
   attr_accessor :execute_on_create
 
@@ -67,15 +69,22 @@ class Import < ActiveRecord::Base
                            content_type: %w(text/plain text/csv)
                        }
 
-  after_create :execute, if: :execute_on_create
+  validates_attachment :failed_file,
+                       content_type: {
+                           content_type: %w(text/plain text/csv)
+                       }
 
-  # I'll add import options in the next release
+  # I'll add import options in the next major release
   # accepts_nested_attributes_for :import_options, allow_destroy: true
 
   def execute
     resource_class.import!(self, execute_on_create)
   end
 
+  def execute!
+    resource_class.import!(self, true)
+  end
+
   def resource_class
     resource.safe_constantize
   end
@@ -86,11 +95,20 @@ class Import < ActiveRecord::Base
 
   ##
   # Override this if you prefer have
-  # a private permissions or you have
+  # private permissions or you have
   # private methods for reading files
   ##
   def import_file
-    local_path? ? file.path : file.url
+    local_path?(file) ? file.path : file.url
+  end
+
+  ##
+  # Override this method if you have
+  # private permissions or you have private methods
+  # for reading/writing uploaded files
+  ##
+  def failed_file_path
+    local_path?(failed_file) ? failed_file.path : failed_file.url
   end
 
   private
@@ -100,13 +118,14 @@ class Import < ActiveRecord::Base
     errors.add(:find_options, "can't be blank") if find_options.blank?
   end
 
-  def local_path?
-    File.exist? import_file.file.path
+  def local_path?(f)
+    File.exist? f.path
   end
 end
 ```
 
-Add `acts_as_importable` to a model to make it importable
+### Add `acts_as_importable` to any ActiveRecord model to make it importable
+
 ```ruby
 class User < ActiveRecord::Base
   acts_as_importable
@@ -120,9 +139,35 @@ class User < ActiveRecord::Base
                                            last_name: 'dela Cruz' },
                      find_options: %i(email),
                      before_save: Proc.new { |user| user.password = 'temporarypassword123' }
+                     after_save: Proc.new { |user| puts "THIS IS CALLED AFTER OBJECT IS SAVED" }
 end
 ```
 
+If you're using ActiveRecord::Store, you may import values to your accessors by including them in the configuration:
+```ruby
+class User < ActiveRecord::Base
+  store :properties, accessors: [:first_key, :second_key]
+
+  acts_as_importable store_accessors: [:first_key, :second_key]
+end
+```
+
+### Add import form
+This is a sample import HAML form:
+```ruby
+    # resource is your Model name
+    = f.input :resource
+    # batch_size is useful for large csv file
+    = f.input :batch_size
+    # insert_methods: [:upsert, :insert, :error_on_duplicate]
+    = f.input :insert_method, collection: insert_methods, class: 'form-control insert-method'
+    # `find_options` are the list of columns you want to use to update a certain instance or
+    # error when a duplicate is found. This is not required when your insert_method is `:insert`
+    = f.input :find_options
+    = f.input :file, as: :file,
+      input_html: { accept: '.csv' }
+```
+
 You may also add some options from the SmarterCSV gem:
 
     | Option                         |  Default
@@ -135,24 +180,51 @@ You may also add some options from the SmarterCSV gem:
     | :chunk_size                    |  500
     | :col_sep                       |  ","
 
-#### I'll add more options SOON!
+https://github.com/tilo/smarter_csv
+
+
+```ruby
+class User < ActiveRecord::Base
+  acts_as_importable csv_opts: {
+                       chunk_size: 2000,
+                       col_sep: '|',
+                       convert_values_to_numeric: { only: [:age, :salary] }
+                     }
+end
+```
+
+`I'll add more options SOON!`
 
-#### Imports Controller:
+
+### Create Imports Controller:
 ```ruby
 class ImportsController < ApplicationController
 
   def create
     @import = Import.create!(import_params)
-    @import.execute_on_create = true
-    @import.execute
+    @import.execute!
   end
 
+  private
+
+  def import_params
+    params.require(:import).permit(:file, :resource, :insert_method, :batch_size)
+  end
 end
 ```
 
-You may include execute_on_create as a checkbox field in your Import form so you don't have to
-include `@import.execute_on_create = true` in your controller.
-
+#### Run it via Rails Console:
+```ruby
+File.open(PATH_TO_CSV_FILE) do |file|
+  @import = Import.create!(
+             resource: 'User',
+             file: file,
+             insert_method: 'upsert',
+             find_options: 'first_name,last_name'
+           )
+end
+@import.execute!
+```
 
 
 ## Development
@@ -169,4 +241,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/michae
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/lib/active_record_importer.rb

@@ -17,11 +17,15 @@ module ActiveRecordImporter
   autoload :Importable,           'active_record_importer/importable'
   autoload :InstanceBuilder,      'active_record_importer/instance_builder'
   autoload :OptionsBuilder,       'active_record_importer/options_builder'
-  autoload :AttributesBuilder,    'active_record_importer/attributes_builder'
-  autoload :FindOptionsBuilder,   'active_record_importer/find_options_builder'
   autoload :TransitionProcessor,  'active_record_importer/transition_processor'
   autoload :ImportCallbacker,     'active_record_importer/import_callbacker'
-  autoload :Helpers,              'active_record_importer/helpers'
+  autoload :FailedFileBuilder,     'active_record_importer/failed_file_builder'
+
+  module Attribute
+    autoload :AttributesBuilder,    'active_record_importer/attribute/attributes_builder'
+    autoload :FindOptionsBuilder,   'active_record_importer/attribute/find_options_builder'
+    autoload :Helpers,              'active_record_importer/attribute/helpers'
+  end
 
   require 'active_record_importer/railtie' if defined?(Rails) && Rails::VERSION::MAJOR >= 3
 end

data/lib/active_record_importer/attribute/attributes_builder.rb

@@ -0,0 +1,69 @@
+module ActiveRecordImporter
+  module Attribute
+    class AttributesBuilder
+      include Attribute::Helpers
+
+      attr_reader :importable, :row_attrs, :processed_attrs
+
+      delegate :importer_options, to: :importable
+      delegate :importable_columns,
+               :default_attributes,
+               :find_assoc_opts,
+               to: :importer_options
+
+      def initialize(importable, row_attrs)
+        @importable = importable
+        @row_attrs = row_attrs
+        @processed_attrs = {}
+      end
+
+      def build
+        force_encode_attributes
+        fetch_time_attributes
+        processed_attrs
+      end
+
+      private
+
+      def default_attrs
+        def_attrs = { importing: true }
+        default_attributes.each do |key, value|
+          def_attrs[key] = fetch_value(value)
+        end
+      end
+
+      def force_encode_attributes
+        @processed_attrs = force_utf8_encode(merged_attributes)
+      end
+
+      def fetch_time_attributes
+        @processed_attrs.merge!(time_attributes(processed_attrs))
+      end
+
+      def fetch_value(value)
+        case value
+          when Proc
+            value.call(row_attrs)
+          when Symbol
+            importable.send(value, row_attrs)
+          else
+            value
+        end
+      end
+
+      def merged_attributes
+        attributes = row_attrs.slice(*importable_columns)
+        row_attrs = attributes.inject({}) do |row_attrs, key_value|
+          row_attrs[key_value.first] = key_value.last if key_value.last.present?
+          row_attrs
+        end
+        default_attrs.merge(row_attrs)
+      end
+
+      def has_column?(column)
+        return if column.blank?
+        importable_columns.include?(column.to_sym)
+      end
+    end
+  end
+end

data/lib/active_record_importer/attribute/find_options_builder.rb

@@ -0,0 +1,54 @@
+module ActiveRecordImporter
+  module Attribute
+    class FindOptionsBuilder
+      include Virtus.model
+      include Helpers
+
+      attribute :resource, String
+      attribute :attrs, Hash, default: {}
+      attribute :find_options, String
+      attribute :prefix, String
+
+      def build
+        get_find_opts
+        slice_attributes
+      end
+
+      private
+
+      def klass
+        resource.safe_constantize
+      end
+
+      delegate :importer_options, to: :klass
+
+      delegate :required_attributes, to: :importer_options
+
+      def get_find_opts
+        @options = strip_and_symbolize
+        @options ||= importer_options.find_options || required_attributes
+        @options
+      end
+
+      def slice_attributes
+        return attrs.slice(*@options).compact if prefix.blank?
+
+        @options.inject({}) do |attr, key|
+          attr[key] = attrs[prefixed_key(key)].presence
+          attr
+        end.compact
+      end
+
+      def prefixed_key(key)
+        "#{prefix}#{key}".to_sym
+      end
+
+      def strip_and_symbolize
+        return if find_options.blank?
+        find_options.split(',').map do |key|
+          key.strip.to_sym
+        end
+      end
+    end
+  end
+end

data/lib/active_record_importer/attribute/helpers.rb

@@ -0,0 +1,28 @@
+module ActiveRecordImporter
+  module Attribute
+    module Helpers
+
+      def parse_datetime(datetime = nil)
+        return if datetime.blank?
+        Time.parse(datetime)
+      end
+
+      def force_utf8_encode(data = {})
+        return data if data.blank?
+
+        data.keys.each do |key|
+          data[key] = data[key].force_encoding('UTF-8') if data[key].is_a?(String)
+        end
+
+        data
+      end
+
+      def time_attributes(data = {})
+        attrs = {}
+        attrs[:created_at] = parse_datetime(data[:created_at]) || Time.now
+        attrs[:updated_at] = parse_datetime(data[:updated_at]) || attrs[:created_at]
+        attrs
+      end
+    end
+  end
+end

data/lib/active_record_importer/batch_importer.rb

@@ -1,32 +1,44 @@
 module ActiveRecordImporter
   class BatchImporter
 
-    attr_reader :data, :import
+    attr_reader :data, :import, :failed_file, :processor
 
     def initialize(import, data)
       @import = import
       @data = data
+      @failed_file = FailedFileBuilder.new(import)
     end
 
     def process!
       @imported_count, @failed_count = 0, 0
-
       data.each do |row|
         next if row.blank?
-        processor = DataProcessor.new(import, row.symbolize_keys!)
-        processor.process ? @imported_count += 1 : @failed_count += 1
+        process_row(row.symbolize_keys!)
       end
 
       set_import_count
+      finalize_batch_import
     end
 
     private
 
+    def process_row(row)
+      processor = DataProcessor.new(import, row)
+      return @imported_count += 1 if processor.process
+
+      @failed_file.failed_rows << row.merge(import_errors: processor.row_errors)
+      @failed_count += 1
+    end
+
     def set_import_count
       Import.update_counters(import.id, imported_rows: @imported_count)
       Import.update_counters(import.id, failed_rows: @failed_count)
     end
 
+    def finalize_batch_import
+      @failed_file.build
+    end
+
     def importable
       import.resource.safe_constantize
     end

data/lib/active_record_importer/data_processor.rb

@@ -38,7 +38,7 @@ module ActiveRecordImporter
     end
 
     def fetch_instance_attributes
-      @attributes = AttributesBuilder.new(
+      @attributes = Attribute::AttributesBuilder.new(
                       importable, row_attrs
                     ).build
     rescue => exception
@@ -46,7 +46,7 @@ module ActiveRecordImporter
     end
 
     def fetch_find_attributes
-      @find_attributes = FindOptionsBuilder.new(
+      @find_attributes = Attribute::FindOptionsBuilder.new(
         resource: import.resource,
         find_options: import.find_options,
         attrs: attributes

data/lib/active_record_importer/errors.rb

@@ -12,6 +12,10 @@ module ActiveRecordImporter
       end
     end
 
-    class DuplicateRecord < StandardError; end
+    class DuplicateRecord < StandardError
+      def initialize
+        super 'Duplicate record found!'
+      end
+    end
   end
 end

data/lib/active_record_importer/failed_file_builder.rb

@@ -0,0 +1,81 @@
+module ActiveRecordImporter
+  class FailedFileBuilder
+    attr_reader :import
+    attr_accessor :failed_rows
+
+    def initialize(import)
+      @import = import
+      @failed_rows = []
+    end
+  
+    def build
+      return if failed_rows.blank?
+
+      create_or_append_to_csv
+      create_import_failed_file
+      destroy_temp_file
+    end
+
+    private
+
+    def create_or_append_to_csv
+      puts 'TEST!!!'
+      if import.failed_file.present?
+        puts 'APPEND!!!!'
+        append_rows_to_file
+      else
+        puts 'WRITE!!!!'
+        write_csv_file
+      end
+    end
+
+    def write_csv_file
+      CSV.open(temp_file_path, 'wb') do |csv|
+        csv << failed_rows.first.keys
+        insert_failed_rows(csv)
+      end
+    end
+
+    def append_rows_to_file
+      return if import.failed_file.blank?
+      CSV.open(import.failed_file_path, 'a+') do |csv|
+        insert_failed_rows(csv)
+      end
+    end
+
+
+    def insert_failed_rows(csv)
+      failed_rows.each do |hash|
+        csv << hash.values
+      end
+    end
+
+    def destroy_temp_file
+      return unless File.exists?(temp_file_path)
+      FileUtils.rm(temp_file_path)
+    end
+
+    def temp_file_path
+      "/tmp/#{target_file_name}"
+    end
+
+    def target_file_name
+      "failed_file_#{import.id}.csv"
+    end
+
+    def create_import_failed_file
+      return if import.failed_file.present?
+      File.open(temp_file_path) do |file|
+        import.failed_file = file
+
+        # I forced to save it as 'text/csv' because
+        # the file is being saved as 'text/x-pascal'
+        # and I still have no idea why?!?
+
+        import.failed_file_content_type = 'text/csv'
+        import.save!
+      end
+
+    end
+  end
+end

data/lib/active_record_importer/version.rb

@@ -1,3 +1,3 @@
 module ActiveRecordImporter
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/app/models/import.rb

@@ -8,6 +8,7 @@ module ActiveRecordImporter
               default: :upsert
 
     has_attached_file :file
+    has_attached_file :failed_file
 
     attr_accessor :execute_on_create
 
@@ -18,15 +19,22 @@ module ActiveRecordImporter
                              content_type: %w(text/plain text/csv)
                          }
 
-    after_create :execute, if: :execute_on_create
+    validates_attachment :failed_file,
+                         content_type: {
+                             content_type: %w(text/plain text/csv)
+                         }
 
-    # I'll add import options in the next release
+    # I'll add import options in the next major release
     # accepts_nested_attributes_for :import_options, allow_destroy: true
 
     def execute
       resource_class.import!(self, execute_on_create)
     end
 
+    def execute!
+      resource_class.import!(self, true)
+    end
+
     def resource_class
       resource.safe_constantize
     end
@@ -37,11 +45,20 @@ module ActiveRecordImporter
 
     ##
     # Override this if you prefer have
-    # a private permissions or you have
+    # private permissions or you have
     # private methods for reading files
     ##
     def import_file
-      local_path? ? file.path : file.url
+      local_path?(file) ? file.path : file.url
+    end
+
+    ##
+    # Override this method if you have
+    # private permissions or you have private methods
+    # for reading/writing uploaded files
+    ##
+    def failed_file_path
+      local_path?(failed_file) ? failed_file.path : failed_file.url
     end
 
     private
@@ -51,8 +68,8 @@ module ActiveRecordImporter
       errors.add(:find_options, "can't be blank") if find_options.blank?
     end
 
-    def local_path?
-      File.exist? import_file.file.path
+    def local_path?(f)
+      File.exist? f.path
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_record_importer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Nera
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-23 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -152,14 +152,15 @@ files:
 - bin/setup
 - db/migrate/1_active_record_importer_migration.rb
 - lib/active_record_importer.rb
-- lib/active_record_importer/attributes_builder.rb
+- lib/active_record_importer/attribute/attributes_builder.rb
+- lib/active_record_importer/attribute/find_options_builder.rb
+- lib/active_record_importer/attribute/helpers.rb
 - lib/active_record_importer/batch_importer.rb
 - lib/active_record_importer/data_processor.rb
 - lib/active_record_importer/dispatcher.rb
 - lib/active_record_importer/engine.rb
 - lib/active_record_importer/errors.rb
-- lib/active_record_importer/find_options_builder.rb
-- lib/active_record_importer/helpers.rb
+- lib/active_record_importer/failed_file_builder.rb
 - lib/active_record_importer/import_callbacker.rb
 - lib/active_record_importer/importable.rb
 - lib/active_record_importer/instance_builder.rb

data/lib/active_record_importer/attributes_builder.rb

@@ -1,67 +0,0 @@
-module ActiveRecordImporter
-  class AttributesBuilder
-    include Helpers
-
-    attr_reader :importable, :row_attrs, :processed_attrs
-
-    delegate :importer_options, to: :importable
-    delegate :importable_columns,
-             :default_attributes,
-             :find_assoc_opts,
-             to: :importer_options
-
-    def initialize(importable, row_attrs)
-      @importable = importable
-      @row_attrs = row_attrs
-      @processed_attrs = {}
-    end
-
-    def build
-      force_encode_attributes
-      fetch_time_attributes
-      processed_attrs
-    end
-
-    private
-
-    def default_attrs
-      def_attrs = { importing: true }
-      default_attributes.each do |key, value|
-        def_attrs[key] = fetch_value(value)
-      end
-    end
-
-    def force_encode_attributes
-      @processed_attrs = force_utf8_encode(merged_attributes)
-    end
-
-    def fetch_time_attributes
-      @processed_attrs.merge!(time_attributes(processed_attrs))
-    end
-
-    def fetch_value(value)
-      case value
-        when Proc
-          value.call(row_attrs)
-        when Symbol
-          importable.send(value, row_attrs)
-        else
-          value
-      end
-    end
-
-    def merged_attributes
-      attributes = row_attrs.slice(*importable_columns)
-      row_attrs = attributes.inject({}) do |row_attrs, key_value|
-        row_attrs[key_value.first] = key_value.last if key_value.last.present?
-        row_attrs
-      end
-      default_attrs.merge(row_attrs)
-    end
-
-    def has_column?(column)
-      return if column.blank?
-      importable_columns.include?(column.to_sym)
-    end
-  end
-end

data/lib/active_record_importer/find_options_builder.rb

@@ -1,52 +0,0 @@
-module ActiveRecordImporter
-  class FindOptionsBuilder
-    include Virtus.model
-    include Helpers
-
-    attribute :resource, String
-    attribute :attrs, Hash, default: {}
-    attribute :find_options, String
-    attribute :prefix, String
-
-    def build
-      get_find_opts
-      slice_attributes
-    end
-
-    private
-
-    def klass
-      resource.safe_constantize
-    end
-
-    delegate :importer_options, to: :klass
-
-    delegate :required_attributes, to: :importer_options
-
-    def get_find_opts
-      @options = strip_and_symbolize
-      @options ||= importer_options.find_options || required_attributes
-      @options
-    end
-
-    def slice_attributes
-      return attrs.slice(*@options).compact if prefix.blank?
-
-      @options.inject({}) do |attr, key|
-        attr[key] = attrs[prefixed_key(key)].presence
-        attr
-      end.compact
-    end
-
-    def prefixed_key(key)
-      "#{prefix}#{key}".to_sym
-    end
-
-    def strip_and_symbolize
-      return if find_options.blank?
-      find_options.split(',').map do |key|
-        key.strip.to_sym
-      end
-    end
-  end
-end

data/lib/active_record_importer/helpers.rb

@@ -1,26 +0,0 @@
-module ActiveRecordImporter
-  module Helpers
-
-    def parse_datetime(datetime = nil)
-      return if datetime.blank?
-      Time.parse(datetime)
-    end
-
-    def force_utf8_encode(data = {})
-      return data if data.blank?
-
-      data.keys.each do |key|
-        data[key] = data[key].force_encoding('UTF-8') if data[key].is_a?(String)
-      end
-
-      data
-    end
-
-    def time_attributes(data = {})
-      attrs = {}
-      attrs[:created_at] = parse_datetime(data[:created_at]) || Time.now
-      attrs[:updated_at] = parse_datetime(data[:updated_at]) || attrs[:created_at]
-      attrs
-    end
-  end
-end