activerecord_csv_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 62b46ebbaa1845d396dba93a645a0e380634c0f5
+  data.tar.gz: 6b3487666658e35b4b195bce03043165b1788675
+SHA512:
+  metadata.gz: 044fab5881d472092998ea2ca7c893b86ac1ff1671b8380ed94462793b93b7088e22e90c1a3f7df5b5dbe69d192c8690c5edcf7aa6a3d9d1be2475af42ee1a2d
+  data.tar.gz: 5be82ce068a1f0651aca75317dbe094d372ad6ee34730cada3cdccd8ddcec95d2fca170a7e54eb4e2066479f961862aefd62f2eae2ffc4417395dd14e38be7b9

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store
+/spec/examples.txt

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,20 @@
+Style/Documentation:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 100
+
+Metrics/MethodLength:
+  Max: 15
+
+Style/BlockDelimiters:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/StructInheritance:
+  Enabled: false
+
+Lint/HandleExceptions:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.13.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in activerecord_csv_importer.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,23 @@
+group :red_green_refactor, halt_on_fail: true do
+  guard :rspec, cmd: 'rspec' do
+    require 'guard/rspec/dsl'
+    dsl = Guard::RSpec::Dsl.new(self)
+
+    # Feel free to open issues for suggestions and improvements
+
+    # RSpec files
+    rspec = dsl.rspec
+    watch(rspec.spec_helper) { rspec.spec_dir }
+    watch(rspec.spec_support) { rspec.spec_dir }
+    watch(rspec.spec_files)
+
+    # Ruby files
+    ruby = dsl.ruby
+    dsl.watch_spec_files_for(ruby.lib_files)
+  end
+
+  guard :rubocop, all_on_start: false, cmd: 'rubocop', cli: '--format fuubar' do
+    watch(/.+\.rb/)
+    watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+  end
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Zulfiqar Ali
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,339 @@
+# ActiveRecordCSVImporter
+
+Using bulk inserts, the goal for ActiveRecordCSVImporter is to speed up the processing of large CSV filed
+
+## Differences from CSVImporter
+
+The key difference with ActiveRecordCSVImporter is the removal of callback support, since that is incompatible with bulk inserts.
+Additional config options are available instead to deal with batching and duplicate index support.
+
+It is compatible with ActiveRecord and any Databases supported by activerecord-import.
+
+## Usage tldr;
+
+Define your CSVImporter:
+
+```ruby
+class ImportUserCSV
+  include ActiveRecordCSVImporter
+
+  model User # an active record like model
+
+  column :email, to: ->(email) { email.downcase }, required: true
+  column :first_name, as: [ /first.?name/i, /pr(é|e)nom/i ]
+  column :last_name,  as: [ /last.?name/i, "nom" ]
+
+  on_duplicate_key(
+    on_duplicate_key_update: {
+      conflict_target: [:email], columns: [:first_name, :last_name]
+    }
+  )
+
+  batch_size 500
+
+  each_batch { |report|
+    puts report.total_count
+    puts report.completed_count
+    puts report.failed_rows
+  }
+end
+```
+
+Run the import:
+
+```ruby
+import = ImportUserCSV.new(file: my_file)
+
+import.valid_header?  # => false
+import.report.message # => "The following columns are required: email"
+
+# Assuming the header was valid, let's run the import!
+
+import.run!
+import.report.sucess? # => true
+import.report.message  # => "Import completed. 4 created, 2 updated, 1 failed to update"
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'activerecord_csv_importer'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem activerecord_csv_importer
+
+## Usage
+
+### Create an Importer
+
+Create a class and include `CSVImporter`.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+end
+```
+
+### Associate an active record model
+
+The `model` is can be a active record model.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+end
+```
+
+It can also be a relation which is handy to preset attributes:
+
+```ruby
+class User
+  scope :pending, -> { where(status: 'pending') }
+end
+
+class ImportUserCSV
+  include CSVImporter
+
+  model User.pending
+end
+```
+
+You can change the configuration at runtime to scope down to associated
+records.
+
+```ruby
+class Team
+  has_many :users
+end
+
+team = Team.find(1)
+
+ImportUserCSV.new(path: "tmp/my_file.csv") do
+  model team.users
+end
+```
+
+
+### Define columns and their mapping
+
+This is where the fun begins.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email
+end
+```
+
+This will map the column named email to the email attribute. By default,
+we downcase and strip the columns so it will work with a column spelled " EMail ".
+
+Now, email could also be spelled "e-mail", or "mail", or even "courriel"
+(oh, canada). Let's give it a couple of aliases then:
+
+
+```ruby
+  column :email, as: [/e.?mail/i, "courriel"]
+```
+
+Nice, emails should be downcased though, so let's do this.
+
+```ruby
+  column :email, as: [/e.?mail/i, "courriel"], to: ->(email) { email.downcase }
+```
+
+Now, what if the user does not provide the email column? It's not worth
+running the import, we should just reject the CSV file right away.
+That's easy:
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, required: true
+end
+
+import = ImportUserCSV.new(content: "name\nbob")
+import.valid_header? # => false
+import.report.status # => :invalid_header
+import.report.message # => "The following columns are required: 'email'"
+```
+
+
+### Upsert
+
+You usually want to prevent duplicates when importing a CSV file. activerecord-import provides ON CONFLICT support for MySQL, SQLite (IGNORE only), and PostgreSQL. See the activerecord-import wiki for detailed syntax.
+
+NOTE: If you have set up a unique index on a field and not set an appropriate ON CONFLICT resolution, activerecord_csv_import will raise an exception on the first duplicate insert.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, to: ->(email) { email.downcase }
+  column :first_name
+  column :last_name
+
+  on_duplicate_key(
+    on_duplicate_key_update: {
+      conflict_target: [:email], columns: [:first_name, :last_name]
+    }
+  )
+end
+```
+
+You are now done defining your importer, let's run it!
+
+### Import from a file, path or string
+
+You can import from a file, path or just the CSV content. Please note
+that we currently load the entire file in memory. Feel free to
+contribute if you need to support CSV files with millions of lines! :)
+
+```ruby
+import = ImportUserCSV.new(file: my_file)
+import = ImportUserCSV.new(path: "tmp/new_users.csv")
+import = ImportUserCSV.new(content: "email,name\nbob@example.com,bob")
+```
+
+### Overwrite configuration at runtime
+
+It is often needed to change the configuration at runtime, that's quite
+easy:
+
+```ruby
+team = Team.find(1)
+import = ImportUserCSV.new(file: my_file) do
+  model team.users
+end
+```
+
+### `each_batch` callback
+
+The number of rows to insert in the bulk query can be set by setting `batch_size` (default 500)
+
+The each batch callback is triggered after each batch is processed and returns the report object for the full process. This is generally useful when you want to display progress.
+
+```ruby
+progress_bar = ProgressBar.new
+
+UserImport.new(file: my_file) do
+  each_batch do |report|
+    progress_bar.increment(report.progress_percentage)
+  end
+end
+```
+Other available methods are:
+- total_count
+- completed_count
+- failed_rows
+
+### Validate the header
+
+On a web application, as soon as a CSV file is uploaded, you can check
+if it has the required columns. This is handy to fail early an provide
+the user with a meaningful error message right away.
+
+```ruby
+import = ImportUserCSV.new(file: params[:csv_file])
+import.valid_header? # => false
+import.report.message # => "The following columns are required: "email""
+```
+
+### Run the import and provide feedback to the user
+
+```ruby
+import = ImportUserCSV.new(file: params[:csv_file])
+import.run!
+import.report.message  # => "Import completed."
+```
+
+You can get your hands dirty and fetch the errored rows and the
+associated error message:
+
+```ruby
+import.report.invalid_rows.map { |row| [row.model.email, row.errors] }
+  # => [ ['INVALID_EMAIL', 'first_name', 'last_name', { 'email' => 'is not an email' }] ]
+```
+
+We do our best to map the errors back to the original column name. So
+with the following definition:
+
+```ruby
+  column :email, as: /e.?mail/i
+```
+
+and csv:
+
+```csv
+E-Mail,name
+INVALID_EMAIL,bob
+```
+
+The error returned should be: `{ "E-Mail" => "is invalid" }`
+
+### Custom quote char
+
+You can handle exotic quote chars with the `quote_char` option.
+
+```csv
+email,name
+bob@example.com,'bob "elvis" wilson'
+```
+
+```ruby
+import = ImportUserCSV.new(content: csv_content)
+import.run!
+import.report.status
+  # => :invalid_csv_file
+import.report.messages
+  # => CSV::MalformedCSVError: Illegal quoting in line 2.
+```
+
+Let's provide a valid quote char:
+
+```ruby
+import = ImportUserCSV.new(content: csv_content, quote_char: "'")
+import.run!
+  # => [ ["bob@example.com", "bob \"elvis\" wilson"] ]
+```
+
+### Custom encoding
+
+You can handle exotic encodings with the `encoding` option.
+
+```ruby
+ImportUserCSV.new(content: "メール,氏名".encode('SJIS'), encoding: 'SJIS:UTF-8')
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/activerecord_csv_importer.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/activerecord_csv_importer.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'activerecord_csv_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'activerecord_csv_importer'
+  spec.version       = ActiveRecordCSVImporter::VERSION
+  spec.authors       = ['Zulfiqar Ali']
+  spec.email         = ['desheikh@gmail.com']
+
+  spec.summary       = 'A modified version of CSV Import using activerecord-import'
+  spec.homepage      = 'https://github.com/desheikh/activerecord_csv_importer'
+  spec.license       = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'virtus', '~> 1.0'
+  spec.add_dependency 'activerecord-import', '~> 0.16'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 10.5'
+
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rubocop', '~> 0.45'
+
+  spec.add_development_dependency 'guard-rspec', '~> 4.7'
+  spec.add_development_dependency 'guard-rubocop', '~> 1.2'
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'activerecord_csv_importer'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/activerecord_csv_importer/column.rb

@@ -0,0 +1,10 @@
+module ActiveRecordCSVImporter
+  # A Column from a CSV file with a `name` (from the csv file) and a matching
+  # `ColumnDefinition` if any.
+  class Column
+    include Virtus.model
+
+    attribute :name, String
+    attribute :definition, ColumnDefinition
+  end
+end

data/lib/activerecord_csv_importer/column_definition.rb

@@ -0,0 +1,67 @@
+module ActiveRecordCSVImporter
+  # Define a column. Called from the DSL via `column.
+  #
+  # Examples
+  #
+  #   # the csv column "email" will be assigned to the `email` attribute
+  #   column :email
+  #
+  #   # the csv column matching /email/i will be assigned to the `email` attribute
+  #   column :email, as: /email/i
+  #
+  #   # the csv column matching "First name" or "Prénom" will be assigned to the
+  #   `first_name` attribute
+  #   column :first_name, as: [/first ?name/i, /pr(é|e)nom/i]
+  #
+  #   # the csv column "first_name" will be assigned to the `f_name` attribute
+  #   column :first_name, to: :f_name
+  #
+  #   # email will be downcased
+  #   column :email, to: ->(email) { email.downcase }
+  #
+  #   # transform `confirmed` to `confirmed_at`
+  #   column :confirmed, to: ->(confirmed, model) do
+  #     model.confirmed_at = confirmed == "true" ? Time.new(2012) : nil
+  #   end
+  #
+  class ColumnDefinition
+    include Virtus.model
+
+    attribute :name, Symbol
+    attribute :to # Symbol or Proc
+    attribute :as # Symbol, String, Regexp, Array
+    attribute :required, Boolean
+
+    # The model attribute that this column targets
+    def attribute
+      if to.is_a?(Symbol)
+        to
+      else
+        name
+      end
+    end
+
+    # rubocop:disable all
+    # Return true if column definition matches the column name passed in.
+    def match?(column_name, search_query = (as || name))
+      return false if column_name.nil?
+
+      downcased_column_name = column_name.downcase
+      underscored_column_name = downcased_column_name.gsub(/\s+/, '_')
+
+      case search_query
+      when Symbol
+        underscored_column_name == search_query.to_s
+      when String
+        downcased_column_name == search_query.downcase
+      when Regexp
+        column_name =~ search_query
+      when Array
+        search_query.any? { |query| match?(column_name, query) }
+      else
+        raise Error, "Invalid `as`. Should be a Symbol, String, Regexp or Array - was #{as.inspect}"
+      end
+    end
+    # rubocop:enable all
+  end
+end

data/lib/activerecord_csv_importer/config.rb

@@ -0,0 +1,16 @@
+module ActiveRecordCSVImporter
+  # The configuration of a ActiveRecordCSVImporter
+  class Config
+    include Virtus.model
+
+    attribute :model
+    attribute :column_definitions, Array[ColumnDefinition], default: proc { [] }
+    attribute :on_duplicate_key, Hash, default: []
+    attribute :batch_size, Integer, default: 500
+    attribute :each_batch_blocks, Array[Proc], default: []
+
+    def each_batch(block)
+      each_batch_blocks << block
+    end
+  end
+end

data/lib/activerecord_csv_importer/csv_reader.rb

@@ -0,0 +1,74 @@
+module ActiveRecordCSVImporter
+  # Reads, sanitize and parse a CSV file
+  class CSVReader
+    include Virtus.model
+
+    attribute :content, String
+    attribute :file # IO
+    attribute :path, String
+    attribute :quote_char, String, default: '"'
+    attribute :encoding, String, default: 'UTF-8:UTF-8'
+
+    def csv_rows
+      @csv_rows ||= begin
+        sane_content = sanitize_content(read_content)
+        separator = detect_separator(sane_content)
+        cells = CSV.parse(
+          sane_content,
+          col_sep: separator,
+          quote_char: quote_char,
+          skip_blanks: true,
+          encoding: encoding
+        )
+        sanitize_cells(cells)
+      end
+    end
+
+    # Returns the header as an Array of Strings
+    def header
+      @header ||= csv_rows.first
+    end
+
+    # Returns the rows as an Array of Arrays of Strings
+    def rows
+      @rows ||= csv_rows[1..-1]
+    end
+
+    private
+
+    def read_content
+      if content
+        content
+      elsif file
+        file.read
+      elsif path
+        File.open(path).read
+      else
+        raise Error, 'Please provide content, file, or path'
+      end
+    end
+
+    def sanitize_content(csv_content)
+      internal_encoding = encoding.split(':').last
+      # Remove invalid byte sequences
+      csv_content
+        .encode(Encoding.find(internal_encoding), invalid: :replace, undef: :replace, replace: '')
+        .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n"
+    end
+
+    SEPARATORS = [',', ';', "\t"].freeze
+
+    def detect_separator(csv_content)
+      SEPARATORS.sort_by { |separator| csv_content.count(separator) }.last
+    end
+
+    # Remove trailing white spaces and ensure we always return a string
+    def sanitize_cells(rows)
+      rows.map do |cells|
+        cells.map do |cell|
+          cell ? cell.strip : ''
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_csv_importer/dsl.rb

@@ -0,0 +1,25 @@
+module ActiveRecordCSVImporter
+  # This Dsl extends a class that includes ActiveRecordCSVImporter
+  # It is a thin proxy to the Config object
+  module Dsl
+    def model(model_klass)
+      config.model = model_klass
+    end
+
+    def column(name, options = {})
+      config.column_definitions << options.merge(name: name)
+    end
+
+    def on_duplicate_key(options)
+      config.on_duplicate_key = options
+    end
+
+    def batch_size(size)
+      config.batch_size = size
+    end
+
+    def each_batch(&block)
+      config.each_batch(block)
+    end
+  end
+end

data/lib/activerecord_csv_importer/header.rb

@@ -0,0 +1,57 @@
+module ActiveRecordCSVImporter
+  # The CSV Header
+  class Header
+    include Virtus.model
+
+    attribute :column_definitions, Array[ColumnDefinition]
+    attribute :column_names, Array[String]
+
+    def columns
+      column_names.map do |column_name|
+        Column.new(
+          name: column_name,
+          definition: find_column_definition(column_name)
+        )
+      end
+    end
+
+    def column_name_for_model_attribute(attribute)
+      column = columns.find { |c|
+        c.definition.attribute == attribute if c.definition
+      }
+      column.name if column
+    end
+
+    def valid?
+      missing_required_columns.empty?
+    end
+
+    # Returns Array[String]
+    def required_columns
+      column_definitions.select(&:required?).map(&:name)
+    end
+
+    # Returns Array[String]
+    def missing_required_columns
+      (column_definitions.select(&:required?) - columns.map(&:definition))
+        .map(&:name).map(&:to_s)
+    end
+
+    # Returns Array[String]
+    def missing_columns
+      (column_definitions - columns.map(&:definition)).map(&:name).map(&:to_s)
+    end
+
+    private
+
+    def find_column_definition(name)
+      column_definitions.find { |column_definition|
+        column_definition.match?(name)
+      }
+    end
+
+    def column_definition_names
+      column_definitions.map(&:name).map(&:to_s)
+    end
+  end
+end

data/lib/activerecord_csv_importer/report.rb

@@ -0,0 +1,92 @@
+module ActiveRecordCSVImporter
+  # The Report you get back from an import.
+  #
+  # * It has a status (pending, invalid_csv_file, invalid_header, in_progress, done, aborted)
+  # * It lists out missing columns
+  # * It reports parser_error
+  # * It lists out (created / updated) * (success / failed) records
+  # * It provides a human readable message
+  #
+  class Report
+    include Virtus.model
+
+    attribute :status, Symbol, default: proc { :pending }
+
+    attribute :missing_columns, Array[Symbol], default: proc { [] }
+
+    attribute :parser_error, String
+
+    attribute :total_count, Integer, default: 0
+    attribute :completed_count, Integer, default: 0
+    attribute :invalid_rows, Array[Array], default: {}
+
+    attribute :message_generator, Class, default: proc { ReportMessage }
+
+    def progress_percentage
+      return 0 if total_count.zero?
+      (completed_count.to_f / total_count * 100).round
+    end
+
+    def success?
+      done? && invalid_rows.empty?
+    end
+
+    def pending?
+      status == :pending
+    end
+
+    def in_progress?
+      status == :in_progress
+    end
+
+    def done?
+      status == :done
+    end
+
+    def aborted?
+      status == :aborted
+    end
+
+    def invalid_header?
+      status == :invalid_header
+    end
+
+    def invalid_csv_file?
+      status == :invalid_csv_file
+    end
+
+    def pending!
+      self.status = :pending
+      self
+    end
+
+    def in_progress!
+      self.status = :in_progress
+      self
+    end
+
+    def done!
+      self.status = :done
+      self
+    end
+
+    def aborted!
+      self.status = :aborted
+      self
+    end
+
+    def invalid_header!
+      self.status = :invalid_header
+      self
+    end
+
+    def invalid_csv_file!
+      self.status = :invalid_csv_file
+      self
+    end
+
+    def message
+      message_generator.call(self)
+    end
+  end
+end

data/lib/activerecord_csv_importer/report_message.rb

@@ -0,0 +1,44 @@
+module ActiveRecordCSVImporter
+  # Generate a human readable message for the given report.
+  class ReportMessage
+    def self.call(report)
+      new(report).to_s
+    end
+
+    def initialize(report)
+      @report = report
+    end
+
+    attr_accessor :report
+
+    def to_s
+      send("report_#{report.status}")
+    end
+
+    private
+
+    def report_pending
+      "Import hasn't started yet"
+    end
+
+    def report_in_progress
+      'Import in progress'
+    end
+
+    def report_done
+      'Import completed'
+    end
+
+    def report_invalid_header
+      "The following columns are required: #{report.missing_columns.join(', ')}"
+    end
+
+    def report_invalid_csv_file
+      report.parser_error
+    end
+
+    def report_aborted
+      'Import aborted'
+    end
+  end
+end

data/lib/activerecord_csv_importer/row.rb

@@ -0,0 +1,40 @@
+module ActiveRecordCSVImporter
+  # A Row from the CSV file.
+  #
+  # returns a formatted version of the row based of the to proc
+  class Row
+    include Virtus.model
+
+    attribute :header, Header
+    attribute :row_array, Array[String]
+
+    # A hash with this row's attributes
+    def csv_attributes
+      @csv_attributes ||= Hash[header.column_names.zip(row_array)]
+    end
+
+    def to_a
+      header.columns.each_with_object([]) do |column, memo|
+        value = csv_attributes[column.name]
+        begin
+          value = value.dup if value
+        rescue TypeError
+          # can't dup Symbols, Integer etc...
+        end
+
+        next if column.definition.nil?
+
+        memo << get_attribute(column.definition, value)
+      end
+    end
+
+    # Set the attribute using the column_definition and the csv_value
+    def get_attribute(column_definition, csv_value)
+      if column_definition.to && column_definition.to.is_a?(Proc)
+        column_definition.to.call(csv_value)
+      else
+        column_definition.attribute
+      end
+    end
+  end
+end

data/lib/activerecord_csv_importer/runner.rb

@@ -0,0 +1,66 @@
+module ActiveRecordCSVImporter
+  # Do the actual import.
+  #
+  # It iterates over the rows' models and persist them. It returns a `Report`.
+  class Runner
+    def self.call(*args)
+      new(*args).call
+    end
+
+    include Virtus.model
+
+    attribute :model
+    attribute :header
+    attribute :rows, Array[CSV::Row]
+    attribute :config, Object
+
+    attribute :report, Report, default: proc { Report.new }
+
+    # Persist the csv rows and return a Report
+    def call
+      report.total_count = rows.count
+      report.in_progress!
+      persist_rows!
+      report.done!
+
+      report
+    end
+
+    private
+
+    def persist_rows!
+      rows.in_groups_of(config.batch_size, false) do |set|
+        response = import_rows(set)
+
+        add_to_report(response, set)
+        config.each_batch_blocks.each { |block| block.call(report) }
+      end
+    end
+
+    def import_rows(set)
+      config.model.import(header.column_names.dup, set, config.on_duplicate_key)
+    end
+
+    def add_to_report(response, set)
+      report.completed_count += set.length
+
+      response.failed_instances.each do |model|
+        report.invalid_rows << (columns_for(model) << build_error(model))
+      end
+    end
+
+    def columns_for(model)
+      model.attributes.values_at(*header.column_names)
+    end
+
+    # Error from the model mapped back to the CSV header if we can
+    def build_error(model)
+      Hash[
+        model.errors.map do |attribute, errors|
+          column_name = header.column_name_for_model_attribute(attribute)
+          column_name ? [column_name, errors] : [attribute, errors]
+        end
+      ]
+    end
+  end
+end

data/lib/activerecord_csv_importer/version.rb

@@ -0,0 +1,3 @@
+module ActiveRecordCSVImporter
+  VERSION = '0.1.0'.freeze
+end

data/lib/activerecord_csv_importer.rb

@@ -0,0 +1,90 @@
+require 'csv'
+require 'virtus'
+
+require 'activerecord_csv_importer/version'
+require 'activerecord_csv_importer/csv_reader'
+require 'activerecord_csv_importer/column_definition'
+require 'activerecord_csv_importer/column'
+require 'activerecord_csv_importer/header'
+require 'activerecord_csv_importer/row'
+require 'activerecord_csv_importer/report'
+require 'activerecord_csv_importer/report_message'
+require 'activerecord_csv_importer/runner'
+require 'activerecord_csv_importer/config'
+require 'activerecord_csv_importer/dsl'
+
+module ActiveRecordCSVImporter
+  class Error < StandardError; end
+
+  # Setup DSL and config object
+  def self.included(klass)
+    klass.extend(Dsl)
+    klass.define_singleton_method(:config) do
+      @config ||= Config.new
+    end
+  end
+
+  # Instance level config will run against this configurator
+  class Configurator < Struct.new(:config)
+    include Dsl
+  end
+
+  # Defines the path, file or content of the csv file.
+  # Also allows you to overwrite the configuration at runtime.
+  #
+  # Example:
+  #
+  #   .new(file: my_csv_file)
+  #   .new(path: "subscribers.csv", model: newsletter.subscribers)
+  #
+  def initialize(*args, &block)
+    @csv = CSVReader.new(*args)
+    @config = self.class.config.dup
+    @config.attributes = args.last
+    @report = Report.new
+    Configurator.new(@config).instance_exec(&block) if block
+  end
+
+  attr_reader :csv, :report, :config
+
+  # Initialize and return the `Header` for the current CSV file
+  def header
+    @header ||= Header.new(
+      column_definitions: config.column_definitions,
+      column_names: csv.header
+    )
+  end
+
+  # Initialize and return the `Row`s for the current CSV file
+  def rows
+    csv.rows.map { |row_array|
+      Row.new(header: header, row_array: row_array).to_a
+    }
+  end
+
+  def valid_header?
+    if @report.pending?
+      if header.valid?
+        @report = Report.new(status: :pending)
+      else
+        @report = Report.new(
+          status: :invalid_header,
+          missing_columns: header.missing_required_columns
+        )
+      end
+    end
+
+    header.valid?
+  end
+
+  # Run the import. Return a Report.
+  def run!
+    if valid_header?
+      @report = Runner.call(header: header, rows: rows, config: config)
+    else
+      @report
+    end
+  rescue CSV::MalformedCSVError => e
+    @report = Report.new(status: :invalid_csv_file, parser_error: e.message)
+  end
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: activerecord_csv_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Zulfiqar Ali
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: virtus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: 
+email:
+- desheikh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- activerecord_csv_importer.gemspec
+- bin/console
+- bin/setup
+- lib/activerecord_csv_importer.rb
+- lib/activerecord_csv_importer/column.rb
+- lib/activerecord_csv_importer/column_definition.rb
+- lib/activerecord_csv_importer/config.rb
+- lib/activerecord_csv_importer/csv_reader.rb
+- lib/activerecord_csv_importer/dsl.rb
+- lib/activerecord_csv_importer/header.rb
+- lib/activerecord_csv_importer/report.rb
+- lib/activerecord_csv_importer/report_message.rb
+- lib/activerecord_csv_importer/row.rb
+- lib/activerecord_csv_importer/runner.rb
+- lib/activerecord_csv_importer/version.rb
+homepage: https://github.com/desheikh/activerecord_csv_importer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A modified version of CSV Import using activerecord-import
+test_files: []