active_admin_import 2.1.2 → 3.0.0.pre

data/lib/active_admin_import/importer.rb

@@ -2,47 +2,33 @@ require 'csv'
 module ActiveAdminImport
   class Importer
 
-    attr_reader :resource,  :options, :result, :headers, :csv_lines, :model
-
-    def store
-      result = @resource.transaction do
-        options[:before_batch_import].call(self) if options[:before_batch_import].is_a?(Proc)
-
-        result = resource.import headers.values, csv_lines, {
-            validate: options[:validate],
-            on_duplicate_key_update: options[:on_duplicate_key_update],
-            ignore: options[:ignore],
-            timestamps: options[:timestamps]
-        }
-        options[:after_batch_import].call(self) if options[:after_batch_import].is_a?(Proc)
-        result
-      end
-      {imported: csv_lines.count - result.failed_instances.count, failed: result.failed_instances}
-    end
+    attr_reader :resource, :options, :result, :model
+    attr_accessor :csv_lines, :headers
 
-    #
-    def prepare_headers(headers)
-      @headers = Hash[headers.zip(headers.map { |el| el.underscore.gsub(/\s+/, '_') })]
-      @headers.merge!(options[:headers_rewrites])
-      @headers
-    end
+    OPTIONS = [
+        :validate,
+        :on_duplicate_key_update,
+        :ignore,
+        :timestamps,
+        :before_import,
+        :after_import,
+        :before_batch_import,
+        :after_batch_import,
+        :headers_rewrites,
+        :batch_size,
+        :batch_transaction,
+        :csv_options
+    ].freeze
 
     def initialize(resource, model, options)
       @resource = resource
       @model = model
-      @options = {batch_size: 1000, validate: true}.merge(options)
       @headers = model.respond_to?(:csv_headers) ? model.csv_headers : []
-      @result= {failed: [], imported: 0}
-      if @options.has_key?(:col_sep) || @options.has_key?(:row_sep)
-        ActiveSupport::Deprecation.warn "row_sep and col_sep options are deprecated, use csv_options to override default CSV options"
-        @csv_options = @options.slice(:col_sep, :row_sep)
-      else
-        @csv_options = @options[:csv_options] || {}
-      end
-      #override csv options from model if it respond_to csv_options
-      @csv_options =  model.csv_options if model.respond_to?(:csv_options)
-      @csv_options.reject! {| key, value | value.blank? }
+      assign_options(options)
+    end
 
+    def import_result
+      @import_result ||= ImportResult.new
     end
 
     def file
@@ -51,28 +37,89 @@ module ActiveAdminImport
 
     def cycle(lines)
       @csv_lines = CSV.parse(lines.join, @csv_options)
-      @result.merge!(self.store) { |key, val1, val2| val1+val2 }
+      import_result.add(batch_import, lines.count)
     end
 
     def import
-      options[:before_import].call(self) if options[:before_import].is_a?(Proc)
-      lines = []
-      batch_size = options[:batch_size].to_i
+      run_callback(:before_import)
+      process_file
+      run_callback(:after_import)
+      import_result
+    end
+
+    def import_options
+      @import_options ||= options.slice(:validate, :on_duplicate_key_update, :ignore, :timestamps, :batch_transaction)
+    end
+
+    def batch_replace(header_key, options)
+      index = header_index(header_key)
+      csv_lines.map! do |line|
+        from = line[index]
+        line[index] = options[from] if options.has_key?(from)
+        line
+      end
+    end
+
+    def values_at(header_key)
+      csv_lines.collect { |line| line[header_index(header_key)] }.uniq
+    end
+
+    def header_index(header_key)
+      headers.values.index(header_key)
+    end
+
+    protected
+
+    def process_file
+      lines, batch_size = [], options[:batch_size].to_i
       File.open(file.path) do |f|
         # capture headers if not exist
-        prepare_headers(headers.any? ? headers : CSV.parse(f.readline, @csv_options).first)
+        prepare_headers { CSV.parse(f.readline, @csv_options).first }
         f.each_line do |line|
-          next if line.blank?
-          lines << line
+          lines << line if line.present?
           if lines.size == batch_size || f.eof?
-            cycle lines
+            cycle(lines)
             lines = []
           end
         end
       end
       cycle(lines) unless lines.blank?
-      options[:after_import].call(self) if options[:after_import].is_a?(Proc)
-      result
     end
+
+    def prepare_headers
+      headers = self.headers.present? ? self.headers.map(&:to_s) : yield
+      @headers = Hash[headers.zip(headers.map { |el| el.underscore.gsub(/\s+/, '_') })].with_indifferent_access
+      @headers.merge!(options[:headers_rewrites].symbolize_keys.slice(*@headers.symbolize_keys.keys))
+      @headers
+    end
+
+    def run_callback(name)
+      options[name].call(self) if options[name].is_a?(Proc)
+    end
+
+    def batch_import
+      batch_result = nil
+      @resource.transaction do
+        run_callback(:before_batch_import)
+        batch_result = resource.import(headers.values, csv_lines, import_options)
+        raise ActiveRecord::Rollback if import_options[:batch_transaction] && batch_result.failed_instances.any?
+        run_callback(:after_batch_import)
+      end
+      batch_result
+    end
+
+    def assign_options(options)
+      @options = {batch_size: 1000, validate: true}.merge(options.slice(*OPTIONS))
+      detect_csv_options
+    end
+
+    def detect_csv_options
+      @csv_options = if model.respond_to?(:csv_options)
+                       model.csv_options
+                     else
+                       options[:csv_options] || {}
+                     end.reject { |_, value| value.blank? }
+    end
+
   end
 end

data/lib/active_admin_import/model.rb

@@ -1,19 +1,23 @@
+# encoding: utf-8
+
+require 'rchardet'
+
 module ActiveAdminImport
   class Model
-    extend ActiveModel::Naming
-    include ActiveModel::Conversion
+
+    include ActiveModel::Model
     include ActiveModel::Validations
     include ActiveModel::Validations::Callbacks
 
-    validates :file, presence: {message: Proc.new { I18n.t('active_admin_import.no_file_error') }},
-              unless: proc { |me| me.new_record? }
-
-    validate :correct_content_type, if: proc { |me| me.file.present? }
-    validate :file_contents_present, if: proc { |me| me.file.present? }
+    validates :file, presence: {
+      message: ->(*_){ I18n.t("active_admin_import.no_file_error") }
+    }, unless: ->(me){ me.new_record? }
 
+    validate :correct_content_type, if: ->(me) { me.file.present? }
+    validate :file_contents_present, if: ->(me) { me.file.present? }
 
-    before_validation :uncompress_file, if: proc { |me| me.archive? && me.allow_archive?  }
-    before_validation :encode_file,  if: proc { |me| me.force_encoding? && me.file.present? }
+    before_validation :unzip_file, if: ->(me) { me.archive? && me.allow_archive? }
+    before_validation :encode_file, if: ->(me) { me.force_encoding? && me.file.present? }
 
     attr_reader :attributes
 
@@ -27,12 +31,7 @@ module ActiveAdminImport
       @attributes.merge!(args)
       @new_record = new_record
       args.keys.each do |key|
-        key = key.to_sym
-        #generate methods for instance object by attributes
-        singleton_class.class_eval do
-          define_method(key) { self.attributes[key] } unless method_defined? key
-          define_method("#{key}=") { |new_value| @attributes[key] = new_value } unless method_defined? "#{key}="
-        end
+        define_methods_for(key.to_sym)
       end if args.is_a?(Hash)
     end
 
@@ -41,11 +40,17 @@ module ActiveAdminImport
     end
 
     def default_attributes
-      {hint: '', file: nil, csv_headers: [], allow_archive: true, force_encoding: 'UTF-8'}
+      {
+        allow_archive: true,
+        csv_headers: [],
+        file: nil,
+        force_encoding: "UTF-8",
+        hint: ""
+      }
     end
 
     def allow_archive?
-      !!@attributes[:allow_archive]
+      !!attributes[:allow_archive]
     end
 
     def new_record?
@@ -53,11 +58,7 @@ module ActiveAdminImport
     end
 
     def force_encoding?
-      !!@attributes[:force_encoding]
-    end
-
-    def to_hash
-      @attributes
+      !!attributes[:force_encoding]
     end
 
     def persisted?
@@ -68,6 +69,8 @@ module ActiveAdminImport
       file_type == 'application/zip'
     end
 
+    alias :to_hash :attributes
+
     protected
 
     def file_path
@@ -79,23 +82,21 @@ module ActiveAdminImport
     end
 
     def encode_file
-      data = File.read(file_path).encode(force_encoding, invalid: :replace, undef: :replace)
+      data = File.read(file_path)
       File.open(file_path, 'w') do |f|
-          f.write(data)
+        f.write(encode(data))
       end
     end
 
-    def uncompress_file
+    def unzip_file
       Zip::File.open(file_path) do |zip_file|
-        self.file = Tempfile.new("active-admin-import-unzipped")
+        self.file = Tempfile.new('active-admin-import-unzipped')
         data = zip_file.entries.select { |f| f.file? }.first.get_input_stream.read
-        data = data.encode(force_encoding, invalid: :replace, undef: :replace) if self.force_encoding?
         self.file << data
         self.file.close
       end
     end
 
-
     def csv_allowed_types
       [
           'text/csv',
@@ -107,7 +108,6 @@ module ActiveAdminImport
       ]
     end
 
-
     def correct_content_type
       unless file.blank? || file.is_a?(Tempfile)
         errors.add(:file, I18n.t('active_admin_import.file_format_error')) unless csv_allowed_types.include? file_type
@@ -125,6 +125,57 @@ module ActiveAdminImport
         ''
       end
     end
+
+    protected
+
+    def define_methods_for(attr_name)
+      #generate methods for instance object by attributes
+      singleton_class.class_eval do
+        define_set_method(attr_name)
+        define_get_method(attr_name)
+      end
+    end
+
+    def encode(data)
+      data = content_encode(data) if force_encoding?
+      data = data.encode(
+        'UTF-8',
+        invalid: :replace, undef: :replace, universal_newline: true
+      )
+      begin
+        data.sub("\xEF\xBB\xBF", '')  # bom
+      rescue StandardError => _
+        data
+      end
+    end
+
+    def detect_encoding?
+      force_encoding == :auto
+    end
+
+    def dynamic_encoding(data)
+      CharDet.detect(data)['encoding']
+    end
+
+    def content_encode(data)
+      encoding_name = if detect_encoding?
+                        dynamic_encoding(data)
+                      else
+                        force_encoding.to_s
+                      end
+      data.force_encoding(encoding_name)
+    end
+
+    class <<self
+      def define_set_method(attr_name)
+        define_method(attr_name) { self.attributes[attr_name] } unless method_defined? attr_name
+      end
+
+      def define_get_method(attr_name)
+        define_method("#{attr_name}=") { |new_value| @attributes[attr_name] = new_value } unless method_defined? "#{attr_name}="
+      end
+    end
+
   end
 end
 

data/lib/active_admin_import/options.rb

@@ -0,0 +1,44 @@
+module ActiveAdminImport
+
+  module Options
+    VALID_OPTIONS = [
+        :back,
+        :csv_options,
+        :validate,
+        :batch_size,
+        :batch_transaction,
+        :before_import,
+        :after_import,
+        :before_batch_import,
+        :after_batch_import,
+        :on_duplicate_key_update,
+        :timestamps,
+        :ignore,
+        :template,
+        :template_object,
+        :resource_class,
+        :resource_label,
+        :plural_resource_label,
+        :headers_rewrites
+    ].freeze
+
+
+    def self.options_for(config, options= {})
+      options[:template_object] = ActiveAdminImport::Model.new  unless options.has_key? :template_object
+
+      {
+          back: {action: :import},
+          csv_options: {},
+          template: "admin/import",
+          resource_class: config.resource_class,
+          resource_label: config.resource_label,
+          plural_resource_label: config.plural_resource_label,
+          headers_rewrites: {}
+      }.deep_merge(options)
+
+
+
+    end
+
+  end
+end

data/lib/active_admin_import/version.rb

@@ -1,3 +1,3 @@
 module ActiveAdminImport
-  VERSION = "2.1.2"
+  VERSION = "3.0.0.pre"
 end

data/lib/active_admin_import.rb

@@ -2,9 +2,12 @@ require 'activerecord-import'
 require 'active_admin'
 require 'active_admin_import/version'
 require 'active_admin_import/engine'
+require 'active_admin_import/import_result'
+require 'active_admin_import/options'
 require 'active_admin_import/dsl'
 require 'active_admin_import/importer'
 require 'active_admin_import/model'
 require 'active_admin_import/authorization'
 ::ActiveAdmin::DSL.send(:include, ActiveAdminImport::DSL)
 
+

data/spec/fixtures/files/author.csv

@@ -0,0 +1,2 @@
+Name,Last name,Birthday
+John,Doe,1986-05-01

data/spec/fixtures/files/author_broken_header.csv

@@ -0,0 +1,2 @@
+Name,Second name,Birthday
+John,Doe,1986-05-01

data/spec/fixtures/files/author_invalid.csv

@@ -0,0 +1,2 @@
+Name,Last name,Birthday
+,Doe,1986-05-01

data/spec/fixtures/files/authors.csv

@@ -0,0 +1,3 @@
+Name,Last name,Birthday
+John,Doe,1986-05-01
+Jane,Roe,1988-11-16

data/spec/fixtures/files/authors_bom.csv

@@ -0,0 +1,3 @@
+Name,Last name,Birthday
+John,Doe,1986-05-01
+Jane,Roe,1988-11-16

data/spec/fixtures/files/authors_invalid_db.csv

@@ -0,0 +1,3 @@
+Name,Last name,Birthday
+John,Doe,1986-05-01
+John,Doe,1986-05-01

data/spec/fixtures/files/authors_invalid_model.csv

@@ -0,0 +1,3 @@
+Name,Last name,Birthday
+Jim,Doe,1986-05-01
+Jane,Roe,1986-05-01

data/spec/fixtures/files/authors_no_headers.csv

@@ -0,0 +1,2 @@
+John,Doe,1986-05-01
+Jane,Roe,1988-11-16

data/spec/fixtures/files/authors_win1251_win_endline.csv

@@ -0,0 +1,3 @@
+Name,Last name,Birthday
+�������,���������,1986-05-01
+Jane,Roe,1988-11-16

data/spec/fixtures/files/authors_with_ids.csv

@@ -0,0 +1,3 @@
+Id,Name,Last name,Birthday
+1,John,Doe,1986-05-01
+2,Jane,Roe,1988-11-16

data/spec/fixtures/files/authors_with_semicolons.csv

@@ -0,0 +1,3 @@
+Name;Last name;Birthday
+John;Doe;1986-05-01
+Jane;Roe;1988-11-16

data/spec/fixtures/files/only_headers.csv

@@ -0,0 +1 @@
+Name,Last name,Birthday

data/spec/fixtures/files/posts.csv

@@ -0,0 +1,4 @@
+"Title","Body","Author Name"
+"title1","some body","John"
+"title2","some body","Jane"
+

data/spec/fixtures/files/posts_for_author.csv

@@ -0,0 +1,3 @@
+"Title","Body"
+"title1","some body"
+"title2","some body"

data/spec/fixtures/files/posts_for_author_no_headers.csv

@@ -0,0 +1,2 @@
+"title1","some body"
+"title2","some body"

data/spec/import_result_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe ActiveAdminImport::ImportResult do
+  context "failed_message" do
+    let(:import_result) { ActiveAdminImport::ImportResult.new }
+
+    before do
+      Author.create(name: 'John', last_name: 'Doe')
+      Author.create(name: 'Jane', last_name: 'Roe')
+
+      @result = double \
+        failed_instances: [
+          Author.create(name: 'Jim', last_name: "Doe"), # {:last_name=>["has already been taken"]}
+          Author.create(name: nil,   last_name: 'Doe')  # {:name=>["can't be blank"], :last_name=>["has already been taken"]}
+        ]
+    end
+
+    it "should work without any failed instances" do
+      expect(import_result.failed_message).to eq("")
+    end
+
+    it "should work" do
+      import_result.add(@result, 4)
+      expect(import_result.failed_message).to eq("Last name has already been taken - Doe ; Name can't be blank - , Last name has already been taken - Doe")
+    end
+
+    it "should work on limit param" do
+      import_result.add(@result, 4)
+      expect(import_result.failed_message(limit: 1)).to eq("Last name has already been taken - Doe")
+    end
+  end
+end