active_admin_import 2.1.2 → 3.0.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8f417cc9a85210d9b683c9153e081542a882820
-  data.tar.gz: d91324749b99737df1100043f3d33aeeb1b81e25
+  metadata.gz: 46875662660bbc61d8dce4fa218234229d0c94d7
+  data.tar.gz: ba7347ef7348492d137e459e52a3430d5607f051
 SHA512:
-  metadata.gz: afe63356cf3a67b25940824541b1e2dc4c7ae752fa7f93ac9d1e4a2403f2f963c0c4f081cc91d1cdf69e3c7e796e7ce4165dee9e8b6f31d762ed18209f45beba
-  data.tar.gz: a3bd48e057590d3c85c01738f2399ae578153fdd2661eaba3f6124e3b5c87cbbab6ca2dbd50743cbdc377cb2b6a1aebc313d3a8cd0456db086e34a036b8b2f1d
+  metadata.gz: 390733a4c6297317e6ab84c0528d05a65e93b422c1ac84961cd38f302a3f859d5303852a937dbf88762e599a8b5d4d845f8597cdcfc50a9ed3f43e1125d86192
+  data.tar.gz: 459e49db7a02bdca7c5fa53d9cad1b7b8e2937ff2e223b479ee7551a8cd9b7e6993f5b4968c1a5c86e33f206467075afd9d7c7bb7c868906f4af050cae2b2754

data/.hound.yml

@@ -0,0 +1,4 @@
+ruby:
+  enabled: true
+StringLiterals:
+  EnforcedStyle: single_quotes  

data/.travis.yml

@@ -0,0 +1,9 @@
+script: bundle exec rspec spec
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.5
+  - 2.2.0
+before_install:
+  - gem update --system
+  - gem --version

data/Gemfile

@@ -2,3 +2,19 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in active_admin_importable.gemspec
 gemspec
+group :test do
+  gem 'rails',  '4.1.9'
+  gem 'rspec-rails'
+  gem 'activeadmin', github: 'activeadmin'  , ref: '54bede0558a99ab759f98f9b24e1b0144063a81e'
+
+  gem 'coveralls', require: false # Test coverage website. Go to https://coveralls.io
+
+  gem 'devise'
+  gem 'sass-rails'
+  gem 'sqlite3'
+  gem 'launchy'
+  gem 'database_cleaner'
+  gem 'capybara'
+  gem 'selenium-webdriver'
+  gem 'poltergeist'
+end

data/README.md

@@ -1,9 +1,18 @@
 # ActiveAdminImport 
-The most fastest and efficient CSV import for Active Admin (based on activerecord-import gem) 
-with support of validations and bulk inserts 
+[The most fastest and efficient CSV import for Active Admin
+with support of validations, bulk inserts and encodings handling](http://activeadmin-plugins.github.io/active_admin_import/)
+ 
+
+
+[![Build Status](https://img.shields.io/travis/activeadmin-plugins/active_admin_import.svg)](https://travis-ci.org/activeadmin-pluginsl/active_admin_import)
+[![Dependency Status](http://img.shields.io/gemnasium/activeadmin-plugins/active_admin_import.svg)](https://gemnasium.com/activeadmin-plugins/active_admin_import)
+[![Coverage Status](https://coveralls.io/repos/activeadmin-plugins/active_admin_import/badge.svg)](https://coveralls.io/r/activeadmin-plugins/active_admin_import)
 
+[![Code Climate](http://img.shields.io/codeclimate/github/activeadmin-plugins/active_admin_import.svg)](https://codeclimate.com/github/activeadmin-plugins/active_admin_import)
+[![Gem Version](http://img.shields.io/gem/v/active_admin_import.svg)](https://rubygems.org/gems/active_admin_import)
+[![License](http://img.shields.io/:license-mit-blue.svg)](http://Fivell.mit-license.org)
 
-Should work with both AA 0.6 and 1.0.0
+master can be used with AA 1.0.0 and Rails >= 4.1
 
 
 #Installation
@@ -11,203 +20,94 @@ Should work with both AA 0.6 and 1.0.0
 Add this line to your application's Gemfile:
 
 ```ruby
-gem "active_admin_import" , '2.1.1'
-```
-	
-And then execute:
-
-    $ bundle
-
-
+gem "active_admin_import" , '2.1.2'
 
-Why yet another import for ActiveAdmin ? Now with activerecord-import ....
-
-    "Because plain-vanilla, out-of-the-box ActiveRecord doesn’t provide support for inserting large amounts of data efficiently"
-
-cool features of activerecord-import
-
-    activerecord-import can perform validations (fast)
-    activerecord-import can perform on duplicate key updates (requires mysql)
-
-
-So active_admin_import features
+```
+or
 
-    Encoding handling
-    Two step importing (see example2)
-    CSV options
-    Ability to prepend CSV headers automatically
-    Bulk import (activerecord-import)
-    Ability to customize template 
-    Callbacks support
-    Zip files import
-    and more ....
+```ruby
+gem "active_admin_import" , github: "activeadmin-plugins/active_admin_import"
 
-Options
+```
 
-    # +back+:: resource action to redirect after processing
-    # +csv_options+:: hash with column separator, row separator, etc 
-    # +validate+:: true|false, means perfoem validations or not
-    # +batch_size+:: integer value of max  record count inserted by 1 query/transaction
-    # +before_import+:: proc for before import action, hook called with  importer object
-    # +after_import+:: proc for after import action, hook called with  importer object
-    # +before_batch_import+:: proc for before each batch action, called with  importer object
-    # +after_batch_import+:: proc for after each batch action, called with  importer object
-    # +on_duplicate_key_update+:: an Array or Hash, tells activerecord-import to use MySQL's ON DUPLICATE KEY UPDATE ability.
-    # +timestamps+::  true|false, tells activerecord-import to not add timestamps (if false) even if record timestamps is disabled in ActiveRecord::Base
-    # +ignore+::  true|false, tells activerecord-import toto use MySQL's INSERT IGNORE ability
-    # +params_keys+:: params values available in callbacks
-    # +template+:: custom template rendering
-    # +template_object+:: object passing to view
-    # +locals+:: local variables for template
-    # +resource_class+:: resource class name
-    # +resource_label+:: resource label value
-    # +headers_rewrites+:: hash with key (csv header) - value (db column name) rows mapping
+And then execute:
 
+    $ bundle
 
 
-Default options values
+# active_admin_import features
+<ol>
+  <li> Replacements/Updates support</li>
+  <li> Encoding handling</li>
+  <li> CSV options</li>
+  <li> Ability to describe/change CSV headers</li>
+  <li> Bulk import (activerecord-import)</li>
+  <li> Callbacks</li>
+  <li> Zip files</li>
+  <li> and more...</li>
+</ol>
 
-```ruby    
-    back: {action: :import},
-    csv_options: {},
-    template: "admin/import",
-    fetch_extra_options_from_params: [],
-    resource_class: config.resource_class,
-    resource_label:  config.resource_label,
-    plural_resource_label: config.plural_resource_label,
-```    
+   
 
-Example1 
+#### Basic usage
 
-```ruby  
-    ActiveAdmin.register Post  do
-       active_admin_import :validate => false,
-                            :csv_options => {:col_sep => ";" },
-                            :before_import => proc{ Post.delete_all},
-                            :batch_size => 1000
-    
-    
-    end
+```ruby
+ActiveAdmin.register Post
+  active_admin_import options
+end
 ```
 
 
-Example2 Importing to mediate table with insert select operation after import completion
-
-This config allows to replace data without downtime
+#### Options
+Tool                    | Description
+---------------------   | -----------
+:back					|resource action to redirect after processing
+:csv_options			|hash with column separator, row separator, etc 
+:validate				|bool means perform validations or not
+:batch_size				|integer value of max  record count inserted by 1 query/transaction
+:batch_transaction    |bool (false by default), if transaction is used when batch importing and works when :validate is set to true
+:before_import			|proc for before import action, hook called with  importer object
+:after_import			|proc for after import action, hook called with  importer object
+:before_batch_import	|proc for before each batch action, called with  importer object
+:after_batch_import		|proc for after each batch action, called with  importer object
+:on_duplicate_key_update|an Array or Hash, tells activerecord-import to use MySQL's ON DUPLICATE KEY UPDATE ability.
+:timestamps				|bool, tells activerecord-import to not add timestamps (if false) even if record timestamps is disabled in ActiveRecord::Base
+:ignore					|bool, tells activerecord-import toto use MySQL's INSERT IGNORE ability
+:template				|custom template rendering
+:template_object		|object passing to view
+:resource_class			|resource class name
+:resource_label			|resource label value
+:plural_resource_label	|pluralized resource label value (default config.plural_resource_label)
+:headers_rewrites		|hash with key (csv header) - value (db column name) rows mapping
 
-```ruby
-    ActiveAdmin.register Post  do
-        active_admin_import :validate => false,
-            :csv_options => {:col_sep => ";" },
-            :resource_class => ImportedPost ,  # we import data into another resource
-            :before_import => proc{ ImportedPost.delete_all },
-            :after_import => proc{
-                Post.transaction do
-                    Post.delete_all
-                    Post.connection.execute("INSERT INTO posts (SELECT * FROM import_posts)")
-                end
-            },
-            :back => proc { config.namespace.resource_for(Post).route_collection_path } # redirect to post index
-    end
-```
 
 
-Example3 Importing file without headers, but we always know file format, so we can predefine it
-
-```ruby
-    ActiveAdmin.register Post  do
-        active_admin_import :validate => true,
-            :template_object => ActiveAdminImport::Model.new(
-                :hint => "file will be imported with such header format: 'body','title','author'",
-                :csv_headers => ["body","title","author"] 
-            )
-    end
-```
- 
-Example4 Importing without forcing to UTF-8 and disallow archives
+#### Wiki
 
+[Check various examples](https://github.com/activeadmin-plugins/active_admin_import/wiki)
 
-```ruby
-    ActiveAdmin.register Post  do
-        active_admin_import :validate => true,
-            :template_object => ActiveAdminImport::Model.new(
-                :hint => "file will be encoded to ISO-8859-1",
-                :force_encoding => "ISO-8859-1",
-                :allow_archive => false  
-            )
-    end
-```
+## Dependencies
 
+Tool                  | Description
+--------------------- | -----------
+[rchardet]            | Character encoding auto-detection in Ruby. As smart as your browser. Open source.
+[activerecord-import] | Powerful library for bulk inserting data using ActiveRecord.
 
-Example5 Callbacks for each bulk insert iteration
+[rchardet]: https://github.com/jmhodges/rchardet
+[activerecord-import]: https://github.com/zdennis/activerecord-import
 
 
-```ruby
-    ActiveAdmin.register Post  do
-        active_admin_import :validate => true,
-        :before_batch_import => proc { |import|
-           import.file #current file used
-           import.resource #ActiveRecord class to import to
-           import.options # options
-           import.result # result before bulk iteration
-           import.headers # CSV headers
-           import.csv_lines #lines to import
-           import.model #template_object instance
-        },
-        :after_batch_import => proc{ |import|
-           #the same
-        }
-    end
-```    
-    
-Example6 dynamic CSV options, template overriding
-
-1) put overrided template to app/views/import.html.erb
-
-```erb
-
-    <p>
-       <%= raw(@active_admin_import_model.hint) %> 
-    </p>
-    <%= semantic_form_for @active_admin_import_model, url: {action: :do_import}, html: {multipart: true} do |f| %>
-        <%= f.inputs do %>
-            <%= f.input :file, as: :file %>
-        <% end %>
-        <%= f.inputs "CSV options", :for => [:csv_options, OpenStruct.new(@active_admin_import_model.csv_options)] do |csv| %>
-            <% csv.with_options :input_html => {:style => 'width:40px;'} do |opts| %>
-                <%= opts.input :col_sep %>
-                <%= opts.input :row_sep %>
-                <%= opts.input :quote_char %>
-            <% end %>
-        <% end %>
-    
-        <%= f.actions do %>
-            <%= f.action :submit, label: t("active_admin_import.import_btn"), button_html: {disable_with: t("active_admin_import.import_btn_disabled")} %>
-        <% end %>
-    <% end %>
-    
-```
+## Contributing
 
-2) call method with following parameters
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
 
-```ruby
-    ActiveAdmin.register Post  do
-        active_admin_import :validate => false,
-                          :template => 'import' ,
-                          :template_object => ActiveAdminImport::Model.new(
-                              :hint => "specify CSV options"
-                              :csv_options => {:col_sep => ";", :row_sep => nil, :quote_char => nil} 
-                          )
-    end                      
-```
 
-#Links
-https://github.com/gregbell/active_admin
 
-https://github.com/zdennis/activerecord-import
 
-#Source Doc
-http://rubydoc.info/gems/active_admin_import/2.1.1/
 
 
 

data/Rakefile

@@ -1,2 +1,7 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"
+require "bundler"
+require 'rake'
+Bundler.setup
+Bundler::GemHelper.install_tasks
+
+# Import all our rake tasks
+FileList['tasks/**/*.rake'].each { |task| import task }

data/active_admin_import.gemspec

@@ -15,9 +15,11 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version = ActiveAdminImport::VERSION 
 
-  gem.add_runtime_dependency 'activerecord-import',  '~> 0.4', '>= 0.4.1'
-  gem.add_runtime_dependency 'activeadmin', '>= 0.6.0'
-  gem.add_runtime_dependency 'rubyzip', '~> 1.0', '>= 1.0.0'
 
+  gem.add_runtime_dependency 'activerecord-import', '~> 0.8.0'
+  gem.add_runtime_dependency 'rchardet', '~> 1.5'
+
+  gem.add_runtime_dependency 'rubyzip', '~> 1.0', '>= 1.0.0'
+  gem.add_dependency "rails", ">= 4.0"
 
 end

data/app/views/admin/import.html.erb

@@ -2,7 +2,7 @@
    <%= raw(@active_admin_import_model.hint) %> 
 </p>
 <%= semantic_form_for @active_admin_import_model, url: {action: :do_import}, html: {multipart: true} do |f| %>
-    <%= f.inputs do %>
+    <%= f.inputs name: t("active_admin_import.details") do %>
         <%= f.input :file, as: :file %>
     <% end %>
     <%= f.actions do %>

data/config/locales/en.yml

@@ -2,15 +2,22 @@ en:
   active_admin:
     import: "Import"
   active_admin_import:
+    file: 'File'
+    file_error: "Error: %{message}"
     file_format_error: "You can import only valid csv file"
     file_empty_error: "You can't import empty file"
     no_file_error: "Please, select file to import"
+    details: "Please, select file to import"
     imported:
       one: "Successfully imported 1 %{model}"
       other: "Successfully imported %{count} %{plural_model}"
     failed:
-      one:   "Failed to import 1 %{model}"
-      other: "Failed to import %{count} %{plural_model}"
-    import_model: "Import %{model}"
+      one:   "Failed to import 1 %{model}: %{message}"
+      other: "Failed to import %{count} %{plural_model}: %{message}"
+    import_model: "Import %{plural_model}"
     import_btn: "Import"
-    import_btn_disabled: "Wait..."
+    import_btn_disabled: "Wait..."
+    csv_options: "CSV options"
+    col_sep: 'Col sep'
+    row_sep: 'Row sep'
+    quote_char: 'Quote char'

data/config/locales/es.yml

@@ -0,0 +1,18 @@
+es:
+  active_admin:
+    import: "Importar"
+  active_admin_import:
+    file_error: "Error: %{message}"
+    file_format_error: "Sólo se puede importar un archivo CSV válido"
+    file_empty_error: "No puedes importar un archivo vacío"
+    no_file_error: "Seleccione un archivo para importar"
+    details: "Seleccione un archivo"
+    imported:
+      one: "Se ha importado exitosamente 1 %{model}"
+      other: "Se ha importado exitosamente %{count} %{plural_model}"
+    failed:
+      one:   "No se pudo importar 1 %{model}"
+      other: "No se pudo importar %{count} %{plural_model}"
+    import_model: "Importar %{plural_model}"
+    import_btn: "Importar"
+    import_btn_disabled: "Espere..."

data/config/locales/it.yml

@@ -2,15 +2,17 @@ it:
   active_admin:
     import: "Importa"
   active_admin_import:
+    file_error: "Errore: %{message}"
     file_format_error: "Puoi importare solo CSV conformi"
     no_file_error: "Per favore, seleziona un file da importare"
     file_empty_error: "Non è possibile importare file vuoto"
+    details: "Per favore, seleziona un file da importare"
     imported:
       one: "Importato 1 %{model} con successo"
       other: "Importati %{count} %{plural_model} con successo"
     failed:
       one:   "Importazione di 1 %{model} fallita"
       other: "Importazione di %{count} %{plural_model} fallita"
-    import_model: "Importa %{model}"
+    import_model: "Importa %{plural_model}"
     import_btn: "Importa"
     import_btn_disabled: "In attesa..."

data/config/locales/zh-CN.yml

@@ -0,0 +1,24 @@
+zh-CN:
+  active_admin:
+    import: "导入"
+    do_import: "导入"
+  active_admin_import:
+    file: '文件'
+    file_error: "导入错误: %{message}"
+    file_format_error: "请您上传 CSV 格式的文件"
+    file_empty_error: "您上传的文件为空"
+    no_file_error: "请您选择要上传的文件"
+    details: "请您选择要上传的文件"
+    imported:
+      one: "成功导入 1 份%{model}"
+      other: "成功导入 %{count} 份%{plural_model}"
+    failed:
+      one:   "未能导入 1 份%{model}: %{message}"
+      other: "未能导入 %{count} 份%{plural_model}: %{message}"
+    import_model: "导入%{plural_model}"
+    import_btn: "导入"
+    import_btn_disabled: "请稍等..."
+    csv_options: "选项"
+    col_sep: '列分隔符'
+    row_sep: '行分隔符'
+    quote_char: '引用字符'

data/lib/active_admin_import/dsl.rb

@@ -1,18 +1,19 @@
 module ActiveAdminImport
   module DSL
+
+
     # Declares import functionality
     #
     # Options
     # +back+:: resource action to redirect after processing
-    # +col_sep+:: column separator used for CSV parsing (deprecated)
-    # +row_sep+:: column separator used for CSV parsing  (deprecated)
     # +csv_options+:: hash to override default CSV options
-    # +validate+:: true|false, means perfoem validations or not
     # +batch_size+:: integer value of max  record count inserted by 1 query/transaction
+    # +batch_transaction+:: bool (false by default), if transaction is used when batch importing and works when :validate is set to true
     # +before_import+:: proc for before import action, hook called with  importer object
     # +after_import+:: proc for after import action, hook called with  importer object
     # +before_batch_import+:: proc for before each batch action, called with  importer object
     # +after_batch_import+:: proc for after each batch action, called with  importer object
+    # +validate+:: true|false, means perform validations or not
     # +on_duplicate_key_update+:: an Array or Hash, tells activerecord-import to use MySQL's ON DUPLICATE KEY UPDATE ability.
     # +timestamps+::  true|false, tells activerecord-import to not add timestamps (if false) even if record timestamps is disabled in ActiveRecord::Base
     # +ignore+::  true|false, tells activerecord-import to use MySQL's INSERT IGNORE ability
@@ -20,58 +21,65 @@ module ActiveAdminImport
     # +template_object+:: object passing to view
     # +resource_class+:: resource class name, override to import to another table (default config.resource_class)
     # +resource_label+:: resource label value (default config.resource_label)
-    # +plural_resource_label+:: plaralized resource label value (default config.plural_resource_label)
+    # +plural_resource_label+:: pluralized resource label value (default config.plural_resource_label)
     #
-    def active_admin_import options = {}, &block
-      default_options = {
-          back: {action: :import},
-          csv_options: {},
-          template: "admin/import",
-          resource_class: config.resource_class,
-          resource_label: config.resource_label,
-          plural_resource_label: config.plural_resource_label,
-          headers_rewrites: {}
-      }
-      options = default_options.deep_merge(options)
-      params_key = ActiveModel::Naming.param_key(options[:template_object] || ActiveAdminImport::Model.new)
+
+
+    DEFAULT_RESULT_PROC =  ->(result, options) do
+
+      model_name = options[:resource_label].downcase
+      plural_model_name = options[:plural_resource_label].downcase
+      if result.empty?
+        flash[:warning] = I18n.t('active_admin_import.file_empty_error')
+      else
+        if result.has_failed?
+          flash[:error] = I18n.t('active_admin_import.failed', count: result.failed.count, model: model_name, plural_model: plural_model_name, message: result.failed_message(limit: 5))
+          return if options[:batch_transaction]
+        end
+        if result.has_imported?
+          flash[:notice] = I18n.t('active_admin_import.imported', count: result.imported_qty, model: model_name, plural_model: plural_model_name)
+        end
+      end
+    end
+
+
+    def active_admin_import(options = {}, &block)
+      options.assert_valid_keys(*Options::VALID_OPTIONS)
+
+      options = Options.options_for(config, options)
+      params_key = ActiveModel::Naming.param_key(options[:template_object])
 
       collection_action :import, method: :get do
         authorize!(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class)
-
-        @active_admin_import_model = options[:template_object] || ActiveAdminImport::Model.new
+        @active_admin_import_model = options[:template_object]
         render template: options[:template]
       end
 
       action_item :import, only: :index do
         if authorized?(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class)
-          link_to(I18n.t('active_admin_import.import_model', model: options[:resource_label]), action: 'import')
+          link_to(I18n.t('active_admin_import.import_model', plural_model: options[:plural_resource_label]), action: :import)
         end
       end
 
       collection_action :do_import, method: :post do
         authorize!(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class)
 
-        @active_admin_import_model = options[:template_object] || ActiveAdminImport::Model.new
+        @active_admin_import_model = options[:template_object]
         @active_admin_import_model.assign_attributes(params[params_key].try(:deep_symbolize_keys) || {})
         #go back to form
         return render template: options[:template] unless @active_admin_import_model.valid?
         @importer = Importer.new(options[:resource_class], @active_admin_import_model, options)
         begin
-          @importer.import
+          result = @importer.import
+
           if block_given?
             instance_eval &block
           else
-            model_name = options[:resource_label].downcase
-            plural_model_name = options[:plural_resource_label].downcase
-            if @importer.result[:imported].to_i > 0
-              flash[:notice] = I18n.t('active_admin_import.imported', count: @importer.result[:imported].to_i, model: model_name, plural_model: plural_model_name)
-            end
-            if  @importer.result[:failed].count > 0
-              flash[:error] = I18n.t('active_admin_import.failed', count: @importer.result[:failed].count, model: model_name, plural_model: plural_model_name)
-            end
+            instance_exec result, options, &DEFAULT_RESULT_PROC
           end
-        rescue ActiveRecord::Import::MissingColumnError, NoMethodError => e
-          flash[:error] = e.message
+        rescue ActiveRecord::Import::MissingColumnError, NoMethodError, ActiveRecord::StatementInvalid => e
+          Rails.logger.error(I18n.t('active_admin_import.file_error', message: e.message))
+          flash[:error] = I18n.t('active_admin_import.file_error', message: e.message[0..200])
         end
         redirect_to options[:back]
       end

data/lib/active_admin_import/import_result.rb

@@ -0,0 +1,39 @@
+module ActiveAdminImport
+  class ImportResult
+    attr_reader :failed, :total
+
+    def initialize
+      @failed = []
+      @total = 0
+    end
+
+    def add(result, qty)
+      @failed += result.failed_instances
+      @total  += qty
+    end
+
+    def imported_qty
+      total - failed.count
+    end
+
+    def has_imported?
+      imported_qty > 0
+    end
+
+    def has_failed?
+      @failed.any?
+    end
+
+    def empty?
+      total == 0
+    end
+
+    def failed_message(options = {})
+      limit = options.fetch(:limit, failed.count)
+      failed.first(limit).map{|record|
+        errors = record.errors
+        (errors.full_messages.zip errors.keys.map{|k| record.send k}).map{|ms| ms.join(' - ')}.join(', ')
+      }.join(" ; ")
+    end
+  end
+end