rails_admin_import 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 741d56674d36948aadddfa2951204d3fd3ece179
-  data.tar.gz: 4535fb361151b7f3dbb615c380a0d9eeb93a6478
+  metadata.gz: 6ad5c6af8848efccee2f00133a0537a41f6ceca8
+  data.tar.gz: 6928c129f69d8653a0bcdd9d18f905c8191e99fe
 SHA512:
-  metadata.gz: f364e695d7e1eac68a4bc821cc08f34c1dc1ea340a8d8c26a317aba5fcbcff7aded642fc7c9657e6c4671e3306b4dc1522ab39ef4c9ae449850e5a53806b5ec8
-  data.tar.gz: dd57d2e866208dfeb6da5acf65d118902c87d650d0cf42828266a901e726734b8d1a159b4cde9569f36fd7260a952792b63d58aeec11bfa9ebff7960418b11c4
+  metadata.gz: 51216c41011b73a8b62c612c4690b740c9d126f0c24f9404086a7354389b05258e79384161d082a62709237936b8e0f0e081a7d40fd376df796391e07ee5e678
+  data.tar.gz: f6574cd2fac75ccb479d80f0749bc136675024748d8c01baefca6023e871c1d09b970274606b6054bd21d1b24c39517330cba3413d1260866f4a1038697322af

data/README.md

@@ -4,14 +4,12 @@
 
 Plugin functionality to add generic import to Rails Admin from CSV, JSON and XLSX files
 
-*This Readme is for version 1.x. If you are still using version 0.1.x, see [this branch](https://github.com/stephskardal/rails_admin_import/tree/legacy)*
-
 ## Installation
 
 * First, add to Gemfile:
 
 ```
-gem "rails_admin_import", "~> 1.2"
+gem "rails_admin_import", "~> 2.1"
 ```
 
 * Define configuration in `config/initializers/rails_admin_import.rb`:
@@ -62,9 +60,10 @@ Both updating existing records and associating records requires the use of
 ### Mapping Keys
 
 Every importable class has a mapping key that uniquely identifies its
-instances. The value for this field can then be provided in import data, either
-to update the existing record or to attach it through an association to another
-model. This concept exists because `id`s are often not constant when moving
+instances. The mapping key can be one or more fields. The value for
+these fields can then be provided in import data, either to update the
+existing record or to attach it through an association to another model.
+This concept exists because `id`s are often not constant when moving
 records between data stores.
 
 For example, a `User` model may have an `email` field. When uploading a set
@@ -81,6 +80,9 @@ michael.bolton@initech.com,Michael,Bolton
 would look for existing users with those emails. If one was found, its name
 fields would be updated. Otherwise, a new one would be created.
 
+For updating building owners, the mapping key could be `street_address` and
+`zip_code`.
+
 Similarly, if each user has favorite books, we could set the mapping key
 for `Book` to be `isbn` and then include the isbn for their books within each
 user record. The syntax for this is to use the name of the associated model as
@@ -104,13 +106,19 @@ RailsAdmin.config do |config|
   config.model 'User' do
     import do
       mapping_key :email
+      # for multiple values, use mapping_key [:first_name, :last_name]
+      mapping_key_list [:email, :some_other_id]
     end
   end
 end
 ```
 
+Since in models with large number of fields it doesn't make sense to use
+most of them as mapping values, you can add `mapping_key_list` to
+restrict which fields can be selected as mapping key in the UI during import.
+
 Note that a matched record must exist when attaching associated models, or the
- imported record will fail and be skipped.
+imported record will fail and be skipped.
 
 Complex associations (`has_one ..., :through` or polymorphic associations)
 need to be dealt with via custom logic called by one of the import hooks
@@ -133,6 +141,8 @@ class Service < ActiveRecord::Base
 end
 ```
 
+Importing new records by id is not recommended since it ignores the sequences of ids in database. That will lead to `ERROR: duplicate key value violates unique constraint` in future. You can work around this issue by adding an `import_id` column to your model, renaming the `id` column in your CSV to `import_id` and using `import_id` as the update lookup field.
+
 ### File format
 
 The format is inferred by the extension (.csv, .json or .xlsx).
@@ -153,6 +163,8 @@ Peter,Gibbons,IT,Management
 Michael,Bolton,IT,
 ```
 
+Blank lines will be skipped.
+
 #### JSON
 
 The file must be an array or an object with a root key the same name as the plural model name, i.e. the default Rails JSON output format with include_root_in_json on or off.
@@ -163,14 +175,41 @@ The Microsoft Excel XLM format (XLSX) is supported, but not the old binary Micro
 
 The expected rows and columns are the same as for the CSV format (first line contains headers, multiple columns for "many" associations).
 
+Blank lines will be skipped.
+
 ## Configuration
 
 ### Global configuration options
 
+```ruby
+RailsAdmin.config do |config|
+  config.actions do
+    all
+    import
+  end
+
+  # Default global RailsAdminImport options
+  config.configure_with(:import) do |config|
+    config.logging = false
+    config.line_item_limit = 1000
+    config.update_if_exists = false
+    config.rollback_on_error = false
+    config.header_converter = lambda do |header|
+      # check for nil/blank headers
+      next if header.blank?
+      header.parameterize.underscore
+    end
+    config.csv_options = {}
+  end
+end
+```
+
 * __logging__ (default `false`): Save a copy of each imported file to log/import and a detailed import log to log/rails_admin_import.log
 
 * __line_item_limit__ (default `1000`): max number of items that can be imported at one time.
 
+* __update_if_exists__ (default `false`): default value for the "Update if exists" checkbox on the import page.
+
 * __rollback_on_error__ (default `false`): import records in a transaction and rollback if there is one error. Only for ActiveRecord, not Mongoid.
 
 * __header_converter__ (default `lambda { ... }`): a lambda to convert each CSV header text string to a model attribute name. The default header converter converts to lowercase and replaces spaces with underscores.
@@ -239,7 +278,7 @@ end
 RailsAdmin.config do |config|
   config.model 'User' do
     import do
-      default_excluded_fields [:created_at, :updated_at]
+      default_excluded_fields [:created_at, :updated_at, :deleted_at, :c_at, :u_at]
     end
   end
 end
@@ -247,18 +286,56 @@ end
 
 ## Import hooks
 
-
-Define instance methods on your models to be hooked into the import process, if special/additional processing is required on the data:
+Define methods on your models to be hooked into the import process, if special/additional processing is required on the data:
 
 ```ruby
 # some model
 class User < ActiveRecord::Base
+  def self.before_import
+    # called on the model class once before importing any individual records
+  end
+
+  def self.before_import_find(record)
+    # called on the model class before finding or creating the new record
+    # maybe modify the import record that will be used to find the model
+    # throw :skip to skip importing this record
+    throw :skip unless record[:email].ends_with? "@mycompany.com"
+  end
+
+  def before_import_attributes(record)
+    # called on the blank new model or the found model before fields are imported
+    # maybe delete fields from the import record that you don't need
+    # throw :skip to skip importing this record
+  end
+
+  def before_import_associations(record)
+    # called on the model with attributes but before associations are imported
+    # do custom import of associations
+    # make sure to delete association fields from the import record to avoid double import
+    record.delete(:my_association)
+    # throw :skip to skip importing this record
+  end
+
   def before_import_save(record)
-    # Your custom special sauce
+    # called on the model before it is saved but after all fields and associations have been imported
+    # make final modifications to the record
+    # throw :skip to skip importing this record
   end
 
   def after_import_save(record)
-    # Your custom special sauce
+    # called on the model after it is saved
+  end
+
+  def after_import_association_error(record)
+    # called on the model when an association cannot be found
+  end
+
+  def after_import_error(record)
+    # called on the model when save fails
+  end
+
+  def self.after_import
+    # called once on the model class after importing all individual records
   end
 end
 ```
@@ -275,6 +352,29 @@ def before_import_save(record)
 end
 ```
 
+* Skip some validations when importing.
+
+```
+class User < ActiveRecord::Base
+  # Non-persistent attribute to allow creating a new user without a password
+  # Password will be set by the user by following a link in the invitation email
+  attr_accessor :allow_blank_password
+
+  devise :validatable
+
+  # Called by Devise to enable/disable password presence validation
+  def password_required?
+    allow_blank_password ? false : super
+  end
+
+  # Don't require a password when importing users
+  def before_import_save(record)
+    self.allow_blank_password = true
+  end
+end
+```
+
+
 ## ORM: ActiveRecord and Mongoid
 
 The gem is tested to work with ActiveRecord and Mongoid.
@@ -292,6 +392,23 @@ If you prefer to eager load all dependecies at boot, use this line in your `Gemf
 gem "rails_admin_import", "~> 1.2.0", require: "rails_admin_import/eager_load"
 ```
 
+## Import error due to Rails class reloading
+
+If you get an error like `Error during import: MyModel(#70286054976500) expected, got MyModel(#70286114743280)`, you need restart the rails server and redo the import. This is due to the fact that Rails reloads the ActiveRecord model classes in development when you make changes to them and Rails Admin is still using the old class.
+
+## Customize the UI
+
+If you want to hide all the advanced fields from the import UI, you can copy [`app/views/rails_admin/main/import.html.haml`](app/views/rails_admin/main/import.html.haml) to your project at the same path. Add `.hidden` at the end of lines you want to hide.
+
+For example:
+```
+    .form-group.control-group.hidden
+      %label.col-sm-2.control-label= t("admin.import.update_if_exists")
+      .col-sm-10.controls
+        = check_box_tag :update_if_exists, '1', true, :class => "form-control"
+        %p.help-block= t('admin.import.help.update_if_exists')
+```
+
 ## Upgrading
 
 * Move global config to `config.configure_with(:import)` in `config/initializers/rails_admin_import.rb`.
@@ -303,11 +420,17 @@ gem "rails_admin_import", "~> 1.2.0", require: "rails_admin_import/eager_load"
 * Update model import hooks to take 1 hash argument instead of 2 arrays with values and headers.
 
 * Support for importing file attributes was removed since I couldn't understand how it works. It should be possible to reimplement it yourself using post import hooks. Open an issue to discuss how to put back support for importing files into the gem.
- 
+
 ## Community-contributed translations
 
 * [Spanish translation](https://gist.github.com/yovasx2/dc0e9512e6c6243f840c) by Giovanni Alberto
 
+* [French translation](https://github.com/rodinux/rails_admin_import.fr-MX.yml) by Rodolphe Robles. (I suggest to translate also rails admin.fr and your locales.fr to resolve an issue with DatePicker)
+
+* [Italian translation](https://gist.github.com/aprofiti/ec3dc452898c8c48534b59eeb2701765) by Alessandro Profiti
+
+* [Japanese translation](https://gist.github.com/higumachan/c4bf669d6446ec509386229f916ba5fc) by Yuta Hinokuma
+
 ## Run tests
 
 1. Clone the repository to your machine
@@ -326,6 +449,13 @@ Original author: [Steph Skardal](https://github.com/stephskardal)
 Maintainer (since May 2015): [Julien Vanier](https://github.com/monkbroc)
 
 
+## Release
+
+- Update `lib/rails_admin_import/version.rb`
+- Update the install instructions at [the top of the readme](#installation)
+- Commit to git
+- `rake release`
+
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:

data/app/views/rails_admin/main/import.html.haml

@@ -30,14 +30,16 @@
     .form-group.control-group
       %label.col-sm-2.control-label= t("admin.import.update_if_exists")
       .col-sm-10.controls
-        = check_box_tag :update_if_exists, '1', false, :class => "form-control"
+        = check_box_tag :update_if_exists, '1', RailsAdminImport.config.update_if_exists, :class => "form-control"
         %p.help-block= t('admin.import.help.update_if_exists')
     .form-group.control-group
       %label.col-sm-2.control-label{for: "update_lookup"}= t("admin.import.update_lookup")
       .col-sm-10.controls
         = select_tag 'update_lookup',
           options_for_select(@import_model.update_lookup_field_names,
-          @import_model.config.mapping_key.to_s), data: { enumeration: true }
+            Array.wrap(@import_model.config.mapping_key).map(&:to_s)),
+          multiple: true,
+          data: { enumeration: true }
 
   - unless @import_model.association_fields.empty?
     %fieldset
@@ -53,7 +55,8 @@
           .col-sm-10.controls
             = select_tag "associations[#{field.name}]",
               options_for_select(@import_model.associated_model_fields(field),
-              @import_model.associated_config(field).mapping_key.to_s), data: { enumeration: true }
+                Array.wrap(@import_model.associated_config(field).mapping_key).first.to_s),
+              data: { enumeration: true }
 
   %br
   .form-actions

data/config/locales/README.md

@@ -0,0 +1,3 @@
+See the [community contributed translation section of the README](https://github.com/stephskardal/rails_admin_import#community-contributed-translations) for more languages.
+
+To contribute a new translation, please put it in a gist and submit a pull request to link your translation in the README.

data/config/locales/import.en.yml

@@ -20,7 +20,7 @@ en:
       missing_update_lookup: "Your file must contain a column for the 'Update lookup field' you selected."
       invalid_json: "The JSON data should be an array of records or an object with a key '%{root_key}' set to an array of records"
       update_if_exists: "Update if exists"
-      update_lookup: "Update lookup field"
+      update_lookup: "Update lookup field(s)"
       mapping: "mapping"
       encoding: "Encoding"
       legend:
@@ -47,5 +47,4 @@ en:
           For "many" associations, you may include multiple columns with the same header in the CSV file.
         update_if_exists: "Update records found with the lookup field below instead of creating new records"
         file_limit: "Please limit upload file to %{limit} line items."
-        encoding: "Choose file encoding. Leave empty to auto-detect. Ignored for JSON."
-
+        encoding: "Choose file encoding. Leave empty to auto-detect. Ignored for JSON."

data/lib/rails_admin_import/config.rb

@@ -6,11 +6,14 @@ module RailsAdminImport
       attr_accessor :logging
       attr_accessor :line_item_limit
       attr_accessor :rollback_on_error
+      attr_accessor :update_if_exists
       attr_accessor :header_converter
       attr_accessor :csv_options
 
       # Default is to downcase headers and add underscores to convert into attribute names
       HEADER_CONVERTER = lambda do |header|
+        # check for nil/blank headers
+        next if header.blank?
         header.parameterize.underscore
       end
 
@@ -31,6 +34,7 @@ module RailsAdminImport
         @logging = false
         @line_item_limit = 1000
         @rollback_on_error = false
+        @update_if_exists = false
         @header_converter = HEADER_CONVERTER
         @csv_options = {}
       end

data/lib/rails_admin_import/config/sections/import.rb

@@ -14,7 +14,7 @@ module RailsAdmin
         end
 
         register_instance_option(:default_excluded_fields) do
-          [:id, :_id, :created_at, :updated_at, :c_at, :u_at]
+          [:id, :_id, :created_at, :updated_at, :c_at, :u_at, :deleted_at]
         end
       end
     end

data/lib/rails_admin_import/formats/csv_importer.rb

@@ -4,6 +4,7 @@ module RailsAdminImport
   module Formats
     class CSVImporter < FileImporter
       Formats.register(:csv, self)
+      Formats.register(:CSV, self)
 
       autoload :CharDet, "rchardet"
 
@@ -16,7 +17,8 @@ module RailsAdminImport
       # A method that yields a hash of attributes for each record to import
       def each_record
         CSV.foreach(filename, csv_options) do |row|
-          yield convert_to_attributes(row)
+          attr = convert_to_attributes(row)
+          yield attr unless attr.all? { |field, value| value.blank? }
         end
       end
 

data/lib/rails_admin_import/formats/json_importer.rb

@@ -2,6 +2,7 @@ module RailsAdminImport
   module Formats
     class JSONImporter < FileImporter
       Formats.register(:json, self)
+      Formats.register(:JSON, self)
 
       # A method that yields a hash of attributes for each record to import
       def each_record

data/lib/rails_admin_import/formats/xlsx_importer.rb

@@ -4,6 +4,7 @@ module RailsAdminImport
   module Formats
     class XLSXImporter < FileImporter
       Formats.register(:xlsx, self)
+      Formats.register(:XLSX, self)
 
       autoload :SimpleXlsxReader, "simple_xlsx_reader"
 
@@ -18,7 +19,8 @@ module RailsAdminImport
         sheet = doc.sheets.first
         @headers = convert_headers(sheet.headers)
         sheet.data.each do |row|
-          yield convert_to_attributes(row)
+          attr = convert_to_attributes(row)
+          yield attr unless attr.all? { |field, value| value.blank? }
         end
       end
 
@@ -33,7 +35,7 @@ module RailsAdminImport
       def convert_to_attributes(row)
         row_with_headers = @headers.zip(row)
         row_with_headers.each_with_object({}) do |(field, value), record|
-          break if field.nil?
+          next if field.nil?
           field = field.to_sym
           if import_model.has_multiple_values?(field)
             field = import_model.pluralize_field(field)

data/lib/rails_admin_import/importer.rb

@@ -16,7 +16,6 @@ module RailsAdminImport
       begin
         init_results
 
-
         if records.count > RailsAdminImport.config.line_item_limit
           return results = {
             success: [],
@@ -24,13 +23,19 @@ module RailsAdminImport
           }
         end
 
+        perform_global_callback(:before_import)
+
         with_transaction do
           records.each do |record|
-            import_record(record)
+            catch :skip do
+              import_record(record)
+            end
           end
 
           rollback_if_error
         end
+
+        perform_global_callback(:after_import)
       rescue Exception => e
         report_general_error("#{e} (#{e.backtrace.first})")
       end
@@ -65,19 +70,24 @@ module RailsAdminImport
     end
 
     def import_record(record)
-      if update_lookup && !record.has_key?(update_lookup)
+      perform_model_callback(import_model.model, :before_import_find, record)
+
+      if update_lookup && !(update_lookup - record.keys).empty?
         raise UpdateLookupError, I18n.t("admin.import.missing_update_lookup")
       end
 
       object = find_or_create_object(record, update_lookup)
+      return if object.nil?
       action = object.new_record? ? :create : :update
 
       begin
+        perform_model_callback(object, :before_import_associations, record)
         import_single_association_data(object, record)
         import_many_association_data(object, record)
       rescue AssociationNotFound => e
         error = I18n.t("admin.import.association_not_found", :error => e.to_s)
         report_error(object, action, error)
+        perform_model_callback(object, :after_import_association_error, record)
         return
       end
 
@@ -88,12 +98,13 @@ module RailsAdminImport
         perform_model_callback(object, :after_import_save, record)
       else
         report_error(object, action, object.errors.full_messages.join(", "))
+        perform_model_callback(object, :after_import_error, record)
       end
     end
 
     def update_lookup
       @update_lookup ||= if params[:update_if_exists] == "1"
-                           params[:update_lookup].to_sym
+                           params[:update_lookup].map(&:to_sym)
                          end
     end
 
@@ -169,6 +180,11 @@ module RailsAdminImport
       end
     end
 
+    def perform_global_callback(method_name)
+      object = import_model.model
+      object.send(method_name) if object.respond_to?(method_name)
+    end
+
     def find_or_create_object(record, update)
       field_names = import_model.model_fields.map(&:name)
       new_attrs = record.select do |field_name, value|
@@ -177,13 +193,19 @@ module RailsAdminImport
 
       model = import_model.model
       object = if update.present?
-                 model.where(update => record[update]).first
+                 query = update.each_with_object({}) do
+                   |field, query| query[field] = record[field]
+                 end
+                 model.where(query).first
                end
 
       if object.nil?
-        object = model.new(new_attrs)
+        object = model.new
+        perform_model_callback(object, :before_import_attributes, record)
+        object.attributes = new_attrs
       else
-        object.attributes = new_attrs.except(update.to_sym)
+        perform_model_callback(object, :before_import_attributes, record)
+        object.attributes = new_attrs.except(update.map(&:to_sym))
       end
       object
     end

data/lib/rails_admin_import/version.rb

@@ -1,3 +1,3 @@
 module RailsAdminImport
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_admin_import
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Steph Skardal
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-05 00:00:00.000000000 Z
+date: 2017-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -39,20 +39,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.6
-- !ruby/object:Gem::Dependency
-  name: haml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: rchardet
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +82,7 @@ files:
 - app/views/rails_admin/main/_results.html.haml
 - app/views/rails_admin/main/_section.html.haml
 - app/views/rails_admin/main/import.html.haml
+- config/locales/README.md
 - config/locales/import.en.yml
 - lib/rails_admin_import.rb
 - lib/rails_admin_import/action.rb
@@ -135,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Import functionality for Rails Admin