rails_admin_import 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a37478fcdb96c3cddb93ac6f85dc1fa43329f32
-  data.tar.gz: 8b36ca27a1b441bebd19a6cb3a80865063975160
+  metadata.gz: d22596961dbc1568f035e4f5c6e5896fcfb01474
+  data.tar.gz: 407a068d2bf2d62e096ec46c4c061b559db98bec
 SHA512:
-  metadata.gz: 037159e831272b29da9d07093f43fdb7e212ce92f4a28c02c7bc836b634239d4285e04f6fe102a6f402203644d4d81bc67388e7b75c1518674d63fb26eda6b49
-  data.tar.gz: c323b72bfd8d3980e11e06eadf83b34c184fcf3fd04d8ed73880dc45741d518b0d37cd0dcc37f698df1d64055e634e94c67db4797e81a652af84f428f7bc754a
+  metadata.gz: d1d2338cbf97015e11c114b1ee6d68994c8517942497ce839f029ac02e36d133f8bd9866634e23e5e2825d9a88033c25e5508016a209f39d48d593edcf2c376f
+  data.tar.gz: 82fa136b73f126c8dcabaa7e83e472b0815b0231286d505be16718c15a7b8f9aa425fee723e67424419dd5e94a51182adbd6133238f4382b2d4f36df5ce96f80

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## [1.2.0] - 2015-08-27
+### Changed
+- Existing records can now be updated based on a belongs_to foreign key. Thanks Diego Carrion!
+- Add Excel file format support
+- Autoload the Excel import and encoding detection gems
+
 ## [1.1.0] - 2015-08-04
 ### Changed
 - `csv_options` config added. Thanks Maksim Burnin!
@@ -23,4 +29,4 @@ Major rework of the gem by new maintainer.
 - Updated/corrected README
 - Merged ImportLogger work
 - Merged modifications to import view
-- Merged post save hook on models
+- Merged post save hook on models

data/README.md

@@ -2,16 +2,16 @@
 
 [![Build Status](https://travis-ci.org/monkbroc/rails_admin_import.svg?branch=master)](https://travis-ci.org/monkbroc/rails_admin_import)
 
-Plugin functionality to add generic import to Rails Admin from CSV and JSON files
+Plugin functionality to add generic import to Rails Admin from CSV, JSON and XLSX files
 
-*This Readme is for version 1.0. If you are still using version 0.1.x, see [this branch](https://github.com/stephskardal/rails_admin_import/tree/legacy)*
+*This Readme is for version 1.x. If you are still using version 0.1.x, see [this branch](https://github.com/stephskardal/rails_admin_import/tree/legacy)*
 
 ## Installation
 
 * First, add to Gemfile:
 
 ```
-gem "rails_admin_import", "~> 1.0.0"
+gem "rails_admin_import", "~> 1.2"
 ```
 
 * Define configuration in `config/initializers/rails_admin_import.rb`:
@@ -52,6 +52,8 @@ can :import, [User, Model1, Model2]
 
 ## File format
 
+The format is inferred by the extension (.csv, .json or .xlsx).
+
 ### CSV
 
 The first line must contain attribute names. They will be converted to lowercase and underscored (First Name ==> first_name).
@@ -72,6 +74,11 @@ Michael,Bolton,IT,
 
 The file must be an array or an object with a root key the same name as the plural model name, i.e. the default Rails JSON output format with include_root_in_json on or off.
 
+### XLSX
+
+The Microsoft Excel XLM format (XLSX) is supported, but not the old binary Microsoft Excel format (XLS).
+
+The expected rows and columns are the same as for the CSV format (first line contains headers, multiple columns for "many" associations).
 
 ## Configuration
 
@@ -83,7 +90,7 @@ The file must be an array or an object with a root key the same name as the plur
 
 * __rollback_on_error__ (default `false`): import records in a transaction and rollback if there is one error. Only for ActiveRecord, not Mongoid.
 
-* __header_converter__ (default `nil`): a lambda to convert each CSV header text string to a model attribute name. The default header converter converts to lowercase and replaces spaces with underscores.
+* __header_converter__ (default `lambda { ... }`): a lambda to convert each CSV header text string to a model attribute name. The default header converter converts to lowercase and replaces spaces with underscores.
 
 * __csv_options__ (default `{}`): a hash of options that will be passed to a new [CSV](http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/CSV.html) instance
 
@@ -175,6 +182,16 @@ The gem is tested to work with ActiveRecord and Mongoid.
 Support for Mongoid is early, so if you can suggest improvements (especially around importing embedded models), open an issue.
 
 
+## Eager loading
+
+Since the import functionality is rarely used in many applications, some gems are autoloaded when first used during an import in order to save memory at boot.
+
+If you prefer to eager load all dependecies at boot, use this line in your `Gemfile`.
+
+```
+gem "rails_admin_import", "~> 1.2.0", require: "rails_admin_import/eager_load"
+```
+
 ## Upgrading
 
 * Move global config to `config.configure_with(:import)` in `config/initializers/rails_admin_import.rb`.
@@ -194,7 +211,7 @@ Support for Mongoid is early, so if you can suggest improvements (especially aro
     git clone https://github.com/stephskardal/rails_admin_import
 
 2. Run `bundle install`
-3. Run `rspec`
+3. Run `bundle exec rspec`
 
 The structure of the tests is taken from the Rails Admin gem.
 
@@ -209,8 +226,8 @@ Maintainer (since May 2015): [Julien Vanier](https://github.com/monkbroc)
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-- [Report bugs](https://github.com/ankane/blazer/issues)
-- Fix bugs and [submit pull requests](https://github.com/ankane/blazer/pulls)
+- [Report bugs](https://github.com/stephskardal/rails_admin_import/issues)
+- Fix bugs and [submit pull requests](https://github.com/stephskardal/rails_admin_import/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
 

data/Rakefile

@@ -1,7 +1,7 @@
 #!/usr/bin/env rake
 
-require 'bundler'
-Bundler::GemHelper.install_tasks
+require 'rubygems/tasks'
+Gem::Tasks.new
 
 # Piggyback off the Rails Admin rake tasks to set up the CI environment
 spec = Gem::Specification.find_by_name 'rails_admin'

data/app/views/rails_admin/main/import.html.haml

@@ -20,12 +20,6 @@
       .col-sm-10.controls
         = file_field_tag :file, :class => "form-control"
         %p.help-block= t('admin.import.help.file_limit', limit: RailsAdminImport.config.line_item_limit)
-    .form-group.control-group
-      %label.col-sm-2.control-label{for: "import_format"}= t("admin.import.format")
-      .col-sm-10.controls
-        = select_tag 'import_format',
-          options_for_select(RailsAdminImport::Formats.all),
-          data: { enumeration: true }
     .form-group.control-group
       %label.col-sm-2.control-label{for: "encoding"}= t("admin.import.encoding")
       .col-sm-10.controls
@@ -42,7 +36,7 @@
       %label.col-sm-2.control-label{for: "update_lookup"}= t("admin.import.update_lookup")
       .col-sm-10.controls
         = select_tag 'update_lookup',
-          options_for_select(@import_model.model_fields.map(&:name),
+          options_for_select(@import_model.update_lookup_field_names,
           @import_model.config.mapping_key.to_s), data: { enumeration: true }
 
   - unless @import_model.association_fields.empty?

data/lib/rails_admin_import/action.rb

@@ -19,8 +19,9 @@ module RailsAdmin
             @import_model = RailsAdminImport::ImportModel.new(@abstract_model)
 
             if request.post?
+              format = RailsAdminImport::Formats.from_file(params[:file])
               record_importer = RailsAdminImport::Formats.for(
-                params[:import_format] || "csv", @import_model, params)
+                format, @import_model, params)
 
               if record_importer.valid?
                 importer = RailsAdminImport::Importer.new(

data/lib/rails_admin_import/config.rb

@@ -9,6 +9,11 @@ module RailsAdminImport
       attr_accessor :header_converter
       attr_accessor :csv_options
 
+      # Default is to downcase headers and add underscores to convert into attribute names
+      HEADER_CONVERTER = lambda do |header|
+        header.parameterize.underscore
+      end
+
       def model(model_name, &block)
         unless @deprecation_shown
           warn "RailsAdminImport::Config#model is deprecated. " \
@@ -26,7 +31,7 @@ module RailsAdminImport
         @logging = false
         @line_item_limit = 1000
         @rollback_on_error = false
-        @header_converter = nil
+        @header_converter = HEADER_CONVERTER
         @csv_options = {}
       end
     end

data/lib/rails_admin_import/eager_load.rb

@@ -0,0 +1,3 @@
+require "rails_admin_import"
+require "rchardet"
+require "simple_xlsx_reader"

data/lib/rails_admin_import/formats.rb

@@ -5,6 +5,11 @@ module RailsAdminImport
         @registry[format.to_s] = klass
       end
 
+      def from_file(file)
+        return unless file
+        File.extname(file.original_filename).sub(/^\./, '')
+      end
+
       def for(format, *args)
         @registry.fetch(format.to_s, DummyImporter).new(*args)
       end
@@ -26,3 +31,4 @@ require "rails_admin_import/formats/dummy_importer"
 require "rails_admin_import/formats/file_importer"
 require "rails_admin_import/formats/csv_importer"
 require "rails_admin_import/formats/json_importer"
+require "rails_admin_import/formats/xlsx_importer"

data/lib/rails_admin_import/formats/csv_importer.rb

@@ -1,20 +1,16 @@
 require "csv"
-require "rchardet"
 
 module RailsAdminImport
   module Formats
     class CSVImporter < FileImporter
       Formats.register(:csv, self)
 
-      # Default is to downcase headers and add underscores to convert into attribute names
-      HEADER_CONVERTER = lambda do |header|
-        header.parameterize.underscore
-      end
+      autoload :CharDet, "rchardet"
 
       def initialize(import_model, params)
         super
         @encoding = params[:encoding]
-        @header_converter = RailsAdminImport.config.header_converter || HEADER_CONVERTER
+        @header_converter = RailsAdminImport.config.header_converter
       end
 
       # A method that yields a hash of attributes for each record to import

data/lib/rails_admin_import/formats/xlsx_importer.rb

@@ -0,0 +1,48 @@
+require "csv"
+
+module RailsAdminImport
+  module Formats
+    class XLSXImporter < FileImporter
+      Formats.register(:xlsx, self)
+
+      autoload :SimpleXlsxReader, "simple_xlsx_reader"
+
+      def initialize(import_model, params)
+        super
+        @header_converter = RailsAdminImport.config.header_converter
+      end
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        doc = SimpleXlsxReader.open(filename)
+        sheet = doc.sheets.first
+        @headers = convert_headers(sheet.headers)
+        sheet.data.each do |row|
+          yield convert_to_attributes(row)
+        end
+      end
+
+      private
+
+      def convert_headers(headers)
+        headers.map do |h|
+          @header_converter.call(h || "")
+        end
+      end
+
+      def convert_to_attributes(row)
+        row_with_headers = @headers.zip(row)
+        row_with_headers.each_with_object({}) do |(field, value), record|
+          break if field.nil?
+          field = field.to_sym
+          if import_model.has_multiple_values?(field)
+            field = import_model.pluralize_field(field)
+            (record[field] ||= []) << value
+          else
+            record[field] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_admin_import/import_model.rb

@@ -44,18 +44,28 @@ module RailsAdminImport
       }
     end
 
-    def belongs_to_fields
-      @belongs_to_fields ||= association_fields.select { |f|
+    def single_association_fields
+      @single_association_fields ||= association_fields.select { |f|
         !f.multiple?
       }
     end
 
-    def many_fields
-      @many_fields ||= association_fields.select {
-        |f| f.multiple?
+    def belongs_to_fields
+      @belongs_to_fields ||= single_association_fields.select { |f|
+        f.type == :belongs_to_association
       }
     end
 
+    def many_association_fields
+      @many_association_fields ||= association_fields.select { |f|
+        f.multiple?
+      }
+    end
+
+    def update_lookup_field_names
+      @update_lookup_field_names ||= model_fields.map(&:name) + belongs_to_fields.map(&:foreign_key)
+    end
+
     def associated_object(field, mapping_field, value)
       klass = association_class(field)
       klass.where(mapping_field => value).first or
@@ -79,11 +89,11 @@ module RailsAdminImport
 
     def has_multiple_values?(field_name)
       plural_name = pluralize_field(field_name)
-      many_fields.any? { |field| field.name == field_name || field.name == plural_name }
+      many_association_fields.any? { |field| field.name == field_name || field.name == plural_name }
     end
 
     def pluralize_field(field_name)
-      @plural_fields ||= many_fields.map(&:name).each_with_object({}) { |name, h|
+      @plural_fields ||= many_association_fields.map(&:name).each_with_object({}) { |name, h|
         h[name.to_s.singularize.to_sym] = name
       }
       @plural_fields[field_name] || field_name

data/lib/rails_admin_import/importer.rb

@@ -70,8 +70,8 @@ module RailsAdminImport
       action = object.new_record? ? :create : :update
 
       begin
-        import_belongs_to_data(object, record)
-        import_has_many_data(object, record)
+        import_single_association_data(object, record)
+        import_many_association_data(object, record)
       rescue AssociationNotFound => e
         error = I18n.t("admin.import.association_not_found", :error => e.to_s)
         report_error(object, action, error)
@@ -185,8 +185,8 @@ module RailsAdminImport
       object
     end
 
-    def import_belongs_to_data(object, record)
-      import_model.belongs_to_fields.each do |field|
+    def import_single_association_data(object, record)
+      import_model.single_association_fields.each do |field|
         mapping_key = params[:associations][field.name]
         value = extract_mapping(record[field.name], mapping_key)
 
@@ -196,8 +196,8 @@ module RailsAdminImport
       end
     end
 
-    def import_has_many_data(object, record)
-      import_model.many_fields.each do |field|
+    def import_many_association_data(object, record)
+      import_model.many_association_fields.each do |field|
         if record.has_key? field.name
           mapping_key = params[:associations][field.name]
           values = record[field.name].reject { |value| value.blank? }.map { |value|

data/lib/rails_admin_import/version.rb

@@ -1,3 +1,3 @@
 module RailsAdminImport
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_admin_import
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Steph Skardal
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-04 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -67,6 +67,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: simple_xlsx_reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - steph@endpoint.com
@@ -88,12 +116,14 @@ files:
 - lib/rails_admin_import/config.rb
 - lib/rails_admin_import/config/legacy_model.rb
 - lib/rails_admin_import/config/sections/import.rb
+- lib/rails_admin_import/eager_load.rb
 - lib/rails_admin_import/engine.rb
 - lib/rails_admin_import/formats.rb
 - lib/rails_admin_import/formats/csv_importer.rb
 - lib/rails_admin_import/formats/dummy_importer.rb
 - lib/rails_admin_import/formats/file_importer.rb
 - lib/rails_admin_import/formats/json_importer.rb
+- lib/rails_admin_import/formats/xlsx_importer.rb
 - lib/rails_admin_import/import_logger.rb
 - lib/rails_admin_import/import_model.rb
 - lib/rails_admin_import/importer.rb
@@ -124,4 +154,3 @@ signing_key:
 specification_version: 4
 summary: Import functionality for Rails Admin
 test_files: []
-has_rdoc: