rails_admin_import 0.1.9 → 1.0.0

data/lib/rails_admin_import/formats/csv_importer.rb

@@ -0,0 +1,75 @@
+require "csv"
+require "rchardet"
+
+module RailsAdminImport
+  module Formats
+    class CSVImporter < FileImporter
+      Formats.register(:csv, self)
+
+      # Default is to downcase headers and add underscores to convert into attribute names
+      HEADER_CONVERTER = lambda do |header|
+        header.parameterize.underscore
+      end
+
+      def initialize(import_model, params)
+        super
+        @encoding = params[:encoding]
+        @header_converter = RailsAdminImport.config.header_converter || HEADER_CONVERTER
+      end
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        CSV.foreach(filename, csv_options) do |row|
+          yield convert_to_attributes(row)
+        end
+      end
+
+      private
+
+      def csv_options
+        {
+          headers: true,
+          header_converters: @header_converter
+        }.tap do |options|
+          add_encoding!(options)
+        end
+      end
+
+      def add_encoding!(options)
+        from_encoding =
+          if !@encoding.blank?
+            @encoding
+          else
+            detect_encoding
+          end
+
+        to_encoding = import_model.abstract_model.encoding
+        if from_encoding && from_encoding != to_encoding
+          options[:encoding] = "#{from_encoding}:#{to_encoding}"
+        end
+      end
+
+      def detect_encoding
+        charset = CharDet.detect File.read(filename)
+        if charset["confidence"] > 0.6
+          from_encoding = charset["encoding"]
+          from_encoding = "UTF-8" if from_encoding == "ascii"
+        end
+        from_encoding
+      end
+
+      def convert_to_attributes(row)
+        row.each_with_object({}) do |(field, value), record|
+          break if field.nil?
+          field = field.to_sym
+          if import_model.has_multiple_values?(field)
+            field = import_model.pluralize_field(field)
+            (record[field] ||= []) << value
+          else
+            record[field] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/dummy_importer.rb

@@ -0,0 +1,16 @@
+module RailsAdminImport
+  module Formats
+    class DummyImporter
+      def initialize(*)
+      end
+
+      def valid?
+        false
+      end
+
+      def error
+        I18n.t("admin.import.invalid_format")
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/file_importer.rb

@@ -0,0 +1,45 @@
+module RailsAdminImport
+  module Formats
+    class FileImporter
+      def initialize(import_model, params)
+        if params.has_key?(:file)
+          @filename = params[:file].tempfile
+        end
+        @import_model = import_model
+      end
+
+      attr_reader :filename, :error, :import_model
+
+      def valid?
+        if filename.nil?
+          @error = I18n.t("admin.import.missing_file")
+          false
+        else
+          true
+        end
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block_given?
+
+        if RailsAdminImport.config.logging && filename
+          copy_uploaded_file_to_log_dir
+        end
+
+        each_record(&block)
+      end
+
+      private
+
+      def each_record
+        raise "Implement each_record in subclasses"
+      end
+
+      def copy_uploaded_file_to_log_dir
+        copy_filename = "#{Time.now.strftime("%Y-%m-%d-%H-%M-%S")}-import.csv"
+        copy_path = File.join(Rails.root, "log", "import", copy_filename)
+        FileUtils.copy(filename, copy_path)
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/json_importer.rb

@@ -0,0 +1,31 @@
+module RailsAdminImport
+  module Formats
+    class JSONImporter < FileImporter
+      Formats.register(:json, self)
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        File.open(filename) do |file|
+          data = JSON.load(file)
+
+          if data.is_a? Hash
+            # Load array from root key
+            data = data[root_key]
+          end
+
+          if !data.is_a? Array
+            raise ArgumentError, I18n.t("admin.import.invalid_json", root_key: root_key)
+          end
+
+          data.each do |record|
+            yield record.symbolize_keys
+          end
+        end
+      end
+
+      def root_key
+        import_model.model.model_name.element.pluralize
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats.rb

@@ -0,0 +1,28 @@
+module RailsAdminImport
+  module Formats
+    class << self
+      def register(format, klass)
+        @registry[format.to_s] = klass
+      end
+
+      def for(format, *args)
+        @registry.fetch(format.to_s, DummyImporter).new(*args)
+      end
+
+      def all
+        @registry.keys
+      end
+
+      def reset
+        @registry = {}
+      end
+    end
+
+    reset
+  end
+end
+
+require "rails_admin_import/formats/dummy_importer"
+require "rails_admin_import/formats/file_importer"
+require "rails_admin_import/formats/csv_importer"
+require "rails_admin_import/formats/json_importer"

data/lib/rails_admin_import/import_logger.rb

@@ -2,13 +2,16 @@ module RailsAdminImport
   class ImportLogger
     attr_reader :logger
 
-    def initialize(log_file_name = 'rails_admin_import.log')
-      @logger = Logger.new("#{Rails.root}/log/#{log_file_name}") if RailsAdminImport.config.logging
+    def initialize(log_file_name = "rails_admin_import.log")
+      if RailsAdminImport.config.logging
+        @logger = Logger.new(File.join(Rails.root, "log", log_file_name))
+      end
     end
     
     def info(message)
-      @logger.info message if RailsAdminImport.config.logging
+      if RailsAdminImport.config.logging
+        @logger.info message
+      end
     end
- 
   end
 end

data/lib/rails_admin_import/import_model.rb

@@ -0,0 +1,94 @@
+module RailsAdminImport
+  class AssociationNotFound < StandardError
+  end
+
+  class ImportModel
+    def initialize(abstract_model)
+      @abstract_model = abstract_model
+      @config = abstract_model.config.import
+      @model = abstract_model.model
+    end
+
+    attr_reader :abstract_model, :model, :config
+
+    def display_name
+      abstract_model.config.label
+    end
+
+    def label_for_model(object)
+      object.public_send(label_method)
+    end
+
+    def label_method
+      @label_method ||= abstract_model.config.object_label_method
+    end
+
+    def importable_fields(model_config = config)
+      @importable_fields ||= {}
+      @importable_fields[model_config] ||= model_config.visible_fields.reject do |f|
+        # Exclude id, created_at and updated_at
+        model_config.default_excluded_fields.include? f.name
+      end
+    end
+
+    def model_fields(model_config = config)
+      @model_fields ||= {}
+      @model_fields[model_config] ||= importable_fields(model_config).select { |f|
+        !f.association? || f.association.polymorphic?
+      }
+    end
+
+    def association_fields
+      @association_fields ||= importable_fields.select { |f|
+        f.association? && !f.association.polymorphic?
+      }
+    end
+
+    def belongs_to_fields
+      @belongs_to_fields ||= association_fields.select { |f|
+        !f.multiple?
+      }
+    end
+
+    def many_fields
+      @many_fields ||= association_fields.select {
+        |f| f.multiple?
+      }
+    end
+
+    def associated_object(field, mapping_field, value)
+      klass = association_class(field)
+      klass.where(mapping_field => value).first or
+        raise AssociationNotFound, "#{klass}.#{mapping_field} = #{value}"
+    end
+
+    def association_class(field)
+      field.association.klass
+    end
+
+    def associated_config(field)
+      field.associated_model_config.import
+    end
+
+    def associated_model_fields(field)
+      @associated_fields ||= {}
+      @associated_fields[field] ||= associated_config(field).visible_fields.select { |f|
+        !f.association?
+      }
+    end
+
+    def has_multiple_values?(field_name)
+      plural_name = pluralize_field(field_name)
+      many_fields.any? { |field| field.name == field_name || field.name == plural_name }
+    end
+
+    def pluralize_field(field_name)
+      @plural_fields ||= many_fields.map(&:name).each_with_object({}) { |name, h|
+        h[name.to_s.singularize.to_sym] = name
+      }
+      @plural_fields[field_name] || field_name
+    end
+  end
+end
+
+

data/lib/rails_admin_import/importer.rb

@@ -0,0 +1,224 @@
+require "rails_admin_import/import_logger"
+
+module RailsAdminImport
+  class Importer
+    def initialize(import_model, params)
+      @import_model = import_model
+      @params = params
+    end
+
+    attr_reader :import_model, :params
+
+    class UpdateLookupError < StandardError
+    end
+
+    def import(records)
+      begin
+        init_results
+
+        # TODO: re-implement file size check
+        # if file_check.readlines.size > RailsAdminImport.config.line_item_limit
+        #   return results = { :success => [], :error => ["Please limit upload file to #{RailsAdminImport.config.line_item_limit} line items."] }
+        # end
+
+        with_transaction do
+          records.each do |record|
+            import_record(record)
+          end
+
+          rollback_if_error
+        end
+      rescue Exception => e
+        report_general_error("#{e} (#{e.backtrace.first})")
+      end
+
+      format_results
+    end
+
+    private
+
+    def init_results
+      @results = { :success => [], :error => [] }
+    end
+
+    def with_transaction(&block)
+      if RailsAdminImport.config.rollback_on_error &&
+        defined?(ActiveRecord)
+
+        ActiveRecord::Base.transaction &block
+      else
+        block.call
+      end
+    end
+
+    def rollback_if_error
+      if RailsAdminImport.config.rollback_on_error &&
+        defined?(ActiveRecord) &&
+        !results[:error].empty?
+
+        results[:success] = []
+        raise ActiveRecord::Rollback
+      end
+    end
+
+    def import_record(record)
+      if update_lookup && !record.has_key?(update_lookup)
+        raise UpdateLookupError, I18n.t("admin.import.missing_update_lookup")
+      end 
+
+      object = find_or_create_object(record, update_lookup)
+      action = object.new_record? ? :create : :update
+
+      begin
+        import_belongs_to_data(object, record)
+        import_has_many_data(object, record)
+      rescue AssociationNotFound => e
+        error = I18n.t("admin.import.association_not_found", :error => e.to_s)
+        report_error(object, action, error)
+        return
+      end
+
+      perform_model_callback(object, :before_import_save, record)
+
+      if object.save
+        report_success(object, action)
+        perform_model_callback(object, :after_import_save, record)
+      else
+        report_error(object, action, object.errors.full_messages.join(", "))
+      end
+    end
+
+    def update_lookup
+      @update_lookup ||= if params[:update_if_exists] == "1"
+                           params[:update_lookup].to_sym
+                         end
+    end
+
+    attr_reader :results
+
+    def logger
+      @logger ||= ImportLogger.new
+    end
+
+    def report_success(object, action)
+      object_label = import_model.label_for_model(object)
+      message = I18n.t("admin.import.import_success.#{action}",
+                       :name => object_label)
+      logger.info "#{Time.now}: #{message}"
+      results[:success] << message
+    end
+
+    def report_error(object, action, error)
+      object_label = import_model.label_for_model(object)
+      message = I18n.t("admin.import.import_error.#{action}",
+                       :name => object_label,
+                       :error => error)
+      logger.info "#{Time.now}: #{message}"
+      results[:error] << message
+    end
+
+    def report_general_error(error)
+      message = I18n.t("admin.import.import_error.general", :error => error)
+      logger.info "#{Time.now}: #{message}"
+      results[:error] << message
+    end
+
+    def format_results
+      imported = results[:success]
+      not_imported = results[:error]
+      unless imported.empty?
+        results[:success_message] = format_result_message("successful", imported)
+      end
+      unless not_imported.empty?
+        results[:error_message] = format_result_message("error", not_imported)
+      end
+
+      results
+    end
+
+    def format_result_message(type, array)
+      result_count = "#{array.size} #{import_model.display_name.pluralize(array.size)}"
+      I18n.t("admin.flash.#{type}",
+             name: result_count,
+               action: I18n.t("admin.actions.import.done"))
+    end
+    
+    def perform_model_callback(object, method_name, record)
+      if object.respond_to?(method_name)
+        # Compatibility: Old import hook took 2 arguments.
+        # Warn and call with a blank hash as 2nd argument.
+        if object.method(method_name).arity == 2
+          report_old_import_hook(method_name)
+          object.send(method_name, record, {})
+        else
+          object.send(method_name, record)
+        end
+      end
+    end
+
+    def report_old_import_hook(method_name)
+      unless @old_import_hook_reported
+        error = I18n.t("admin.import.import_error.old_import_hook",
+                       model: import_model.display_name,
+                       method: method_name)
+        report_general_error(error)
+        @old_import_hook_reported = true
+      end
+    end
+
+    def find_or_create_object(record, update)
+      field_names = import_model.model_fields.map(&:name)
+      new_attrs = record.select do |field_name, value|
+        field_names.include?(field_name) && !value.blank?
+      end
+
+      model = import_model.model
+      object = if update.present?
+                 model.find_by(update => record[update])
+               end 
+
+      if object.nil?
+        object = model.new(new_attrs)
+      else
+        object.attributes = new_attrs.except(update.to_sym)
+      end
+      object
+    end
+
+    def import_belongs_to_data(object, record)
+      import_model.belongs_to_fields.each do |field|
+        mapping_key = params[:associations][field.name]
+        value = extract_mapping(record[field.name], mapping_key)
+
+        if !value.blank?
+          object.send "#{field.name}=", import_model.associated_object(field, mapping_key, value)
+        end
+      end
+    end
+
+    def import_has_many_data(object, record)
+      import_model.many_fields.each do |field|
+        if record.has_key? field.name
+          mapping_key = params[:associations][field.name]
+          values = record[field.name].reject { |value| value.blank? }.map { |value|
+            extract_mapping(value, mapping_key)
+          }
+
+          if !values.empty?
+            associated = values.map { |value| import_model.associated_object(field, mapping_key, value) }
+            object.send "#{field.name}=", associated
+          end
+        end
+      end
+    end
+
+    def extract_mapping(value, mapping_key)
+      if value.is_a? Hash
+        value[mapping_key]
+      else
+        value
+      end
+    end
+  end
+end
+

data/lib/rails_admin_import/rails_admin_plugin.rb

@@ -0,0 +1,23 @@
+# Load the Rails Admin gem if not already done
+require "rails_admin"
+
+# Add the Import action
+require "rails_admin_import/action"
+
+# Add the import configuration section for models
+require "rails_admin_import/config/sections/import"
+
+# Register the configuration adapter for Rails Admin
+# to allow configure_with(:import)
+module RailsAdminImport
+  module Extension
+    class ConfigurationAdapter < SimpleDelegator
+      def initialize
+        super RailsAdminImport::Config
+      end
+    end
+  end
+end
+
+RailsAdmin.add_extension :import, RailsAdminImport::Extension,
+  configuration: true

data/lib/rails_admin_import/version.rb

@@ -1,3 +1,3 @@
 module RailsAdminImport
-  VERSION = "0.1.9"
+  VERSION = "1.0.0"
 end

data/lib/rails_admin_import.rb

@@ -1,12 +1,15 @@
 require "rails_admin_import/engine"
-require "rails_admin_import/import"
+require "rails_admin_import/import_model"
+require "rails_admin_import/formats"
+require "rails_admin_import/importer"
 require "rails_admin_import/config"
+require "rails_admin_import/rails_admin_plugin"
 
 module RailsAdminImport
   def self.config(entity = nil, &block)
     if entity
       RailsAdminImport::Config.model(entity, &block)
-    elsif block_given? && ENV['SKIP_RAILS_ADMIN_INITIALIZER'] != "true"
+    elsif block_given? && ENV["SKIP_RAILS_ADMIN_INITIALIZER"] != "true"
       block.call(RailsAdminImport::Config)
     else
       RailsAdminImport::Config
@@ -18,40 +21,3 @@ module RailsAdminImport
   end
 end
 
-require 'rails_admin/config/actions'
-
-module RailsAdmin
-  module Config
-    module Actions
-      class Import < Base
-        RailsAdmin::Config::Actions.register(self)
-        
-        register_instance_option :collection do
-          true
-        end
-
-        register_instance_option :http_methods do
-          [:get, :post]
-        end
-
-        register_instance_option :controller do
-          Proc.new do
-            @response = {}
-
-            if request.post?
-              results = @abstract_model.model.run_import(params)
-              @response[:notice] = results[:success].join("<br />").html_safe if results[:success].any?
-              @response[:error] = results[:error].join("<br />").html_safe if results[:error].any?
-            end
-
-            render :action => @action.template_name
-          end
-        end
-
-        register_instance_option :link_icon do
-          'icon-folder-open'
-        end
-      end
-    end
-  end
-end