rails_admin_import_no_encoding 0.1.0

data/lib/rails_admin_import/formats/dummy_importer.rb

@@ -0,0 +1,16 @@
+module RailsAdminImport
+  module Formats
+    class DummyImporter
+      def initialize(*)
+      end
+
+      def valid?
+        false
+      end
+
+      def error
+        I18n.t("admin.import.invalid_format")
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/file_importer.rb

@@ -0,0 +1,47 @@
+module RailsAdminImport
+  module Formats
+    class FileImporter
+      def initialize(import_model, params)
+        if params.has_key?(:file)
+          @filename = params[:file].tempfile
+        end
+        @import_model = import_model
+      end
+
+      attr_reader :filename, :error, :import_model
+
+      def valid?
+        if filename.nil?
+          @error = I18n.t("admin.import.missing_file")
+          false
+        else
+          true
+        end
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block_given?
+
+        if RailsAdminImport.config.logging && filename
+          copy_uploaded_file_to_log_dir
+        end
+
+        each_record(&block)
+      end
+
+      private
+
+      def each_record
+        raise "Implement each_record in subclasses"
+      end
+
+      def copy_uploaded_file_to_log_dir
+        copy_filename = "#{Time.now.strftime("%Y-%m-%d-%H-%M-%S")}-import.csv"
+        dir_path = File.join(Rails.root, "log", "import")
+        FileUtils.mkdir_p(dir_path)
+        copy_path = File.join(dir_path, copy_filename)
+        FileUtils.copy(filename, copy_path)
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/json_importer.rb

@@ -0,0 +1,32 @@
+module RailsAdminImport
+  module Formats
+    class JSONImporter < FileImporter
+      Formats.register(:json, self)
+      Formats.register(:JSON, self)
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        File.open(filename) do |file|
+          data = JSON.load(file)
+
+          if data.is_a? Hash
+            # Load array from root key
+            data = data[root_key]
+          end
+
+          if !data.is_a? Array
+            raise ArgumentError, I18n.t("admin.import.invalid_json", root_key: root_key)
+          end
+
+          data.each do |record|
+            yield record.symbolize_keys
+          end
+        end
+      end
+
+      def root_key
+        import_model.model.model_name.element.pluralize
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats/xlsx_importer.rb

@@ -0,0 +1,50 @@
+require "csv"
+
+module RailsAdminImport
+  module Formats
+    class XLSXImporter < FileImporter
+      Formats.register(:xlsx, self)
+      Formats.register(:XLSX, self)
+
+      autoload :SimpleXlsxReader, "simple_xlsx_reader"
+
+      def initialize(import_model, params)
+        super
+        @header_converter = RailsAdminImport.config.header_converter
+      end
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        doc = SimpleXlsxReader.open(filename)
+        sheet = doc.sheets.first
+        @headers = convert_headers(sheet.headers)
+        sheet.data.each do |row|
+          attr = convert_to_attributes(row)
+          yield attr unless attr.all? { |field, value| value.blank? }
+        end
+      end
+
+      private
+
+      def convert_headers(headers)
+        headers.map do |h|
+          @header_converter.call(h || "")
+        end
+      end
+
+      def convert_to_attributes(row)
+        row_with_headers = @headers.zip(row)
+        row_with_headers.each_with_object({}) do |(field, value), record|
+          next if field.nil?
+          field = field.to_sym
+          if import_model.has_multiple_values?(field)
+            field = import_model.pluralize_field(field)
+            (record[field] ||= []) << value
+          else
+            record[field] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_admin_import/formats.rb

@@ -0,0 +1,34 @@
+module RailsAdminImport
+  module Formats
+    class << self
+      def register(format, klass)
+        @registry[format.to_s] = klass
+      end
+
+      def from_file(file)
+        return unless file
+        File.extname(file.original_filename).sub(/^\./, '')
+      end
+
+      def for(format, *args)
+        @registry.fetch(format.to_s, DummyImporter).new(*args)
+      end
+
+      def all
+        @registry.keys
+      end
+
+      def reset
+        @registry = {}
+      end
+    end
+
+    reset
+  end
+end
+
+require "rails_admin_import/formats/dummy_importer"
+require "rails_admin_import/formats/file_importer"
+require "rails_admin_import/formats/csv_importer"
+require "rails_admin_import/formats/json_importer"
+require "rails_admin_import/formats/xlsx_importer"

data/lib/rails_admin_import/import_logger.rb

@@ -0,0 +1,17 @@
+module RailsAdminImport
+  class ImportLogger
+    attr_reader :logger
+
+    def initialize(log_file_name = "rails_admin_import.log")
+      if RailsAdminImport.config.logging
+        @logger = Logger.new(File.join(Rails.root, "log", log_file_name))
+      end
+    end
+    
+    def info(message)
+      if RailsAdminImport.config.logging
+        @logger.info message
+      end
+    end
+  end
+end

data/lib/rails_admin_import/import_model.rb

@@ -0,0 +1,112 @@
+module RailsAdminImport
+  class AssociationNotFound < StandardError
+  end
+
+  class ImportModel
+    def initialize(abstract_model)
+      @abstract_model = abstract_model
+      @config = abstract_model.config.import
+      @model = abstract_model.model
+    end
+
+    attr_reader :abstract_model, :model, :config
+
+    def display_name
+      abstract_model.config.label
+    end
+
+    def label_for_model(object)
+      object.public_send(label_method)
+    end
+
+    def label_method
+      @label_method ||= abstract_model.config.object_label_method
+    end
+
+    def importable_fields(model_config = config)
+      @importable_fields ||= {}
+      @importable_fields[model_config] ||= model_config.visible_fields.reject do |f|
+        # Exclude id, created_at and updated_at
+        model_config.default_excluded_fields.include? f.name
+      end
+    end
+
+    def model_fields(model_config = config)
+      @model_fields ||= {}
+      @model_fields[model_config] ||= importable_fields(model_config).select { |f|
+        !f.association? || f.association.polymorphic?
+      }
+    end
+
+    def association_fields
+      @association_fields ||= importable_fields.select { |f|
+        f.association? && !f.association.polymorphic?
+      }
+    end
+
+    def single_association_fields
+      @single_association_fields ||= association_fields.select { |f|
+        !f.multiple?
+      }
+    end
+
+    def belongs_to_fields
+      @belongs_to_fields ||= single_association_fields.select { |f|
+        f.type == :belongs_to_association
+      }
+    end
+
+    def many_association_fields
+      @many_association_fields ||= association_fields.select { |f|
+        f.multiple?
+      }
+    end
+
+    def update_lookup_field_names
+      if @config.mapping_key_list.present?
+        @update_lookup_field_names = @config.mapping_key_list
+      else
+        @update_lookup_field_names ||= model_fields.map(&:name) + belongs_to_fields.map(&:foreign_key)
+      end
+    end
+
+    def associated_object(field, mapping_field, value)
+      klass = association_class(field)
+      klass.where(mapping_field => value).first or
+        raise AssociationNotFound, "#{klass}.#{mapping_field} = #{value}"
+    end
+
+    def association_class(field)
+      field.association.klass
+    end
+
+    def associated_config(field)
+      field.associated_model_config.import
+    end
+
+    def associated_model_fields(field)
+      @associated_fields ||= {}
+      if associated_config(field).mapping_key_list.present?
+        @associated_fields[field] ||= associated_config(field).mapping_key_list
+      else
+        @associated_fields[field] ||= associated_config(field).visible_fields.select { |f|
+          !f.association?
+        }.map(&:name)
+      end
+    end
+
+    def has_multiple_values?(field_name)
+      plural_name = pluralize_field(field_name)
+      many_association_fields.any? { |field| field.name == field_name || field.name == plural_name }
+    end
+
+    def pluralize_field(field_name)
+      @plural_fields ||= many_association_fields.map(&:name).each_with_object({}) { |name, h|
+        h[name.to_s.singularize.to_sym] = name
+      }
+      @plural_fields[field_name] || field_name
+    end
+  end
+end
+
+

data/lib/rails_admin_import/importer.rb

@@ -0,0 +1,248 @@
+require "rails_admin_import/import_logger"
+
+module RailsAdminImport
+  class Importer
+    def initialize(import_model, params)
+      @import_model = import_model
+      @params = params
+    end
+
+    attr_reader :import_model, :params
+
+    class UpdateLookupError < StandardError
+    end
+
+    def import(records)
+      begin
+        init_results
+
+        if records.count > RailsAdminImport.config.line_item_limit
+          return results = {
+            success: [],
+            error: [I18n.t('admin.import.import_error.line_item_limit', limit: RailsAdminImport.config.line_item_limit)]
+          }
+        end
+
+        perform_global_callback(:before_import)
+
+        with_transaction do
+          records.each do |record|
+            catch :skip do
+              import_record(record)
+            end
+          end
+
+          rollback_if_error
+        end
+
+        perform_global_callback(:after_import)
+      rescue Exception => e
+        report_general_error("#{e} (#{e.backtrace.first})")
+      end
+
+      format_results
+    end
+
+    private
+
+    def init_results
+      @results = { :success => [], :error => [] }
+    end
+
+    def with_transaction(&block)
+      if RailsAdminImport.config.rollback_on_error &&
+        defined?(ActiveRecord)
+
+        ActiveRecord::Base.transaction &block
+      else
+        block.call
+      end
+    end
+
+    def rollback_if_error
+      if RailsAdminImport.config.rollback_on_error &&
+        defined?(ActiveRecord) &&
+        !results[:error].empty?
+
+        results[:success] = []
+        raise ActiveRecord::Rollback
+      end
+    end
+
+    def import_record(record)
+      perform_model_callback(import_model.model, :before_import_find, record)
+
+      if update_lookup && !(update_lookup - record.keys).empty?
+        raise UpdateLookupError, I18n.t("admin.import.missing_update_lookup")
+      end
+
+      object = find_or_create_object(record, update_lookup)
+      return if object.nil?
+      action = object.new_record? ? :create : :update
+
+      begin
+        perform_model_callback(object, :before_import_associations, record)
+        import_single_association_data(object, record)
+        import_many_association_data(object, record)
+      rescue AssociationNotFound => e
+        error = I18n.t("admin.import.association_not_found", :error => e.to_s)
+        report_error(object, action, error)
+        perform_model_callback(object, :after_import_association_error, record)
+        return
+      end
+
+      perform_model_callback(object, :before_import_save, record)
+
+      if object.save
+        report_success(object, action)
+        perform_model_callback(object, :after_import_save, record)
+      else
+        report_error(object, action, object.errors.full_messages.join(", "))
+        perform_model_callback(object, :after_import_error, record)
+      end
+    end
+
+    def update_lookup
+      @update_lookup ||= if params[:update_if_exists] == "1"
+                           params[:update_lookup].map(&:to_sym)
+                         end
+    end
+
+    attr_reader :results
+
+    def logger
+      @logger ||= ImportLogger.new
+    end
+
+    def report_success(object, action)
+      object_label = import_model.label_for_model(object)
+      message = I18n.t("admin.import.import_success.#{action}",
+                       :name => object_label)
+      logger.info "#{Time.now}: #{message}"
+      results[:success] << message
+    end
+
+    def report_error(object, action, error)
+      object_label = import_model.label_for_model(object)
+      message = I18n.t("admin.import.import_error.#{action}",
+                       :name => object_label,
+                       :error => error)
+      logger.info "#{Time.now}: #{message}"
+      results[:error] << message
+    end
+
+    def report_general_error(error)
+      message = I18n.t("admin.import.import_error.general", :error => error)
+      logger.info "#{Time.now}: #{message}"
+      results[:error] << message
+    end
+
+    def format_results
+      imported = results[:success]
+      not_imported = results[:error]
+      unless imported.empty?
+        results[:success_message] = format_result_message("successful", imported)
+      end
+      unless not_imported.empty?
+        results[:error_message] = format_result_message("error", not_imported)
+      end
+
+      results
+    end
+
+    def format_result_message(type, array)
+      result_count = "#{array.size} #{import_model.display_name.pluralize(array.size)}"
+      I18n.t("admin.flash.#{type}",
+             name: result_count,
+               action: I18n.t("admin.actions.import.done"))
+    end
+
+    def perform_model_callback(object, method_name, record)
+      if object.respond_to?(method_name)
+        # Compatibility: Old import hook took 2 arguments.
+        # Warn and call with a blank hash as 2nd argument.
+        if object.method(method_name).arity == 2
+          report_old_import_hook(method_name)
+          object.send(method_name, record, {})
+        else
+          object.send(method_name, record)
+        end
+      end
+    end
+
+    def report_old_import_hook(method_name)
+      unless @old_import_hook_reported
+        error = I18n.t("admin.import.import_error.old_import_hook",
+                       model: import_model.display_name,
+                       method: method_name)
+        report_general_error(error)
+        @old_import_hook_reported = true
+      end
+    end
+
+    def perform_global_callback(method_name)
+      object = import_model.model
+      object.send(method_name) if object.respond_to?(method_name)
+    end
+
+    def find_or_create_object(record, update)
+      field_names = import_model.model_fields.map(&:name)
+      new_attrs = record.select do |field_name, value|
+        field_names.include?(field_name) && !value.blank?
+      end
+
+      model = import_model.model
+      object = if update.present?
+                 query = update.each_with_object({}) do
+                   |field, query| query[field] = record[field]
+                 end
+                 model.where(query).first
+               end
+
+      if object.nil?
+        object = model.new
+        perform_model_callback(object, :before_import_attributes, record)
+        object.attributes = new_attrs
+      else
+        perform_model_callback(object, :before_import_attributes, record)
+        object.attributes = new_attrs.except(update.map(&:to_sym))
+      end
+      object
+    end
+
+    def import_single_association_data(object, record)
+      import_model.single_association_fields.each do |field|
+        mapping_key = params[:associations][field.name]
+        value = extract_mapping(record[field.name], mapping_key)
+
+        if !value.blank?
+          object.send "#{field.name}=", import_model.associated_object(field, mapping_key, value)
+        end
+      end
+    end
+
+    def import_many_association_data(object, record)
+      import_model.many_association_fields.each do |field|
+        if record.has_key? field.name
+          mapping_key = params[:associations][field.name]
+          values = record[field.name].reject { |value| value.blank? }.map { |value|
+            extract_mapping(value, mapping_key)
+          }
+
+          if !values.empty?
+            associated = values.map { |value| import_model.associated_object(field, mapping_key, value) }
+            object.send "#{field.name}=", associated
+          end
+        end
+      end
+    end
+
+    def extract_mapping(value, mapping_key)
+      if value.is_a? Hash
+        value[mapping_key]
+      else
+        value
+      end
+    end
+  end
+end