rails_admin_import_no_encoding 0.1.0

data/Rakefile

@@ -0,0 +1,17 @@
+#!/usr/bin/env rake
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+
+# Piggyback off the Rails Admin rake tasks to set up the CI environment
+spec = Gem::Specification.find_by_name 'rails_admin'
+Dir["#{spec.gem_dir}/lib/tasks/*.rake"].each { |rake| load rake }
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task test: :spec
+
+task :default do
+  system("bundle exec rake spec")
+end

data/app/views/rails_admin/main/_results.html.haml

@@ -0,0 +1,18 @@
+- if @results && @results[:success].any?
+  .alert.alert-success.form-horizontal.denser
+    %fieldset
+      %legend
+        %i.icon-chevron-right
+        = @results[:success_message]
+      %ul.control-group{style: 'display: none'}
+        - @results[:success].each do |message|
+          %li= message
+- if @results && @results[:error].any?
+  .alert.alert-danger.form-horizontal.denser
+    %fieldset
+      %legend
+        %i.icon-chevron-down
+        = @results[:error_message]
+      %ul.control-group
+        - @results[:error].each do |message|
+          %li= message

data/app/views/rails_admin/main/_section.html.haml

@@ -0,0 +1,10 @@
+- if fields.any?
+  .form-group.control-group
+    %label.col-sm-2.control-label= t("admin.import.#{section}")
+    .col-sm-10.controls
+      %ul.list-unstyled
+        - fields.each do |field|
+          %li
+            %label= capitalize_first_letter(field.label)
+      %p.help-block= t("admin.import.help.#{section}")
+

data/app/views/rails_admin/main/import.html.haml

@@ -0,0 +1,69 @@
+= render "results"
+
+= form_tag import_path(@abstract_model), :multipart => true, class: 'form-horizontal denser' do
+
+  %input{name: "send_data", type: "hidden", value: "true"}/
+  %fieldset
+    %legend
+      %i.icon-chevron-down
+      = t('admin.import.legend.fields')
+
+    = render "section", section: "model_fields", fields: @import_model.model_fields
+    = render "section", section: "association_fields", fields: @import_model.association_fields
+
+  %fieldset
+    %legend
+      %i.icon-chevron-down
+      = t('admin.import.legend.upload')
+    .form-group.control-group
+      %label.col-sm-2.control-label{for: "file"}= t("admin.import.file")
+      .col-sm-10.controls
+        = file_field_tag :file, :class => "form-control"
+        %p.help-block= t('admin.import.help.file_limit', limit: RailsAdminImport.config.line_item_limit)
+    .form-group.control-group
+      %label.col-sm-2.control-label{for: "encoding"}= t("admin.import.encoding")
+      .col-sm-10.controls
+        = select_tag 'encoding',
+          options_for_select(Encoding.name_list.sort),
+          include_blank: true, data: { enumeration: true }
+        %p.help-block= t('admin.import.help.encoding', name: 'UTF-8')
+    .form-group.control-group
+      %label.col-sm-2.control-label= t("admin.import.update_if_exists")
+      .col-sm-10.controls
+        = check_box_tag :update_if_exists, '1', RailsAdminImport.config.update_if_exists, :class => "form-control"
+        %p.help-block= t('admin.import.help.update_if_exists')
+    .form-group.control-group
+      %label.col-sm-2.control-label{for: "update_lookup"}= t("admin.import.update_lookup")
+      .col-sm-10.controls
+        = select_tag 'update_lookup',
+          options_for_select(@import_model.update_lookup_field_names,
+            Array.wrap(@import_model.config.mapping_key).map(&:to_s)),
+          multiple: true,
+          data: { enumeration: true }
+
+  - unless @import_model.association_fields.empty?
+    %fieldset
+      %legend
+        %i.icon-chevron-down
+        = t('admin.import.legend.mapping')
+
+      - @import_model.association_fields.each do |field|
+        .form-group.control-group
+          %label.col-sm-2.control-label
+            = capitalize_first_letter(field.label)
+            = t("admin.import.mapping")
+          .col-sm-10.controls
+            = select_tag "associations[#{field.name}]",
+              options_for_select(@import_model.associated_model_fields(field),
+                Array.wrap(@import_model.associated_config(field).mapping_key).first.to_s),
+              data: { enumeration: true }
+
+  %br
+  .form-actions
+    %input{type: :hidden, name: 'return_to', value: (request.params[:return_to].presence || request.referer)}
+    %button.btn.btn-primary{type: "submit", name: "commit", data: {disable_with: "Uploading..."} }
+      %i.icon-white.icon-ok
+      = t("admin.form.save")
+    %button.btn{type: "submit", name: "_continue"}
+      %i.icon-remove
+      = t("admin.form.cancel")

data/config/locales/README.md

@@ -0,0 +1,3 @@
+See the [community contributed translation section of the README](https://github.com/stephskardal/rails_admin_import#community-contributed-translations) for more languages.
+
+To contribute a new translation, please put it in a gist and submit a pull request to link your translation in the README.

data/config/locales/import.en.yml

@@ -0,0 +1,50 @@
+
+en:
+  admin:
+    actions:
+      import:
+        title: "Import"
+        menu: "Import"
+        breadcrumb: "Import"
+        link: "Import"
+        bulk_link: "Import"
+        done: "Imported"
+    import:
+      model_fields: "Model fields"
+      association_fields: "Association fields"
+
+      file: "Data file"
+      missing_file: "You must select a file"
+      format: "File format"
+      invalid_format: "Invalid import format."
+      missing_update_lookup: "Your file must contain a column for the 'Update lookup field' you selected."
+      invalid_json: "The JSON data should be an array of records or an object with a key '%{root_key}' set to an array of records"
+      update_if_exists: "Update if exists"
+      update_lookup: "Update lookup field(s)"
+      mapping: "mapping"
+      encoding: "Encoding"
+      legend:
+        fields: "Fields to import"
+        upload: "Upload file"
+        mapping: "Related fields mapping"
+      import_success:
+        create: "Created %{name}"
+        update: "Updated %{name}"
+      import_error:
+        create: "Failed to create %{name}: %{error}"
+        update: "Failed to update %{name}: %{error}"
+        general: "Error during import: %{error}"
+        line_item_limit: "Please limit upload file to %{limit} line items."
+        old_import_hook: >
+          The import hook %{model}.%{method} should take only 1 argument.
+          Data may not imported correctly.
+          See Upgrading section readme in Rails Admin Import.
+      association_not_found: "Association not found. %{error}"
+      help:
+        model_fields: "The fields above may be included in the import file."
+        association_fields: >
+          These fields map to other tables in the database, lookup via attribute selected below.
+          For "many" associations, you may include multiple columns with the same header in the CSV file.
+        update_if_exists: "Update records found with the lookup field below instead of creating new records"
+        file_limit: "Please limit upload file to %{limit} line items."
+        encoding: "Choose file encoding. Leave empty to auto-detect. Ignored for JSON."

data/lib/rails_admin_import/action.rb

@@ -0,0 +1,46 @@
+require "rails_admin/config/actions"
+
+module RailsAdmin
+  module Config
+    module Actions
+      class Import < Base
+        RailsAdmin::Config::Actions.register(self)
+
+        register_instance_option :collection do
+          true
+        end
+
+        register_instance_option :http_methods do
+          [:get, :post]
+        end
+
+        register_instance_option :controller do
+          proc do
+            @import_model = RailsAdminImport::ImportModel.new(@abstract_model)
+
+            if request.post?
+              format = RailsAdminImport::Formats.from_file(params[:file])
+              record_importer = RailsAdminImport::Formats.for(
+                format, @import_model, params)
+
+              if record_importer.valid?
+                importer = RailsAdminImport::Importer.new(
+                  @import_model, params)
+
+                @results = importer.import(record_importer.each)
+              else
+                flash[:error] = record_importer.error
+              end
+            end
+
+            render action: @action.template_name
+          end
+        end
+
+        register_instance_option :link_icon do
+          "icon-folder-open"
+        end
+      end
+    end
+  end
+end

data/lib/rails_admin_import/config/legacy_model.rb

@@ -0,0 +1,42 @@
+module RailsAdminImport
+  module Config
+    class LegacyModel
+      attr_reader :model_name
+      def initialize(model_name)
+        @model_name = model_name
+      end
+
+      def label(_value)
+        # Ignored now
+        # RailsAdmin object_label_method will be used
+      end
+
+      def mapping_key(value)
+        config = RailsAdmin.config(model_name)
+        config.mapping_key(value)
+      end
+
+      def excluded_fields(values)
+        config = RailsAdmin.config(model_name)
+
+        # Call appropriate Rails Admin field list methods
+        config.import do
+          include_all_fields
+          exclude_fields *values
+        end
+      end
+
+      def extra_fields(values)
+        config = RailsAdmin.config(model_name)
+
+        # Call appropriate Rails Admin field list methods
+        config.import do
+          include_all_fields
+          values.each do |value|
+            field value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_admin_import/config/sections/import.rb

@@ -0,0 +1,34 @@
+require "rails_admin/config/sections/base"
+
+module RailsAdmin
+  module Config
+    module Sections
+      # Configuration of the navigation view
+      class Import < RailsAdmin::Config::Sections::Base
+        register_instance_option(:mapping_key) do
+          :name
+        end
+
+        register_instance_option(:mapping_key_list) do
+          []
+        end
+
+        register_instance_option(:default_excluded_fields) do
+          [:id, :_id, :created_at, :updated_at, :c_at, :u_at, :deleted_at]
+        end
+      end
+    end
+  end
+end
+
+section = RailsAdmin::Config::Sections::Import
+name = :import
+
+# Manually add to Rails Admin as a model configuration section until
+# there is a better API to do this
+RailsAdmin::Config::Model.send(:define_method, name) do |&block|
+  @sections = {} unless @sections
+  @sections[name] = section.new(self) unless @sections[name]
+  @sections[name].instance_eval(&block) if block
+  @sections[name]
+end

data/lib/rails_admin_import/config.rb

@@ -0,0 +1,46 @@
+require "rails_admin_import/config/legacy_model"
+
+module RailsAdminImport
+  module Config
+    class << self
+      attr_accessor :logging
+      attr_accessor :line_item_limit
+      attr_accessor :rollback_on_error
+      attr_accessor :update_if_exists
+      attr_accessor :header_converter
+      attr_accessor :csv_options
+
+      # Default is to downcase headers and add underscores to convert into attribute names
+      HEADER_CONVERTER = lambda do |header|
+        # check for nil/blank headers
+        next if header.blank?
+        header.parameterize.underscore
+      end
+
+      def model(model_name, &block)
+        unless @deprecation_shown
+          warn "RailsAdminImport::Config#model is deprecated. " \
+            "Add a import section for your model inside the rails_admin " \
+            "config block. See the Readme.md for more details"
+          @deprecation_shown = true
+        end
+        legacy_config = RailsAdminImport::Config::LegacyModel.new(model_name)
+        legacy_config.instance_eval(&block) if block
+        legacy_config
+      end
+
+      # Reset all configurations to defaults.
+      def reset
+        @logging = false
+        @line_item_limit = 1000
+        @rollback_on_error = false
+        @update_if_exists = false
+        @header_converter = HEADER_CONVERTER
+        @csv_options = {}
+      end
+    end
+
+    # Set default values for configuration options on load
+    self.reset
+  end
+end

data/lib/rails_admin_import/eager_load.rb

@@ -0,0 +1,3 @@
+require "rails_admin_import"
+require "rchardet"
+require "simple_xlsx_reader"

data/lib/rails_admin_import/engine.rb

@@ -0,0 +1,4 @@
+module RailsAdminImport
+  class Engine < Rails::Engine
+  end
+end

data/lib/rails_admin_import/formats/csv_importer.rb

@@ -0,0 +1,81 @@
+require "csv"
+
+module RailsAdminImport
+  module Formats
+    class CSVImporter < FileImporter
+      Formats.register(:csv, self)
+      Formats.register(:CSV, self)
+
+      autoload :CharDet, "rchardet"
+
+      def initialize(import_model, params)
+        super
+        @encoding = params[:encoding]
+        @header_converter = RailsAdminImport.config.header_converter
+      end
+
+      # A method that yields a hash of attributes for each record to import
+      def each_record
+        CSV.foreach(filename, csv_options) do |row|
+          attr = convert_to_attributes(row)
+          yield attr unless attr.all? { |field, value| value.blank? }
+        end
+      end
+
+      private
+
+      def csv_options
+        defaults = RailsAdminImport.config.csv_options
+        options = {
+          headers: true,
+          header_converters: @header_converter,
+          encoding: encoding,
+        }
+
+        defaults.merge(options)
+      end
+
+      def encoding
+        from_encoding =
+          if !@encoding.blank?
+            @encoding
+          else
+            detect_encoding
+          end
+
+        to_encoding = import_model.abstract_model.encoding
+
+        if from_encoding && from_encoding != to_encoding
+          "#{from_encoding}:#{to_encoding}"
+        else
+          nil
+        end
+      end
+
+      def detect_encoding
+        # Su: Remove CharlockHolmes since it require to another software http://site.icu-project.org/home
+        # which is not available on server 
+        # charset = CharlockHolmes::EncodingDetector.detect File.read(filename)
+        # if charset[:confidence] > 0.6
+        #   from_encoding = charset[:encoding]
+        #   from_encoding = "UTF-8" if from_encoding == "ascii"
+        # end
+        from_encoding = "UTF-8"
+        from_encoding
+      end
+
+      def convert_to_attributes(row)
+        row.each_with_object({}) do |(field, value), record|
+          next if field.blank?
+          field = field.to_sym
+          if import_model.has_multiple_values?(field)
+            field = import_model.pluralize_field(field)
+            (record[field] ||= []) << value
+          else
+            record[field] = value
+          end
+        end
+      end
+    end
+  end
+end