csv_import_magic 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2ae480281a547b7efbd60c64d4b1e0e0e825bd37
+  data.tar.gz: a3536a24a78375f15846e17ebf5686363c515a60
+SHA512:
+  metadata.gz: 30ed14a1f60c59fc07a17edfd39a959fc3605913eb38dbcea63687a8adf7cb674a5f80f82039237b062dfa42a95fb9a82d47556d236002a8257ff2c3cd13f21f
+  data.tar.gz: b4cdabaf4481a3f18199f8eb9eec4b530f9ac7c534237c68193ef361253275e2dee76ea22e84df4ec621b6cc55bd4f5baf7a56a65a4398eb719baac087f3cacc

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2017 Saulo Santiago
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,36 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'CsvImportMagic'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path('../spec/dummy/Rakefile', __FILE__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
+  t.pattern = 'spec/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/app/assets/javascripts/csv_import_magic/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/csv_import_magic/_theme.scss

@@ -0,0 +1,75 @@
+.page-title {
+  text-align: center
+}
+
+.column {
+  float: left;
+  width: 25%;
+}
+
+.row {
+  margin-right: -15px;
+  margin-left: -15px;
+}
+
+.csv_import {
+  display: block;
+  overflow: hidden;
+}
+
+.portlet {
+  border-radius: 4px;
+  box-shadow: 0 2px 5px 2px rgba(0,0,0,0.1);
+  margin: 0 20px 25px;
+  padding: 0 10px 20px;
+}
+
+.portlet.bordered {
+  border: 1px solid #e1e1e1 !important;
+}
+
+.portlet.light {
+  background: #fff
+}
+
+.portlet-body {
+  header {
+    background: rgba(0, 0, 0, 0.05);
+    margin: 0 -10px 8px;
+    padding: .3em 10px;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  select {
+    display: block;
+    width: 100%;
+    height: 34px;
+    padding: 6px 12px;
+    font-size: 14px;
+    line-height: 1.42857143;
+    color: #555;
+    background-color: #fff;
+    background-image: none;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,0.075);
+    box-shadow: inset 0 1px 1px rgba(0,0,0,0.075);
+    -webkit-transition: border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;
+    -o-transition: border-color ease-in-out .15s,box-shadow ease-in-out .15s;
+    transition: border-color ease-in-out .15s,box-shadow ease-in-out .15s;
+  }
+
+  .hint {
+    color: #737373;
+    display: block;
+    font-size: 12px;
+    margin: 10px 0;
+  }
+
+  td.number {
+    width: 15px;
+    color: #c7c7c7;
+  }
+}

data/app/assets/stylesheets/csv_import_magic/application.scss

@@ -0,0 +1 @@
+@import "csv_import_magic/_theme";

data/app/controllers/csv_import_magic/base_controller.rb

@@ -0,0 +1,4 @@
+module CsvImportMagic
+  class BaseController < ActionController::Base
+  end
+end

data/app/controllers/csv_import_magic/importers_controller.rb

@@ -0,0 +1,64 @@
+module CsvImportMagic
+  class ImportersController < CsvImportMagic::BaseController
+    helper_method :import_file_csv
+    layout 'csv_import_magic'
+
+    def show
+      @importer = ::Importer.find(params[:id])
+    end
+
+    def create
+      @importer = ::Importer.new(importer_params)
+
+      if @importer.save! && import_file_csv
+        redirect_to edit_importer_path(@importer), alert: t('csv_import_magic.importers_controller.create.alert')
+      end
+    rescue ActiveRecord::RecordInvalid, CSV::MalformedCSVError => e
+      redirect_to request.referrer, flash: { error: e.message }
+    end
+
+    def edit
+      @importer = ::Importer.find(params[:id])
+    end
+
+    def update
+      @importer = ::Importer.find(params[:id])
+
+      if @importer.update(csv_importer_magic_update_params)
+        CsvImportMagic::ImporterWorker.perform_async(importer_id: @importer.id, resources: resources)
+        redirect_to importer_path(@importer), flash: { notice: t('csv_import_magic.importers_controller.update.notice') }
+      else
+        errors = @importer.errors.full_messages.to_sentence
+        flash[:alert] = errors.present? ? errors : t('csv_import_magic.importers_controller.update.alert')
+        render :edit
+      end
+    end
+
+    private
+
+    def resources
+      method_name = "#{@importer.source}_resources"
+      respond_to?(method_name) ? send(method_name) : nil
+    end
+
+    def import_file_csv
+      @csv ||= begin
+        content = Paperclip.io_adapters.for(@importer.attachment).read.force_encoding('UTF-8')
+        content = content.encode('UTF-8', content.encoding, invalid: :replace, undef: :replace)
+        ::CSV.parse(content, headers: true, col_sep: column_separator)
+      end
+    end
+
+    def column_separator
+      ::CsvImportMagic::Importer.new(@importer.id).column_separator
+    end
+
+    def importer_params
+      params.require(:importer).permit!
+    end
+
+    def csv_importer_magic_update_params
+      params.require(:importer).permit(columns: [])
+    end
+  end
+end

data/app/models/importer.rb

@@ -0,0 +1,77 @@
+class Importer < ActiveRecord::Base
+  include Paperclip::Glue
+
+  STATUS = %w(pending success error).freeze
+
+  serialize :columns, Array
+
+  has_attached_file :attachment
+  has_attached_file :attachment_error
+
+  validates_attachment_content_type :attachment, :attachment_error, content_type: ['text/plain', 'text/csv', 'application/vnd.ms-excel']
+  validates_inclusion_of :status, in: STATUS
+  validates :attachment, attachment_presence: true
+  validates :source, presence: true
+  validate :uniqueness_columns
+  validate :required_columns, on: :update
+
+  belongs_to :importable, polymorphic: true
+
+  before_validation :set_parser
+
+  def source_klass
+    return if source.blank?
+    source.classify.constantize
+  end
+
+  def parser_klass
+    parser.classify.constantize
+  end
+
+  def importable_columns(name_of_parser = parser)
+    source_klass.columns_names(name_of_parser.to_sym)
+  end
+
+  private
+
+  def set_parser
+    return if source_klass.blank? || parser.present?
+    self.parser = source_klass.csv_parser_default_name
+  end
+
+  def required_columns
+    required_columns = parser_klass.new(content: nil).config.column_definitions.map do |column|
+      next unless column.required
+      column.name.to_s
+    end.compact
+
+    sameness = (columns & required_columns)
+    return if sameness == required_columns
+
+    columns_to_translate = required_columns - sameness
+    return if columns_to_translate.blank?
+
+    transalated_name_of_columns = columns_to_translate.map do |column|
+      source_klass.human_attribute_name(column)
+    end.to_sentence
+
+    errors.add(:columns, I18n.t('errors.messages.missing', count: columns_to_translate.size, columns: transalated_name_of_columns))
+  end
+
+  def uniqueness_columns
+    return if columns.empty?
+
+    headers = columns.clone
+    headers.delete('ignore')
+    duplicate_headers = headers.find_all { |element| headers.count(element) > 1 }
+
+    return if duplicate_headers.blank?
+
+    headers_to_translate = duplicate_headers.uniq
+    headers_transalated = headers_to_translate.map do |header|
+      source_klass.human_attribute_name(header)
+    end.to_sentence
+
+    errors.add(:columns, I18n.t('errors.messages.uniq', count: headers_to_translate.size, columns: headers_transalated))
+  end
+end

data/app/services/csv_import_magic/failure.rb

@@ -0,0 +1,54 @@
+module CsvImportMagic
+  class Failure
+    attr_reader :rows, :report, :csv_parsed, :importer, :identifier
+
+    def initialize(csv_parsed, importer_id)
+      @importer = ::Importer.find(importer_id)
+      @csv_parsed = csv_parsed
+      @report = csv_parsed.report
+      @rows = report.invalid_rows
+      @identifier = csv_parsed.config.identifiers.first
+    end
+
+    def generate
+      set_message_to_success
+      set_attachement_error
+    end
+
+    private
+
+    def set_message_to_success
+      return if !csv_parsed.valid_header? || rows.present?
+
+      importer.update(status: 'success', message: I18n.t('csv_import_magic.services.success'))
+    end
+
+    def set_attachement_error
+      return if rows.blank?
+
+      CSV.open(tmp_failures_file.path, 'wb', col_sep: ';') do |csv|
+        append_header(csv)
+        append_records(csv)
+      end
+
+      importer.update(status: 'error', message: I18n.t('csv_import_magic.services.failure.records_error'), attachment_error: tmp_failures_file)
+    end
+
+    def tmp_failures_file
+      @file ||= Tempfile.new(['failures', '.csv'])
+    end
+
+    def append_header(csv)
+      csv << rows.first.header.column_names.map do |column_name|
+        importer.source_klass.human_attribute_name(column_name)
+      end + [I18n.t('csv_import_magic.services.error_label')]
+    end
+
+    def append_records(csv)
+      rows.each do |row|
+        record = row.model
+        csv << row.row_array + [record.errors.full_messages.to_sentence]
+      end
+    end
+  end
+end

data/app/services/csv_import_magic/importer.rb

@@ -0,0 +1,53 @@
+module CsvImportMagic
+  class Importer
+    attr_reader :source_klass, :importer, :csv_parser_class, :resources
+
+    def initialize(importer_id, resources = nil)
+      @importer = ::Importer.find(importer_id)
+      @source_klass = importer.source_klass
+      @csv_parser_class = importer.parser_klass
+      @resources = resources.try(:symbolize_keys!)
+      @model = model_with_relation || @source_klass
+    end
+
+    def call
+      csv_parsed.run!
+      csv_parsed.report
+      csv_parsed
+    end
+
+    def column_separator
+      header = content.lines("\r")[0].gsub(/[^,;\t]/, '_')
+      header.scan(/[,;\t]/).first
+    end
+
+    private
+
+    def model_with_relation
+      return if resources.blank?
+      resources[:model].classify.constantize.find(resources[:id]).send(resources[:relation])
+    end
+
+    def csv_parsed
+      model = @model
+
+      @csv_parsed ||= csv_parser_class.new(content: content_with_new_header) do
+        model model
+      end
+    end
+
+    def content
+      @content ||= begin
+        content = Paperclip.io_adapters.for(@importer.attachment).read.force_encoding('UTF-8')
+        content.encode('UTF-8', content.encoding, invalid: :replace, undef: :replace, universal_newline: true)
+      end
+    end
+
+    def content_with_new_header
+      new_header = importer.columns.join(column_separator.to_s)
+
+      body = content.lines[1..-1].join
+      body.prepend(new_header + "\n")
+    end
+  end
+end

data/app/views/csv_import_magic/importers/edit.html.erb

@@ -0,0 +1,43 @@
+<% columns = @importer.importable_columns(@importer.parser).map { |column| [@importer.source_klass.human_attribute_name(column), column] }.unshift([t('csv_import_magic.views.importers.edit.ignore_column_label'), :ignore]) %>
+
+<h3 class='page-title'>
+  <%= t('csv_import_magic.views.importers.edit.title') %>
+  </br>
+  <small><%= t('csv_import_magic.views.importers.edit.description') %></small>
+</h3>
+
+<%= simple_form_for @importer, url: importer_path(@importer) do |f| %>
+  <div class="row csv_import">
+    <% import_file_csv.headers.each_with_index do |header, i| %>
+    <div class="column">
+      <div class="portlet light bordered">
+        <div class="portlet-body form">
+          <header>
+            <%= t('csv_import_magic.views.importers.edit.column') %> <b><%= header %></b>
+          </header>
+
+          <% selected = columns.find { |c| c.join(' ').match(/#{header}/i) }.try(:last).presence || :ignore %>
+          <%= f.input "columns][", as: :select, collection: columns, label: false, hint: t('csv_import_magic.views.importers.edit.hint'), include_blank: false, selected: f.object.columns[i] || selected %>
+
+          <header class="examples"><%= t('csv_import_magic.views.importers.edit.example_of_values') %></header>
+          <table>
+            <tbody>
+              <% import_file_csv.first(5).each_with_index do |row, c| %>
+                <tr>
+                  <td class="number"><%= c + 1 %></td>
+                  <td><%= row[header] || t('csv_import_magic.views.importers.edit.empty_values') %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+    <% end %>
+  </div>
+
+  <div class="form-row actions">
+    <%= link_to t('csv_import_magic.views.importers.edit.buttons.cancel'), 'javascript:history.back()', class: 'button button--secondary' %>
+    <%= f.submit t('csv_import_magic.views.importers.edit.buttons.import', count: import_file_csv.count), class: 'button button--primary' %>
+  </div>
+<% end %>

data/app/views/csv_import_magic/importers/show.html.erb

@@ -0,0 +1,31 @@
+<h2 class='page-title'>
+  <%= t('csv_import_magic.views.importers.show.title', model_translated: t("activemodel.models.#{@importer.source}.other")) %>
+</h2>
+
+<form style='text-align: center'>
+  <div class='form-row' id="<%= @importer.status %>">
+    <% if @importer.status == 'pending' %>
+      <%= t('csv_import_magic.views.importers.show.waiting') %>
+    <% else %>
+      <%= @importer.message %>
+    <% end %>
+  </div>
+
+  <% if @importer.status == 'pending' %>
+    <span class='loading'></span>
+  <% end %>
+
+  <% if @importer.attachment_error.present? %>
+    <%= link_to t('csv_import_magic.views.importers.show.buttons.error_file'), @importer.attachment_error.url, class: 'button button--secondary' %>
+  <% end %>
+
+  <%= link_to t('csv_import_magic.views.importers.show.buttons.back'), '/', class: 'button button--primary' %>
+</form>
+
+<script>
+  if (document.getElementById('pending') != void 0) {
+    setInterval(function(){
+      location.reload();
+    }, 5000);
+  }
+</script>

data/app/views/layouts/csv_import_magic.html.erb

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="pt">
+  <head>
+    <meta charset="utf-8">
+    <title>Title</title>
+
+    <link href="/favicon.ico" rel="shortcut icon">
+
+    <%= stylesheet_link_tag "application", :media => "all" %>
+    <%= csrf_meta_tags %>
+  </head>
+
+  <body class="csv_import_magic">
+    <%- if flash %>
+      <% flash.each do |key, value| %>
+        <div class="flash-message <%= key %>"><%= value %></div>
+      <% end %>
+    <% end %>
+
+    <div class="wrapper">
+      <main>
+        <div class="container">
+          <%= yield %>
+        </div>
+      </main>
+    </div>
+
+    <%= javascript_include_tag "application" %>
+    <%= yield :javascript %>
+  </body>
+</html>

data/app/workers/csv_import_magic/importer_worker.rb

@@ -0,0 +1,13 @@
+module CsvImportMagic
+  class ImporterWorker
+    include ::Sidekiq::Worker
+
+    sidekiq_options retry: 1
+
+    def perform(options)
+      options.symbolize_keys!
+      csv_parsed = ::CsvImportMagic::Importer.new(options[:importer_id], options[:resources]).call
+      CsvImportMagic::Failure.new(csv_parsed, options[:importer_id]).generate
+    end
+  end
+end

data/config/initializers/initializer.rb

@@ -0,0 +1 @@
+# Add initialization content here

data/config/locales/en.yml

@@ -0,0 +1,71 @@
+en:
+  models_and_attributes: &models_and_attributes
+    models:
+      importer:
+        one: Importer
+        other: Importers
+    attributes:
+      importer:
+        attachment: CSV
+        attachment_error: CSV with errors
+        source: Source
+        columns: Columns
+        status: Status
+        message: Message
+
+  activerecord:
+    <<: *models_and_attributes
+  activemodel:
+    <<: *models_and_attributes
+
+  csv_import_magic:
+    importers_controller:
+      create:
+        alert: Please, tell us which columns we can find the information you want.
+      update:
+        notice: File uploaded for processing.
+        alert: Unable to send file for processing, please try again.
+    services:
+      error_label: Errors
+      success: Records has been imported!
+      failure:
+        columns_error:
+          one: 'This missing the following column: %{columns}'
+          other: 'This missing the following columns: %{columns}'
+        records_error: Some record were not imported because they have errors!
+    views:
+      importers:
+        new:
+          title: Importer of %{model_translated}
+          description: Import your records of %{model_translated} in the fast way
+          hint: 'Upload a file <b>CSV</b> with title and headers.'
+          buttons:
+            import: Import
+        show:
+          title: Importer of %{model_translated}
+          waiting: Processing your file...
+          buttons:
+            success: Success
+            back: Back
+            error_file: See error file
+        edit:
+          title: Adjustment of importater
+          description: Indicate which columns should care
+          ignore_column_label: Ignore column
+          empty_values: Empty
+          column: Column
+          hint: Choose what type of information this column contains.
+          example_of_values: Example of values
+          buttons:
+            import:
+              one: 'Import %{count} record'
+              other: 'Import %{count} records'
+            cancel: Cancel
+  errors:
+    messages:
+      uniq:
+        one: '%{columns} should be unique'
+        other: '%{columns} should be unique'
+      missing:
+        one: '%{columns} is missing'
+        other: '%{columns} are missing'

data/config/routes.rb

@@ -0,0 +1,3 @@
+CsvImportMagic::Engine.routes.draw do
+  resources :importers, only: [:create, :edit, :update, :show]
+end

data/db/migrate/20170111145652_create_companies.rb

@@ -0,0 +1,15 @@
+class CreateCompanies < ActiveRecord::Migration
+  def change
+    create_table :companies do |t|
+      t.references :user
+      t.string :name
+      t.string :street
+      t.string :number
+      t.string :neighborhood
+      t.string :city
+      t.string :state
+      t.string :country
+      t.boolean :active
+    end
+  end
+end

data/db/migrate/20170111145657_create_importers.rb

@@ -0,0 +1,15 @@
+class CreateImporters < ActiveRecord::Migration
+  def change
+    create_table :importers do |t|
+      t.attachment :attachment
+      t.attachment :attachment_error
+      t.string :source
+      t.string :parser
+      t.string :columns
+      t.string :message
+      t.string :status, default: 'pending'
+      t.references :importable, polymorphic: true
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20170112125734_create_user.rb

@@ -0,0 +1,8 @@
+class CreateUser < ActiveRecord::Migration
+  def change
+    create_table :users do |t|
+      t.string :name
+      t.string :email
+    end
+  end
+end

data/lib/csv_import_magic/engine.rb

@@ -0,0 +1,5 @@
+module CsvImportMagic
+  class Engine < ::Rails::Engine
+    isolate_namespace CsvImportMagic
+  end
+end