importo 2.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a20759f65aaabdc3812973d96d19ba69854eeeb75ec421b779b6da6890b2d916
+  data.tar.gz: faaba7cb0ceef82f36b161049fdd0eda76c0b14363e7a78636756624cecd6b00
+SHA512:
+  metadata.gz: 59e08eb163844c774f4cdf74bcbb1365b971049c7d986687e80d2acc99512c509907a1b148dd41f4d3e99dd1345c09b2a80595e9987fca06cf8479dff48e34bf
+  data.tar.gz: 0e750e8752d3467e4ffedd4305c1b81d17f981c604a02d48200464278a9914b87b946e6fc7a9a74b0f73900ad79508ecaf9c6e0ba44e9ac7fe8e63519e3a2f59

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2017 Andre Meij
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# Importo
+
+Short description and motivation.
+
+## Usage
+
+Add an `app/importers` folder to your Rails app which will contain all importers.
+It's recommended to add an ApplicationImporter that inherits from `Importo::BaseImporter` and that all other importers inherit from.
+
+```ruby
+class ApplicationImporter < Importo::BaseImporter
+end
+```
+
+```ruby
+class ProductsImporter < ApplicationImporter
+  includes_header true
+  allow_duplicates false
+  allow_export true
+
+  model Product
+  friendly_name 'Product'
+
+  introduction %i[what columns required_column first_line save_locally translated more_information]
+
+  column attribute: :id
+
+  # attributes
+  column attribute: :name
+  column attribute: :number
+  column attribute: :description, strip_tags: false
+  column attribute: :images do |value|
+    value.split(',').map do |image|
+      uri = URI.parse(image)
+
+      { filename: File.basename(uri.to_s), io: URI.open(uri) }
+    end
+  end
+
+  def export_scope
+    Current.account.products
+  end
+end
+```
+
+You should add translations to your locale files:
+
+```yaml
+en:
+  importers:
+    products_importer:
+      introduction:
+        what: "With this Excel sheet multiple shipments can be imported at once. Mind the following:"
+        columns: "- Columns may be deleted or their order may be changed."
+        required_column: "- Columns in red are mandatory."
+        first_line: "- The first line is an example and must be removed."
+        save_locally: "- You can save this Excel file locally and fill it in partially, so you can re-use it."
+        translated: "- Columns and contents of this sheet are translated based on your locale, make sure you import in the same locale as you download the sample file."
+        more_information: 'Check the comments with each column and the "Explanation" sheet for more information.'
+      column:
+        name: Name
+        number: Number
+        description: Description
+        images: Images
+      explanation:
+        id: Record-id, only needed if you want to update an existing record
+      hint:
+        id: 36 characters, existing of hexadecimal numbers, separated by dashes
+        images: Allows multiple image urls, separated by comma
+      introduction: null
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'importo'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install importo
+```
+
+## Contributing
+
+Contribution directions go here.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Importo'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test
+
+# Adds the Auxilium semver task
+spec = Gem::Specification.find_by_name 'auxilium'
+load "#{spec.gem_dir}/lib/tasks/semver.rake"
+

data/app/assets/config/importo_manifest.js

@@ -0,0 +1,2 @@
+//= link_directory ../javascripts/importo .js
+//= link_directory ../stylesheets/importo .css

data/app/assets/javascripts/importo/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/importo/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/concerns/maintenance_standards.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module MaintenanceStandards
+  # Informs the user and redirects when needed
+  #
+  # @param result [Boolean] was update or create succesful
+  # @param path [URL] where to redirect to
+  # @param notice [String] What to show on success
+  # @param error [String] What to show on error
+  # @param render_action [Symbol] What to render
+  #
+  def flash_and_redirect(result, path, notice, error, render_action = :edit)
+    if result
+      if params[:commit] == 'continue'
+        flash.now[:notice] = notice
+      else
+        redirect_to(path, notice: notice) && return
+      end
+    else
+      flash.now[:error] = error
+    end
+    render render_action
+  end
+end

data/app/controllers/importo/application_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Importo
+  class ApplicationController < Importo.config.base_controller.constantize
+    include MaintenanceStandards
+    include Importo.config.admin_authentication_module.constantize if Importo.config.admin_authentication_module
+  end
+end

data/app/controllers/importo/imports_controller.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_dependency 'importo/application_controller'
+
+module Importo
+  class ImportsController < ApplicationController
+    def new
+      @import = Import.new(kind: params[:kind], locale: I18n.locale)
+    end
+
+    def create
+      unless import_params
+        @import = Import.new(kind: params[:kind], locale: I18n.locale)
+        flash[:error] = t('.flash.no_file')
+        render :new
+        return
+      end
+      @import = Import.new(import_params.merge(locale: I18n.locale, importo_ownable: Importo.config.current_import_owner))
+      if @import.valid? && @import.schedule!
+        flash[:notice] = t('.flash.success', id: @import.id)
+        redirect_to action: :index
+      else
+        flash[:error] = t('.flash.error')
+        render :new
+      end
+    end
+
+    def undo
+      @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
+      if @import.can_revert? && @import.revert
+        redirect_to action: :index, notice: 'Import reverted'
+      else
+        redirect_to action: :index, alert: 'Import could not be reverted'
+      end
+    end
+
+    def destroy
+      @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
+      redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy(@import)
+
+      @import.destroy
+      redirect_to action: :index
+    end
+
+    def sample
+      import = Import.new(kind: params[:kind], locale: I18n.locale)
+      send_data import.importer.sample_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
+    end
+
+    def export
+      import = Import.new(kind: params[:kind], locale: I18n.locale)
+      send_data import.importer.export_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
+    end
+
+    def index
+      @imports = Importo.config.admin_visible_imports.order(created_at: :desc).limit(50)
+    end
+
+    private
+
+    def import_params
+      params.require(:import).permit(:original, :kind, :column_overrides, column_overrides: params.dig(:import, :column_overrides)&.keys)
+    end
+  end
+end

data/app/helpers/importo/application_helper.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Importo
+  module ApplicationHelper
+    def respond_to_missing?(method)
+      method.ends_with?('_url') || method.ends_with?('_path')
+    end
+
+    def method_missing(method, *args, &block)
+      if main_app.respond_to?(method)
+        main_app.send(method, *args)
+      else
+        super
+      end
+    end
+  end
+end

data/app/importers/concerns/exportable.rb

@@ -0,0 +1,128 @@
+
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Exportable
+  extend ActiveSupport::Concern
+
+  #
+  # Generates a sample excel file as a stream
+  #
+  def sample_file
+    export(sample_data)
+  end
+
+  def sample_data
+    [export_columns.map { |_, c| c.options[:example] || '' }]
+  end
+
+  #
+  # Generates an export based on the attributes and scope.
+  #
+  def export_file
+    export(export_data)
+  end
+
+  def export_data
+    export_scope.map { |record| export_row(record) }
+  end
+
+  def export_scope
+    if self.class.allow_export?
+      model.all
+    else
+      model.none
+    end
+  end
+
+  def export_row(record)
+    export_columns.map do |_, c|
+      value = ''
+
+      if c.options[:attribute]
+        if record.respond_to?(c.options[:attribute])
+          value = record.send(c.options[:attribute])
+          value = value&.body&.to_html if value.is_a?(ActionText::RichText)
+        end
+
+        value ||= record.attributes[c.options[:attribute].to_s]
+      end
+
+      value
+    end
+  end
+
+  def export(data_rows)
+    xls = Axlsx::Package.new
+    xls.use_shared_strings = true
+    workbook = xls.workbook
+    sheet = workbook.add_worksheet(name: friendly_name&.pluralize || model.name.demodulize.pluralize)
+    workbook.styles do |style|
+      introduction_style = style.add_style(bg_color: 'E2EEDA')
+      header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
+      header_required_style = style.add_style(b: true, bg_color: 'A8D08E', fg_color: 'C00100', border: { style: :thin, color: '000000' })
+
+      # Introduction
+      introduction.each_with_index do |intro, i|
+        text = intro.is_a?(Symbol) ? I18n.t(intro, scope: [:importers, self.class.name.underscore.to_s, :introduction]) : intro
+        sheet.add_row [text], style: [introduction_style] * export_columns.count
+        sheet.merge_cells "A#{i + 1}:#{nr_to_col(export_columns.count - 1)}#{i + 1}"
+      end
+
+      # Header row
+      sheet.add_row export_columns.values.map(&:name), style: export_columns.map { |_, c| c.options[:required] ? header_required_style : header_style }
+
+      export_columns.each.with_index do |f, i|
+        field = f.last
+        sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: '', text: field.hint, visible: false if field.hint.present?
+      end
+
+      number = workbook.styles.add_style format_code: '#'
+      text = workbook.styles.add_style format_code: '@'
+
+      styles = export_columns.map { |_, c| c.options[:example].is_a?(Numeric) ? number : text }
+
+      # Examples
+      data_rows.each do |data|
+        sheet.add_row data, style: styles
+      end
+    end
+
+    sheet.column_info[0].width = 10
+
+    sheet = workbook.add_worksheet(name: I18n.t('importo.sheet.explanation.name'))
+
+    workbook.styles do |style|
+      introduction_style = style.add_style(bg_color: 'E2EEDA')
+      header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
+
+      column_style = style.add_style(b: true)
+      required_style = style.add_style(b: true, fg_color: 'C00100')
+      wrap_style = workbook.styles.add_style alignment: { wrap_text: true }
+
+      # Introduction
+      introduction.each_with_index do |intro, i|
+        text = intro.is_a?(Symbol) ? I18n.t(intro, scope: [:importers, self.class.name.underscore.to_s, :introduction]) : intro
+        sheet.add_row [text], style: [introduction_style] * 2
+        sheet.merge_cells "A#{i + 1}:B#{i + 1}"
+      end
+
+      # Header row
+      sheet.add_row [I18n.t('importo.sheet.explanation.column'), I18n.t('importo.sheet.explanation.explanation')], style: [header_style] * 2
+      export_columns.each do |_, c|
+        styles = [c.options[:required] ? required_style : column_style, wrap_style]
+        sheet.add_row [c.name, c.explanation], style: styles
+      end
+    end
+
+    sheet.column_info[0].width = 40
+    sheet.column_info[1].width = 150
+
+    xls.to_stream
+  end
+
+  def export_columns
+    @export_columns ||= columns.reject { |_, column| column.options[:hidden] }
+  end
+end

data/app/importers/concerns/importable.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Importable
+  extend ActiveSupport::Concern
+
+  #
+  # Build a record based on the row, when you override build, depending on your needs you will need to
+  # call populate yourself, or skip this altogether.
+  #
+  def build(row)
+    populate(row)
+  end
+
+  def convert_values(row)
+    return row if row.instance_variable_get('@importo_converted_values')
+
+    row.instance_variable_set('@importo_converted_values', true)
+
+    cols_to_populate = columns.select do |_, v|
+      v.options[:attribute].present?
+    end
+
+    cols_to_populate.each do |k, col|
+      attr = col.options[:attribute]
+
+      row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute]
+
+      if col.collection
+        # see if the value is part of the collection  of (name, id) pairs, error if not.
+        value = col.collection.find { |item| item.last == row[k] || item.first == row[k] }&.last
+        raise StandardError, "#{row[k]} is not a valid value for #{col.name}" if value.nil? && row[k].present?
+      else
+        value ||= row[k]
+      end
+
+      if value.present? && col.proc
+        proc = col.proc
+        proc_result = instance_exec value, row, &proc
+        value = proc_result if proc_result
+      end
+      value ||= col.options[:default]
+
+      row[k] = value
+    end
+    row
+  end
+
+  #
+  # Assists in pre-populating the record for you
+  # It wil try and find the record by id, or initialize a new record with it's attributes set based on the mapping from columns
+  #
+  def populate(row, record = nil)
+    raise 'No attributes set for columns' unless columns.any? { |_, v| v.options[:attribute].present? }
+
+    row = convert_values(row)
+
+    result = if record
+               record
+             else
+               raise 'No model set' unless model
+
+               model.find_or_initialize_by(id: row['id'])
+             end
+
+    attributes = {}
+    cols_to_populate = columns.select do |_, v|
+      v.options[:attribute].present?
+    end
+
+    cols_to_populate.each do |k, col|
+      attr = col.options[:attribute]
+      attributes = set_attribute(attributes, attr, row[k]) if row[k].present?
+    end
+
+    result.assign_attributes(attributes)
+
+    result
+  end
+
+  #
+  # Mangle the record before saving
+  #
+  def before_save(_record, _row)
+    # Implement optionally in child class to mangle
+  end
+
+  #
+  # Any updates that have to be done after saving
+  #
+  def after_save(_record, _row)
+    # Implement optionally in child class to perform other updates
+  end
+
+  #
+  # Does the actual import
+  #
+  def import!
+    raise ArgumentError, 'Invalid data structure' unless structure_valid?
+
+    results = loop_data_rows do |attributes, index|
+      process_data_row(attributes, index)
+    end
+    @import.result.attach(io: results_file, filename: @import.importer.file_name('results'),
+                          content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+
+    @import.result_message = I18n.t('importo.importers.result_message', nr: results.compact.count, of: results.count,
+                                                                        start_row: data_start_row)
+    @import.complete!
+
+    true
+  rescue StandardError => e
+    @import.result_message = "Exception: #{e.message}"
+    Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(';')}"
+    @import.failure!
+
+    false
+  end
+
+  private
+
+  ##
+  # Overridable failure method
+  #
+  def failure(_row, _record, index, exception)
+    Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(';')}"
+  end
+
+  def process_data_row(attributes, index)
+    record = nil
+    row_hash = Digest::SHA256.base64digest(attributes.inspect)
+    duplicate_import = nil
+
+    begin
+      ActiveRecord::Base.transaction(requires_new: true) do
+        register_result(index, hash: row_hash, state: :processing)
+
+        record = build(attributes)
+        record.validate!
+        before_save(record, attributes)
+        record.save!
+        after_save(record, attributes)
+        duplicate_import = duplicate?(row_hash, record.id)
+        raise Importo::DuplicateRowError if duplicate_import
+
+        register_result(index, class: record.class.name, id: record.id, state: :success)
+      end
+      record
+    rescue Importo::DuplicateRowError
+      record_id = duplicate_import.results.find { |data| data['hash'] == row_hash }['id']
+      register_result(index, id: record_id, state: :duplicate,
+                             message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
+      nil
+    rescue StandardError => e
+      errors = record.respond_to?(:errors) && record.errors.full_messages.join(', ')
+      error_message = "#{e.message} (#{e.backtrace.first.split('/').last})"
+      failure(attributes, record, index, e)
+      register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
+      nil
+    end
+  end
+
+  def set_attribute(hash, path, value)
+    tmp_hash = path.to_s.split('.').reverse.inject(value) { |h, s| { s => h } }
+    hash.deep_merge(tmp_hash)
+  end
+end