importo 2.0.4

data/app/importers/concerns/importer_dsl.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module ImporterDsl
+  extend ActiveSupport::Concern
+
+  included do
+    delegate :allow_revert?, :overridable_columns, to: :class
+  end
+
+  class_methods do
+    def friendly_name(friendly_name = nil)
+      @friendly_name = friendly_name if friendly_name
+      @friendly_name || model || name
+    end
+
+    def introduction(introduction = nil)
+      @introduction ||= []
+      @introduction = introduction if introduction
+      @introduction
+    end
+
+    def columns
+      @columns ||= {}
+      @columns
+    end
+
+    #
+    # Adds a column definition
+    #
+    # @param [Object] args
+    # @param [Object] block which will filter the results before storing the value in the attribute, this is useful for lookups or reformatting
+    def column(*args, &block)
+      options = args.extract_options!
+
+      name = args[0]
+      name ||= options[:name]
+      name ||= options[:attribute]
+
+      hint = args[1]
+      hint ||= options[:hint]
+
+      options[:scope] = self.name.underscore.to_s.tr('/', '.').to_sym
+
+      columns[name] = Importo::ImportColumn.new(name, hint, options[:explanation], options, &block)
+    end
+
+    def model(model = nil)
+      @model = model if model
+      @model
+    end
+
+    def allow_duplicates(duplicates)
+      @allow_duplicates = duplicates if duplicates
+      @allow_duplicates
+    end
+
+    def allow_revert(allow)
+      @allow_revert = allow
+    end
+
+    def allow_export(allow)
+      @allow_export = allow
+    end
+
+    def includes_header(includes_header)
+      @includes_header = includes_header if includes_header
+      @includes_header
+    end
+
+    def ignore_header(ignore_header)
+      @ignore_header = ignore_header if ignore_header
+      @ignore_header
+    end
+
+    def csv_options(csv_options = nil)
+      @csv_options = csv_options if csv_options
+      @csv_options
+    end
+
+    ##
+    # Set to true to allow duplicate rows to be processed, if false (default) duplicate rows will be marked duplicate and ignored.
+    #
+    def allow_duplicates?
+      @allow_duplicates
+    end
+
+    ##
+    # Set to true when a header is/needs to be present in the file.
+    #
+    def includes_header?
+      @includes_header
+    end
+
+    ##
+    # Allow reverting the import
+    # by default the successfully created records will be destroyed, override this behaviour with the undo method
+    #
+    def allow_revert?
+      @allow_revert
+    end
+
+    ##
+    # Allow exporting data
+    #
+    def allow_export?
+      @allow_export
+    end
+
+    ##
+    # Set to true when we need to ignore the header for structure check
+    #
+    def ignore_header?
+      @ignore_header
+    end
+
+    def overridable_columns
+      columns.select { |_name, column| column.overridable? }&.map(&:last)
+    end
+  end
+end

data/app/importers/concerns/original.rb

@@ -0,0 +1,150 @@
+
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Original
+  extend ActiveSupport::Concern
+
+  def original
+    return @original if @original && !@original.is_a?(Hash)
+
+    if import.respond_to?(:attachment_changes) && import.attachment_changes['original']
+      @original ||= import.attachment_changes['original']&.attachable
+
+      if @original.is_a?(Hash)
+        tempfile = Tempfile.new(['ActiveStorage', import.original.filename.extension_with_delimiter])
+        tempfile.binmode
+        tempfile.write(@original[:io].read)
+        @original[:io].rewind
+        tempfile.rewind
+        @original = tempfile
+      end
+    else
+      return unless import&.original&.attachment
+
+      @original = Tempfile.new(['ActiveStorage', import.original.filename.extension_with_delimiter])
+      @original.binmode
+      import.original.download { |block| @original.write(block) }
+      @original.flush
+      @original.rewind
+    end
+
+    @original
+  end
+
+  def structure_valid?
+    return true if !includes_header? || ignore_header?
+
+    invalid_header_names.count.zero?
+  end
+
+  def invalid_header_names
+    invalid_header_names_for_row(header_row)
+  end
+
+  def col_for(translated_name)
+    col = columns.detect { |k, v| v.name == translated_name || k == translated_name }
+    col ||= columns.detect { |k, v| v.allowed_names.include?(translated_name) }
+    col
+  end
+
+  private
+
+  def headers_added_by_import
+    %w[import_state import_created_id import_message import_errors].map(&:dup)
+  end
+
+  def cells_from_row(index, clean = true)
+    spreadsheet.row(index).map { |c| clean ? cleaned_data_from_cell(c) : c }
+  end
+
+  def cleaned_data_from_cell(cell)
+    return cell unless cell.respond_to?(:strip)
+
+    strip_tags cell.strip
+  end
+
+  def data_start_row
+    header_row + 1
+  end
+
+  def header_row
+    return 0 unless includes_header?
+    return @header_row if @header_row
+
+    most_valid_counts = (1..20).map do |row_nr|
+      [row_nr, cells_from_row(row_nr).reject(&:nil?).size - invalid_header_names_for_row(row_nr).size]
+    end
+    @header_row = most_valid_counts.max_by(&:last).first
+  end
+
+  def invalid_header_names_for_row(index)
+    stripped_headers = allowed_header_names.map { |name| name.to_s.gsub(/[^A-Za-z]/, '').downcase }
+    cells_from_row(index).reject { |header| stripped_headers.include?(header.to_s.gsub(/[^A-Za-z]/, '').downcase) }
+  end
+
+  def allowed_header_names
+    @allowed_header_names ||= columns.values.map(&:allowed_names).flatten + headers_added_by_import
+  end
+
+  def spreadsheet
+    @spreadsheet ||= case File.extname(original.path)
+                     when '.csv' then
+                       Roo::CSV.new(original.path, csv_options: csv_options)
+                     when '.xls' then
+                       Roo::Excel.new(original.path)
+                     when '.xlsx' then
+                       Roo::Excelx.new(original.path)
+                     else
+                       raise "Unknown file type: #{original.path.split('/').last}"
+                     end
+  end
+
+  def duplicate(row_hash, id)
+    Importo::Import.where("results @> '[{\"hash\": \"#{row_hash}\", \"state\": \"success\"}]' AND id <> :id", id: id).first
+  end
+
+  def duplicate?(row_hash, id)
+    return false if allow_duplicates? || row_hash['id'] == id
+
+    duplicate(row_hash, id)
+  end
+
+  def loop_data_rows
+    (data_start_row..spreadsheet.last_row).map do |index|
+      row = cells_from_row(index, false)
+      attributes = Hash[[attribute_names, row].transpose]
+      attributes = attributes.map do |column, value|
+        value = strip_tags(value.strip) if value.respond_to?(:strip) && columns[column]&.options[:strip_tags] != false
+        [column, value]
+      end.to_h
+      attributes.reject! { |k, _v| headers_added_by_import.include?(k) }
+
+      yield attributes, index
+    end
+  end
+
+  def row_count
+    (spreadsheet.last_row - data_start_row) + 1
+  end
+
+  def nr_to_col(number)
+    ('A'..'ZZ').to_a[number]
+  end
+
+  def attribute_names
+    return columns.keys if !includes_header? || ignore_header?
+
+    translated_header_names = cells_from_row(header_row)
+    @header_names = translated_header_names.map do |name|
+      col_for(name)&.first
+    end
+  end
+
+  def header_names
+    return columns.values.map(&:name) if !includes_header? || ignore_header?
+
+    @header_names ||= cells_from_row(header_row)
+  end
+end

data/app/importers/concerns/result_feedback.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module ResultFeedback
+  extend ActiveSupport::Concern
+
+
+    #
+    # Generates a result excel file as a stream
+    #
+    def results_file
+      xls = Axlsx::Package.new
+      xls.use_shared_strings = true
+      workbook = xls.workbook
+      workbook.styles do |style|
+        alert_cell = style.add_style(bg_color: 'dd7777')
+        duplicate_cell = style.add_style(bg_color: 'ddd777')
+
+        sheet = workbook.add_worksheet(name: I18n.t('importo.sheet.results.name'))
+
+        headers = (header_names - headers_added_by_import) + headers_added_by_import
+        rich_text_headers = headers.map { |header| Axlsx::RichText.new.tap { |rt| rt.add_run(header.dup, b: true) } }
+        sheet.add_row rich_text_headers
+        loop_data_rows do |attributes, index|
+          row_state = result(index, 'state')
+
+          style = case row_state
+                  when 'duplicate'
+                    duplicate_cell
+                  when 'failure'
+                    alert_cell
+                  end
+          sheet.add_row attributes.values + results(index), style: Array.new(attributes.values.count) + Array.new(headers_added_by_import.count) { style }
+        end
+
+        sheet.auto_filter = "A1:#{sheet.dimension.last_cell_reference}"
+      end
+
+      xls.to_stream
+    end
+
+    def file_name(suffix = nil)
+      base = friendly_name || model.class.name
+      base = base.to_s unless base.is_a?(String)
+      base = base.gsub(/[_\s-]/, '_').pluralize.downcase
+      "#{base}#{suffix.present? ? "_#{suffix}" : '' }.xlsx"
+    end
+
+  private
+
+  def register_result(index, details)
+    @import.results ||= []
+    i = @import.results.index { |data| data[:row] == index }
+    if i
+      @import.results[i].merge!(details)
+    else
+      @import.results << details.merge(row: index)
+    end
+  end
+
+  def results(index)
+    [result(index, :state), result(index, :id), result(index, :message), result(index, :errors)]
+  end
+
+  def result(index, field)
+    (@import.results.find { |result| result[:row] == index } || {}).fetch(field, nil)
+  end
+end

data/app/importers/concerns/revertable.rb

@@ -0,0 +1,41 @@
+
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Revertable
+  extend ActiveSupport::Concern
+
+  def revert!
+    undo_all
+
+    import.reverted!
+  rescue StandardError => e
+    import.result_message = "Exception: #{e.message}"
+    Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(';')}"
+    import.failure!
+  end
+
+  private
+
+  def undo_all
+    revertable_results = import.results.select { |result| result['state'] == 'success' }
+
+    revertable_results.each do |revertable_result|
+      next unless revertable_result['state'] == 'success'
+
+      begin
+        undo(revertable_result['class'], revertable_result['id'], cells_from_row(revertable_result['row']))
+        revertable_result['state'] = 'reverted'
+        revertable_result.delete('message')
+        revertable_result.delete('errors')
+      rescue StandardError => e
+        result['message'] = "Not reverted: #{e.message}"
+      end
+    end
+  end
+
+  def undo_row(klass, id, _row)
+    klass.constantize.find(id).destroy
+  end
+end

data/app/importers/importo/base_importer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Importo
+  class BaseImporter
+    include ActionView::Helpers::SanitizeHelper
+    include Importable
+    include Exportable
+    include Revertable
+    include Original
+    include ResultFeedback
+    include ImporterDsl
+    # include ActiveStorage::Downloading
+
+    delegate :friendly_name, :introduction, :model, :columns, :csv_options, :allow_duplicates?, :includes_header?,
+             :ignore_header?, :t, to: :class
+    attr_reader :import, :blob
+
+    def initialize(imprt = nil)
+      @import = imprt
+      I18n.locale = import.locale if import&.locale # Should we do this?? here??
+    end
+
+    class << self
+      def t(key, options = {})
+        if I18n.exists? "importers.#{name.underscore}#{key}".to_sym
+          I18n.t(key,
+                 options.merge(scope: "importers.#{name.underscore}".to_sym))
+        end
+      end
+    end
+  end
+end

data/app/mailers/importo/application_mailer.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Importo
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/importo/application_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Importo
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/importo/import.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Importo
+  class Import < Importo::ApplicationRecord
+    # include ActiveStorage::Downloading
+
+    belongs_to :importo_ownable, polymorphic: true
+
+    has_many :message_instances, as: :messagable
+
+    validates :kind, presence: true
+    validates :original, presence: true
+    validate :content_validator
+
+    begin
+      has_one_attached :original
+      has_one_attached :result
+    rescue NoMethodError
+      # Weird loading sequence error, is fixed by the lib/importo/helpers
+    end
+
+    state_machine :state, initial: :new do
+      audit_trail class: ResourceStateTransition, as: :resource if "ResourceStateTransition".safe_constantize
+
+      state :importing
+      state :scheduled
+      state :completed
+      state :failed
+      state :reverted
+
+      after_transition any => any do |imprt, transition|
+        CallbackService.perform_later(import: imprt, callback: transition.to_name)
+      end
+
+      after_transition any => :scheduled, do: :schedule_import
+      after_transition any => :reverting, do: :schedule_revert
+
+      event :schedule do
+        transition new: :scheduled
+      end
+
+      event :import do
+        transition new: :importing
+        transition scheduled: :importing
+        transition failed: :importing
+      end
+
+      event :complete do
+        transition importing: :completed
+      end
+
+      event :failure do
+        transition any => :failed
+      end
+
+      event :revert do
+        transition completed: :reverting
+      end
+
+      event :revert do
+        transition reverting: :reverted
+      end
+    end
+
+    def can_revert?
+      importer.allow_revert? && super
+    end
+
+    def allow_export?
+      importer.class.allow_export?
+    end
+
+    def content_validator
+      errors.add(:original, I18n.t('importo.errors.structure_invalid', invalid_headers: importer.invalid_header_names.join(', '))) unless importer.structure_valid?
+    rescue => e
+      errors.add(:original, I18n.t('importo.errors.parse_error', error: e.message))
+    end
+
+    def importer
+      @importer ||= "#{kind.camelize}Importer".constantize.new(self)
+    end
+
+    private
+
+    def schedule_import
+      ImportService.perform_later(import: self)
+    end
+
+    def schedule_revert
+      RevertService.perform_later(import: self)
+    end
+  end
+end

data/app/services/importo/application_context.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Importo
+  class ApplicationContext < Importo.config.base_service_context.constantize
+  end
+end

data/app/services/importo/application_service.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Importo
+  class ApplicationService < Importo.config.base_service.constantize
+    def self.queue_name
+      Importo.config.queue_name
+    end
+  end
+end

data/app/services/importo/callback_service.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Importo
+  class CallbackService < ApplicationService
+    context do
+      attribute :import
+      attribute :callback
+    end
+
+    def perform
+      Importo.config.import_callback(context.import, context.callback)
+    end
+  end
+end

data/app/services/importo/import_context.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Importo
+  class ImportContext < ApplicationContext
+    input do
+      attribute :import, type: Import, typecaster: ->(value) { value.is_a?(Import) ? value : Import.find(value) }
+    end
+  end
+end

data/app/services/importo/import_service.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Importo
+  class ImportService < ApplicationService
+    def perform
+      sleep 1
+
+      context.import.import!
+      context.import.importer.import!
+    rescue StandardError
+      context.import.failure!
+      context.fail!
+    end
+  end
+end

data/app/services/importo/revert_service.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Importo
+  class RevertService < ApplicationService
+    def perform
+      sleep 1
+
+      context.import.importer.revert!
+    rescue StandardError
+      context.import.failure!
+      context.fail!
+    end
+  end
+end

data/app/tables/importo/imports_table.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Importo::ImportsTable < ActionTable::ActionTable
+  model Importo::Import
+
+  column(:created_at) { |import| l(import.created_at.in_time_zone(Time.zone), format: :short).to_s }
+  column(:user, filter: { parameter: :ownable, collection_proc: -> { Importo::Import.order(created_at: :desc).limit(200).map(&:importo_ownable).uniq.sort_by(&:name).map { |o| [o.name, "#{o.class.name}##{o.id}"] } } } ) { |import| import.importo_ownable.name }
+  column(:kind, sortable: false)
+  column(:original, sortable: false) { |import| link_to(import.original.filename, main_app.rails_blob_path(import.original, disposition: "attachment"), target: '_blank') }
+  column(:state)
+  column(:result, sortable: false) { |import| import.result.attached? ? link_to(import.result_message, main_app.rails_blob_path(import.result, disposition: "attachment"), target: '_blank') : import.result_message }
+  column(:extra_links, sortable: false) { |import| Importo.config.admin_extra_links(import).map { |name, link| link_to(link[:text], link[:url], title: link[:title], target: '_blank', class: link[:icon]) }}
+
+  column :actions, title: '', sortable: false do |import|
+    content_tag(:span) do
+      if import.can_revert?
+        concat link_to(content_tag(:i, nil, class: 'fa fa-undo'), importo.undo_import_path(import), data: { turbo_method: :post, turbo_confirm: 'Are you sure? This will undo this import.' })
+      end
+      if Importo.config.admin_can_destroy(import)
+        concat link_to(content_tag(:i, nil, class: 'fa fa-trash'), importo.import_path(import), class: 'float-right', data: {  turbo_method: :delete, turbo_confirm: 'Are you sure? This will prevent duplicate imports from being detected.' })
+      end
+    end
+  end
+
+  initial_order :created_at, :desc
+
+  private
+
+  def scope
+    @scope = Importo.config.admin_visible_imports
+  end
+
+  def filtered_scope
+    @filtered_scope = scope
+    @filtered_scope = @filtered_scope.where(importo_ownable_type: params[:ownable].split('#').first, importo_ownable_id: params[:ownable].split('#').last) if params[:ownable]
+
+    @filtered_scope
+  end
+end

data/app/views/importo/imports/index.html.slim

@@ -0,0 +1,2 @@
+= sts.card title: t('.title'), icon: 'fad fa-file-import', content_padding: false do |card|
+  = sts.table :importo_imports

data/app/views/importo/imports/new.html.slim

@@ -0,0 +1,24 @@
+= sts.form_for(@import) do |f|
+  = f.input :kind, as: :hidden
+  = sts.card title: t('.title'), icon: 'fad fa-file-spreadsheet' do |card|
+    - card.action
+      = f.submit
+
+    .grid.grid-cols-12.gap-4
+      .col-span-12
+        .prose
+          p= t('.explanation_html', name: @import.importer.class.friendly_name, sample_path: sample_import_path(kind: @import.kind))
+          - if @import.allow_export?
+            p= t('.export_html', export_path: export_path(kind: @import.kind))
+
+      .col-span-12
+        - @import.importer.overridable_columns.each do |column|
+          = f.fields_for :column_overrides do |fff|
+            - if column.collection
+              = fff.input column.attribute, as: :select, label: column.name, collection: column.collection, include_blank: true, required: false
+            - else
+              =  fff.input column.attribute, label: column.name, required: false
+
+      .col-span-12
+        = f.input :original, as: :file
+