importo 2.0.5 → 3.0.9

data/app/importers/concerns/original.rb

@@ -1,7 +1,6 @@
-
 # frozen_string_literal: true
 
-require 'active_support/concern'
+require "active_support/concern"
 
 module Original
   extend ActiveSupport::Concern
@@ -9,11 +8,11 @@ module Original
   def original
     return @original if @original && !@original.is_a?(Hash)
 
-    if import.respond_to?(:attachment_changes) && import.attachment_changes['original']
-      @original ||= import.attachment_changes['original']&.attachable
+    if import.respond_to?(:attachment_changes) && import.attachment_changes["original"]
+      @original ||= import.attachment_changes["original"]&.attachable
 
       if @original.is_a?(Hash)
-        tempfile = Tempfile.new(['ActiveStorage', import.original.filename.extension_with_delimiter])
+        tempfile = Tempfile.new(["ActiveStorage", import.original.filename.extension_with_delimiter])
         tempfile.binmode
         tempfile.write(@original[:io].read)
         @original[:io].rewind
@@ -23,7 +22,7 @@ module Original
     else
       return unless import&.original&.attachment
 
-      @original = Tempfile.new(['ActiveStorage', import.original.filename.extension_with_delimiter])
+      @original = Tempfile.new(["ActiveStorage", import.original.filename.extension_with_delimiter])
       @original.binmode
       import.original.download { |block| @original.write(block) }
       @original.flush
@@ -80,8 +79,8 @@ module Original
   end
 
   def invalid_header_names_for_row(index)
-    stripped_headers = allowed_header_names.map { |name| name.to_s.gsub(/[^A-Za-z]/, '').downcase }
-    cells_from_row(index).reject { |header| stripped_headers.include?(header.to_s.gsub(/[^A-Za-z]/, '').downcase) }
+    stripped_headers = allowed_header_names.map { |name| name.to_s.gsub(/[^A-Za-z]/, "").downcase }
+    cells_from_row(index).reject { |header| stripped_headers.include?(header.to_s.gsub(/[^A-Za-z]/, "").downcase) }
   end
 
   def allowed_header_names
@@ -90,23 +89,23 @@ module Original
 
   def spreadsheet
     @spreadsheet ||= case File.extname(original.path)
-                     when '.csv' then
-                       Roo::CSV.new(original.path, csv_options: csv_options)
-                     when '.xls' then
-                       Roo::Excel.new(original.path)
-                     when '.xlsx' then
-                       Roo::Excelx.new(original.path)
-                     else
-                       raise "Unknown file type: #{original.path.split('/').last}"
-                     end
+    when ".csv"
+      Roo::CSV.new(original.path, csv_options: csv_options)
+    when ".xls"
+      Roo::Excel.new(original.path)
+    when ".xlsx"
+      Roo::Excelx.new(original.path)
+    else
+      raise "Unknown file type: #{original.path.split("/").last}"
+    end
   end
 
   def duplicate(row_hash, id)
-    Importo::Import.where("results @> '[{\"hash\": \"#{row_hash}\", \"state\": \"success\"}]' AND id <> :id", id: id).first
+    Importo::Result.where("details @> '[{\"hash\": \"#{row_hash}\", \"state\": \"success\"}]' AND import_id <> :import_id", import_id: id).first
   end
 
   def duplicate?(row_hash, id)
-    return false if allow_duplicates? || row_hash['id'] == id
+    return false if allow_duplicates? || row_hash["id"] == id
 
     duplicate(row_hash, id)
   end
@@ -114,9 +113,10 @@ module Original
   def loop_data_rows
     (data_start_row..spreadsheet.last_row).map do |index|
       row = cells_from_row(index, false)
-      attributes = Hash[[attribute_names, row].transpose]
+
+      attributes = Hash[[attribute_names, row].transpose].reject { |k, _| k.nil? }
       attributes = attributes.map do |column, value|
-        value = strip_tags(value.strip) if value.respond_to?(:strip) && columns[column]&.options[:strip_tags] != false
+        value = strip_tags(value.strip) if value.respond_to?(:strip) && columns[column]&.options&.[](:strip_tags) != false
         [column, value]
       end.to_h
       attributes.reject! { |k, _v| headers_added_by_import.include?(k) }
@@ -130,11 +130,11 @@ module Original
   end
 
   def nr_to_col(number)
-    ('A'..'ZZ').to_a[number]
+    ("A".."ZZ").to_a[number]
   end
 
   def attribute_names
-    return columns.keys if !includes_header? || ignore_header?
+    return columns.keys if !includes_header?
 
     translated_header_names = cells_from_row(header_row)
     @header_names = translated_header_names.map do |name|

data/app/importers/concerns/result_feedback.rb

@@ -1,62 +1,80 @@
 # frozen_string_literal: true
 
-require 'active_support/concern'
+require "active_support/concern"
 
 module ResultFeedback
   extend ActiveSupport::Concern
 
+  #
+  # Generates a result excel file as a stream
+  #
+  def results_file
+    xls = Axlsx::Package.new
+    xls.use_shared_strings = true
+    workbook = xls.workbook
+    workbook.styles do |style|
+      alert_cell_bg_color = "dd7777"
+      duplicate_cell_bg_color = "ddd777"
 
-    #
-    # Generates a result excel file as a stream
-    #
-    def results_file
-      xls = Axlsx::Package.new
-      xls.use_shared_strings = true
-      workbook = xls.workbook
-      workbook.styles do |style|
-        alert_cell = style.add_style(bg_color: 'dd7777')
-        duplicate_cell = style.add_style(bg_color: 'ddd777')
-
-        sheet = workbook.add_worksheet(name: I18n.t('importo.sheet.results.name'))
-
-        headers = (header_names - headers_added_by_import) + headers_added_by_import
-        rich_text_headers = headers.map { |header| Axlsx::RichText.new.tap { |rt| rt.add_run(header.dup, b: true) } }
-        sheet.add_row rich_text_headers
-        loop_data_rows do |attributes, index|
-          row_state = result(index, 'state')
-
-          style = case row_state
-                  when 'duplicate'
-                    duplicate_cell
-                  when 'failure'
-                    alert_cell
-                  end
-          sheet.add_row attributes.values + results(index), style: Array.new(attributes.values.count) + Array.new(headers_added_by_import.count) { style }
+      sheet = workbook.add_worksheet(name: I18n.t("importo.sheet.results.name"))
+      headers = (header_names - headers_added_by_import) + headers_added_by_import
+      headers_style = headers.map do |header|
+        workbook.styles.add_style(bg_color: "619657")
+      end
+      rich_text_headers = headers.map { |header| Axlsx::RichText.new.tap { |rt| rt.add_run(header.dup, b: true) } }
+      sheet.add_row rich_text_headers, style: headers_style
+      loop_data_rows do |attributes, index|
+        row_state = result(index, :state)
+        bg_color = case row_state.to_s
+        when "duplicate"
+          duplicate_cell_bg_color
+        when "failure"
+          alert_cell_bg_color
         end
-
-        sheet.auto_filter = "A1:#{sheet.dimension.last_cell_reference}"
+        styles = []
+        attributes.map do |column, value|
+          export_format = columns[column]&.options&.dig(:export, :format)
+          format_code = if export_format == "number" || (export_format.nil? && value.is_a?(Numeric))
+            "#"
+          elsif export_format == "text" || (export_format.nil? && value.is_a?(String))
+            "@"
+          elsif export_format
+            export_format.to_s
+          else
+            "General"
+          end
+          config_style = {}
+          config_style.merge!(columns[column]&.options&.[](:style)) unless columns[column]&.options&.[](:style).nil?
+          config_style.merge!({format_code: format_code, bg_color: bg_color})
+          styles << workbook.styles.add_style(config_style)
+        end
+        header_array = []
+        headers_added_by_import.count.times do |i|
+          header_array << workbook.styles.add_style(bg_color: bg_color)
+        end
+        styles += header_array
+        sheet.add_row attributes.values + results(index), style: styles
       end
 
-      xls.to_stream
+      sheet.auto_filter = "A1:#{sheet.dimension.last_cell_reference}"
     end
 
-    def file_name(suffix = nil)
-      base = friendly_name || model.class.name
-      base = base.to_s unless base.is_a?(String)
-      base = base.gsub(/[_\s-]/, '_').pluralize.downcase
-      "#{base}#{suffix.present? ? "_#{suffix}" : '' }.xlsx"
-    end
+    xls.to_stream
+  end
+
+  def file_name(suffix = nil)
+    base = friendly_name || model.class.name
+    base = base.to_s unless base.is_a?(String)
+    base = base.gsub(/[_\s-]/, "_").pluralize.downcase
+    "#{base}#{suffix.present? ? "_#{suffix}" : ""}.xlsx"
+  end
 
   private
 
   def register_result(index, details)
-    @import.results ||= []
-    i = @import.results.index { |data| data[:row] == index }
-    if i
-      @import.results[i].merge!(details)
-    else
-      @import.results << details.merge(row: index)
-    end
+    r = @import.results.find_or_create_by(row_index: index)
+    r.details = details
+    r.save
   end
 
   def results(index)
@@ -64,6 +82,6 @@ module ResultFeedback
   end
 
   def result(index, field)
-    (@import.results.find { |result| result[:row] == index } || {}).fetch(field, nil)
+    (@import.results.find_by(row_index: index)&.details || {}).fetch(field.to_s, nil)
   end
 end

data/app/importers/concerns/revertable.rb

@@ -1,7 +1,6 @@
-
 # frozen_string_literal: true
 
-require 'active_support/concern'
+require "active_support/concern"
 
 module Revertable
   extend ActiveSupport::Concern
@@ -10,27 +9,27 @@ module Revertable
     undo_all
 
     import.reverted!
-  rescue StandardError => e
+  rescue => e
     import.result_message = "Exception: #{e.message}"
-    Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(';')}"
+    Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(";")}"
     import.failure!
   end
 
   private
 
   def undo_all
-    revertable_results = import.results.select { |result| result['state'] == 'success' }
+    revertable_results = import.results.select { |result| result["state"] == "success" }
 
     revertable_results.each do |revertable_result|
-      next unless revertable_result['state'] == 'success'
+      next unless revertable_result["state"] == "success"
 
       begin
-        undo(revertable_result['class'], revertable_result['id'], cells_from_row(revertable_result['row']))
-        revertable_result['state'] = 'reverted'
-        revertable_result.delete('message')
-        revertable_result.delete('errors')
-      rescue StandardError => e
-        result['message'] = "Not reverted: #{e.message}"
+        undo(revertable_result["class"], revertable_result["id"], cells_from_row(revertable_result["row"]))
+        revertable_result["state"] = "reverted"
+        revertable_result.delete("message")
+        revertable_result.delete("errors")
+      rescue => e
+        result["message"] = "Not reverted: #{e.message}"
       end
     end
   end

data/app/importers/importo/base_importer.rb

@@ -2,6 +2,7 @@
 
 module Importo
   class BaseImporter
+    include ActiveSupport::Callbacks
     include ActionView::Helpers::SanitizeHelper
     include Importable
     include Exportable
@@ -9,10 +10,17 @@ module Importo
     include Original
     include ResultFeedback
     include ImporterDsl
+
     # include ActiveStorage::Downloading
 
+    define_callbacks :row_import
+
+    Importo::Import.state_machine.states.map(&:name).each do |state|
+      define_callbacks state
+    end
+
     delegate :friendly_name, :introduction, :model, :columns, :csv_options, :allow_duplicates?, :includes_header?,
-             :ignore_header?, :t, to: :class
+      :ignore_header?, :t, to: :class
     attr_reader :import, :blob
 
     def initialize(imprt = nil)
@@ -20,11 +28,15 @@ module Importo
       I18n.locale = import.locale if import&.locale # Should we do this?? here??
     end
 
+    def state_changed(_import, transition)
+      run_callbacks(transition.to_name) do
+      end
+    end
+
     class << self
       def t(key, options = {})
-        if I18n.exists? "importers.#{name.underscore}#{key}".to_sym
-          I18n.t(key,
-                 options.merge(scope: "importers.#{name.underscore}".to_sym))
+        if I18n.exists? :"importers.#{name.underscore}#{key}"
+          I18n.t(key, options.merge(scope: :"importers.#{name.underscore}"))
         end
       end
     end

data/app/importers/importo/import_job_callback.rb

@@ -0,0 +1,29 @@
+module Importo
+  class ImportJobCallback
+    include Rails.application.routes.url_helpers
+
+    def on_complete(_status, options)
+      options = options.deep_stringify_keys
+      import = Import.find(options["import_id"])
+      if import.present?
+        import.result.attach(io: import.importer.results_file, filename: import.importer.file_name("results"),
+          content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+
+        ActiveRecord::Base.uncached do
+          import.result_message = I18n.t("importo.importers.result_message",
+            nr: import.results.where("details @> ?", {state: "success"}.to_json).count, of: import.importer.send(:row_count))
+        end
+
+        if import.can_complete?
+          import.complete!
+        else
+          import.save!
+        end
+      end
+    end
+
+    def on_success(status, options)
+      on_complete(status, options)
+    end
+  end
+end

data/app/jobs/importo/import_job.rb

@@ -0,0 +1,40 @@
+module Importo
+  class ImportJob
+    include Sidekiq::Job
+    sidekiq_options retry: 5
+    # queue_as :integration
+    queue_as Importo.config.queue_name
+
+    sidekiq_retries_exhausted do |msg, _e|
+      attributes = msg["args"][0]
+      index = msg["args"][1]
+      import_id = msg["args"][2]
+
+      execute_row(attributes, index, import_id, true, msg["bid"])
+    end
+
+    sidekiq_retry_in do |_count, exception, _jobhash|
+      case exception
+      when Importo::RetryError
+        exception.delay
+      end
+    end
+
+    def perform(attributes, index, import_id)
+      self.class.execute_row(attributes, index, import_id, false, bid)
+    end
+
+    def self.execute_row(attributes, index, import_id, last_attempt, bid)
+      attributes = JSON.load(attributes).deep_symbolize_keys if attributes.is_a?(String)
+
+      import = Import.find(import_id)
+      record = import.importer.process_data_row(attributes, index, last_attempt: last_attempt)
+
+      batch = Sidekiq::Batch.new(bid)
+
+      if !import.completed? && import.can_complete? && batch.status.complete?
+        ImportJobCallback.new.on_complete(batch.status, {import_id: import_id})
+      end
+    end
+  end
+end

data/app/mailers/importo/application_mailer.rb

@@ -2,7 +2,7 @@
 
 module Importo
   class ApplicationMailer < ActionMailer::Base
-    default from: 'from@example.com'
-    layout 'mailer'
+    default from: "from@example.com"
+    layout "mailer"
   end
 end

data/app/models/importo/import.rb

@@ -7,6 +7,7 @@ module Importo
     belongs_to :importo_ownable, polymorphic: true
 
     has_many :message_instances, as: :messagable
+    has_many :results, class_name: "Importo::Result", dependent: :delete_all
 
     validates :kind, presence: true
     validates :original, presence: true
@@ -20,8 +21,6 @@ module Importo
     end
 
     state_machine :state, initial: :new do
-      audit_trail class: ResourceStateTransition, as: :resource if "ResourceStateTransition".safe_constantize
-
       state :importing
       state :scheduled
       state :completed
@@ -29,11 +28,11 @@ module Importo
       state :reverted
 
       after_transition any => any do |imprt, transition|
-        CallbackService.perform_later(import: imprt, callback: transition.to_name)
+        imprt.importer.state_changed(imprt, transition)
       end
 
-      after_transition any => :scheduled, do: :schedule_import
-      after_transition any => :reverting, do: :schedule_revert
+      after_transition any => :scheduled, :do => :schedule_import
+      after_transition any => :reverting, :do => :schedule_revert
 
       event :schedule do
         transition new: :scheduled
@@ -46,7 +45,7 @@ module Importo
       end
 
       event :complete do
-        transition importing: :completed
+        transition importing: :completed, if: ->(import) { import.no_processing? && import.results.count == import.importer.send(:row_count) }
       end
 
       event :failure do
@@ -71,15 +70,43 @@ module Importo
     end
 
     def content_validator
-      errors.add(:original, I18n.t('importo.errors.structure_invalid', invalid_headers: importer.invalid_header_names.join(', '))) unless importer.structure_valid?
+      unless importer.structure_valid?
+        errors.add(:original,
+          I18n.t("importo.errors.structure_invalid",
+            invalid_headers: importer.invalid_header_names.join(", ")))
+      end
     rescue => e
-      errors.add(:original, I18n.t('importo.errors.parse_error', error: e.message))
+      errors.add(:original, I18n.t("importo.errors.parse_error", error: e.message))
     end
 
     def importer
       @importer ||= "#{kind.camelize}Importer".constantize.new(self)
     end
 
+    def failure?
+      results.where("details @> ?", '{"state":"failure"}').any?
+    end
+
+    def no_failure?
+      results.where("details @> ?", '{"state":"failure"}').none?
+    end
+
+    def success?
+      results.where("details @> ?", '{"state":"success"}').any?
+    end
+
+    def no_succes?
+      results.where("details @> ?", '{"state":"success"}').none?
+    end
+
+    def processing?
+      results.where("details @> ?", '{"state":"processing"}').any?
+    end
+
+    def no_processing?
+      results.where("details @> ?", '{"state":"processing"}').none?
+    end
+
     private
 
     def schedule_import

data/app/models/importo/result.rb

@@ -0,0 +1,5 @@
+module Importo
+  class Result < ApplicationRecord
+    belongs_to :import, class_name: "Importo::Import"
+  end
+end

data/app/services/importo/import_service.rb

@@ -3,11 +3,9 @@
 module Importo
   class ImportService < ApplicationService
     def perform
-      sleep 1
-
       context.import.import!
       context.import.importer.import!
-    rescue StandardError
+    rescue
       context.import.failure!
       context.fail!
     end

data/app/services/importo/revert_service.rb

@@ -6,7 +6,7 @@ module Importo
       sleep 1
 
       context.import.importer.revert!
-    rescue StandardError
+    rescue
       context.import.failure!
       context.fail!
     end

data/app/tables/importo/imports_table.rb

@@ -3,37 +3,33 @@
 class Importo::ImportsTable < ActionTable::ActionTable
   model Importo::Import
 
-  column(:created_at) { |import| l(import.created_at.in_time_zone(Time.zone), format: :short).to_s }
-  column(:user, filter: { parameter: :ownable, collection_proc: -> { Importo::Import.order(created_at: :desc).limit(200).map(&:importo_ownable).uniq.sort_by(&:name).map { |o| [o.name, "#{o.class.name}##{o.id}"] } } } ) { |import| import.importo_ownable.name }
+  column(:created_at, html_value: proc { |import| l(import.created_at.in_time_zone(Time.zone), format: :short).to_s })
+  column(:user, sortable: false) { |import| import.importo_ownable.name }
   column(:kind, sortable: false)
-  column(:original, sortable: false) { |import| link_to(import.original.filename, main_app.rails_blob_path(import.original, disposition: "attachment"), target: '_blank') }
+  column(:original, sortable: false) { |import| link_to(import.original.filename, main_app.rails_blob_path(import.original, disposition: "attachment"), target: "_blank") }
   column(:state)
-  column(:result, sortable: false) { |import| import.result.attached? ? link_to(import.result_message, main_app.rails_blob_path(import.result, disposition: "attachment"), target: '_blank') : import.result_message }
-  column(:extra_links, sortable: false) { |import| Importo.config.admin_extra_links(import).map { |name, link| link_to(link[:text], link[:url], title: link[:title], target: '_blank', class: link[:icon]) }}
+  column(:result, sortable: false) { |import| import.result.attached? ? link_to(import.result_message, main_app.rails_blob_path(import.result, disposition: "attachment"), target: "_blank") : import.result_message }
+  column(:extra_links, sortable: false) { |import| Importo.config.admin_extra_links.call(import).map { |name, link| link_to(link[:text], link[:url], title: link[:title], target: "_blank", class: link[:icon]) } }
 
-  column :actions, title: '', sortable: false do |import|
+  column :actions, title: "", sortable: false do |import|
     content_tag(:span) do
       if import.can_revert?
-        concat link_to(content_tag(:i, nil, class: 'fa fa-undo'), importo.undo_import_path(import), data: { turbo_method: :post, turbo_confirm: 'Are you sure? This will undo this import.' })
+        concat link_to(content_tag(:i, nil, class: "fa fa-undo"), importo.undo_import_path(import), data: {turbo_method: :post, turbo_confirm: "Are you sure? This will undo this import."})
       end
-      if Importo.config.admin_can_destroy(import)
-        concat link_to(content_tag(:i, nil, class: 'fa fa-trash'), importo.import_path(import), class: 'float-right', data: {  turbo_method: :delete, turbo_confirm: 'Are you sure? This will prevent duplicate imports from being detected.' })
+      if Importo.config.admin_can_destroy.call(import)
+        concat link_to(content_tag(:i, nil, class: "fa fa-trash"), importo.import_path(import), class: "float-right", data: {turbo_method: :delete, turbo_confirm: "Are you sure? This will prevent duplicate imports from being detected."})
       end
     end
   end
 
+  # filter(:importo_ownable){ parameter: :ownable, collection_proc: -> { Importo::Import.order(created_at: :desc).limit(200).map(&:importo_ownable).uniq.sort_by(&:name).map { |o| [o.name, "#{o.class.name}##{o.id}"] } } } )
+
   initial_order :created_at, :desc
 
   private
 
   def scope
-    @scope = Importo.config.admin_visible_imports
-  end
-
-  def filtered_scope
-    @filtered_scope = scope
-    @filtered_scope = @filtered_scope.where(importo_ownable_type: params[:ownable].split('#').first, importo_ownable_id: params[:ownable].split('#').last) if params[:ownable]
-
-    @filtered_scope
+    @scope = Importo.config.admin_visible_imports.call
+    @scope
   end
 end

data/app/tables/importo/mensa_imports_table.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+if defined? Mensa
+  class Importo::MensaImportsTable < Mensa::Base
+    definition do
+      model Importo::Import
+
+      column(:created_at)
+      column(:user)
+      column(:kind)
+      column(:original)
+      column(:state)
+      column(:result)
+
+      order created_at: :desc
+    end
+
+    private
+
+    def scope
+      @scope = Importo.config.admin_visible_imports.call
+      @scope = @scope.joins("LEFT JOIN users on importo_imports.importo_ownable_type = 'User' and importo_imports.importo_ownable_id = users.id ")
+    end
+
+    def filtered_scope
+      @filtered_scope = scope
+      @filtered_scope = @filtered_scope.where(importo_ownable_type: params[:ownable].split("#").first, importo_ownable_id: params[:ownable].split("#").last) if params[:ownable]
+      @filtered_scope = @filtered_scope.where(kind: params[:kind]) if params[:kind]
+      @filtered_scope
+    end
+  end
+else
+  class Importo::MensaImportsTable
+  end
+end

data/app/views/importo/imports/index.html.slim

@@ -1,2 +1,6 @@
-= sts.card title: t('.title'), icon: 'fad fa-file-import', content_padding: false do |card|
-  = sts.table :importo_imports
+= sts.card :importo_imports, icon: 'fad fa-file-import', content_padding: false do |card|
+  - if defined? Mensa
+    = sts.mensa(:importo_mensa_imports, parameters: { kind: @import&.kind })
+  - else
+    = card.with_table(:importo_imports, parameters: { kind: @import&.kind }, custom_views: false)
+