importo 3.0.21 → 3.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcca807f9c6f4ca09145b27128b1df2564b226bae7ff922618d399298cf8c231
-  data.tar.gz: b63912b0490807c5f99dc6928dd69b189cfcd0b92611ac6982d27c45a64b33e4
+  metadata.gz: cd095c43d38bb324dea0b158ebe7743bd84630c526b755fc7b031e761fff8e91
+  data.tar.gz: 772c031fe4446ce6db81e22c90f57ae9fb1dbfe1cbadd8ee62e0500fd3fe7a4e
 SHA512:
-  metadata.gz: 9b01d49b3e56b5823331572c42cccf441325aae495699bc7d52ecb1fb7aaf6468f685ffbfe3412ebc0f64c9d0c34b99f7340f21f420ce655d7804a0aad5beeaa
-  data.tar.gz: 553dadb41cbe13aae4857265964fd3cf449c641f0bc199938b22db259bc3ce897ea7a34b340f2bb4bf7380647ec5f6a216e4efc0a378d15714bb89f38669e0ce
+  metadata.gz: ffc02a4f1a09ad49a624206c615cfcd44a81dd489c28d7c7cdbf6780b451c786e5fde236503585cbf864faaa3e870781de5286b64e10dad21d53d50d1b8e75e5
+  data.tar.gz: 75c6b5d26d837b23e5c7ac49514007121346dfb6ef1a7978e80a5105ebe41c666daf986a44c20fe3484bfefddc9d126d0fdad07e379036ccad214a4ca6aec243

data/README.md

@@ -92,8 +92,12 @@ Add this line to your application's Gemfile:
 gem 'importo'
 ```
 
-Importo depends on Sidekiq Pro's batch functionality, 
-though you can use [sidekiq-batch](https://github.com/entdec/sidekiq-batch) as a drop-in for that.
+Importo needs Batch functionality, this can either be:
+- Sidekiq Pro's batch functionality,
+- [sidekiq-batch](https://github.com/entdec/sidekiq-batch)
+- GoodJob's Batch 
+
+Additionally it either needs [Servitium](https://github.com/entdec/servitium) or [Facio](https://github.com/entdec/facio)
 
 And then execute:
 

data/app/controllers/importo/imports_controller.rb

@@ -8,6 +8,24 @@ module Importo
       @import = Import.new(kind: params[:kind], locale: I18n.locale)
     end
 
+    def preview
+      @import = Import.find(params[:id])
+      if @import&.original&.attachment.present?
+        @original = Tempfile.new(['ActiveStorage', @import.original.filename.extension_with_delimiter])
+        @original.binmode
+        @import.original.download { |block| @original.write(block) }
+        @original.flush
+        @original.rewind
+        sheet =  Roo::Excelx.new(@original.path)
+        @sheet_data = sheet.parse(headers: true)
+        @check_header = @sheet_data.reject{ |h| h.keys == h.values }.map{|h| h.transform_values(&:to_s).compact_blank}.reduce({}, :merge).keys
+      end
+      if @sheet_data.nil? || @sheet_data.reject{ |h| h.keys == h.values }.blank?
+        Signum.error(Current.user, text: t('.no_file'))
+        redirect_to action: :new
+      end
+    end
+
     def create
       unless import_params
         @import = Import.new(kind: params[:kind], locale: I18n.locale)
@@ -16,21 +34,46 @@ module Importo
         return
       end
       @import = Import.new(import_params.merge(locale: I18n.locale,
-        importo_ownable: Importo.config.current_import_owner.call))
-      if @import.valid? && @import.schedule!
-        redirect_to importo.new_import_path(params[:kind] || @import.kind)
+                                               importo_ownable: Importo.config.current_import_owner.call))
+      if params["commit"] == "upload" && @import.valid? && @import.save!
+        @import.confirm!
+        @import.schedule!
+        redirect_to importo.new_import_path(params[:kind] || @import.kind)                                         
+      elsif params["commit"] == "preview" && @import.valid?
+        @import.save!
+        redirect_to action: :preview, id: @import.id, kind: @import.kind 
       else
         Signum.error(Current.user, text: t(".flash.error", error: @import.errors&.full_messages&.join(".")))
         render :new
       end
     end
 
+    def cancel
+      @import = Import.find(params[:id])
+      @import.original.purge if @import.concept?
+      Signum.error(Current.user, text: t('.flash.cancel', id: @import.id))
+      # flash[:notice] = t('.flash.cancel', id: @import.id)
+      redirect_to action: :new, kind: @import.kind
+    end
+
     def undo
       @import = Import.where(importo_ownable: Importo.config.current_import_owner.call).find(params[:id])
       if @import.can_revert? && @import.revert
         redirect_to action: :index, notice: "Import reverted"
       else
-        redirect_to action: :index, alert: "Import could not be reverted"
+        redirect_to action: :index, alert: 'Import could not be reverted'
+      end
+    end
+
+    def upload
+      @import = Import.find(params[:id])
+      @import.checked_columns = params[:selected_items]&.reject { |element| element == "0" }
+      @import.confirm! if @import.can_confirm?
+      if @import.valid? && @import.schedule!
+        redirect_to action: :index
+      else
+        Signum.error(Current.user, text: t('.flash.error', id: @import.id))
+        render :new
       end
     end
 

data/app/importers/concerns/importable.rb

@@ -123,28 +123,36 @@ module Importable
   #
   # Does the actual import
   #
-  def import!
-    raise ArgumentError, "Invalid data structure" unless structure_valid?
+  def import!(checked_columns)
+    raise ArgumentError, 'Invalid data structure' unless structure_valid?
 
-    batch = Importo::SidekiqBatchAdapter.new
+    batch = Importo.config.batch_adapter.new
     batch.description = "#{import.original.filename} - #{import.kind}"
     batch.properties = {import_id: import.id}
-    batch.on_success("Importo::ImportJobCallback")
+    if Importo.config.batch_adapter == Importo::SidekiqBatchAdapter
+      batch.on_success("Importo::ImportJobCallback")
+    else
+      batch.on_success = "Importo::ImportJobCallback"
+    end
 
     batch.add do
       column_with_delay = columns.select { |k, v| v.delay.present? }
-      loop_data_rows do |attributes, index|
+      loop_data_rows(checked_columns) do |attributes, index|
         if column_with_delay.present?
           delay = column_with_delay.filter_map do |k, v|
             next unless attributes[k].present?
             v.delay.call(attributes[k])
           end
         end
-        Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present?
-        Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present?
+        Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_later(JSON.dump(attributes), index, import.id) if delay.present?
+        Importo::ImportJob.perform_later(JSON.dump(attributes), index, import.id) unless delay.present?
       end
     end
 
+    if defined?(GoodJob::Batch) && Importo.config.batch_adapter == GoodJob::Batch
+      batch.enqueue
+    end
+
     true
   rescue => e
     @import.result_message = "Exception: #{e.message}"

data/app/importers/concerns/original.rb

@@ -110,7 +110,7 @@ module Original
     duplicate(row_hash, id)
   end
 
-  def loop_data_rows
+  def loop_data_rows(checked_columns =  nil)
     (data_start_row..spreadsheet.last_row).map do |index|
       row = cells_from_row(index, false)
 
@@ -120,7 +120,10 @@ module Original
         [column, value]
       end.to_h
       attributes.reject! { |k, _v| headers_added_by_import.include?(k) }
-
+      if checked_columns&.dig(:checked_columns).present?
+        selected_columns = checked_columns[:checked_columns].map{|i| col_for(i)&.first}
+        attributes.reject!{|k, _v| selected_columns.exclude?(k) }
+      end
       yield attributes, index
     end
   end

data/app/importers/importo/import_job_callback.rb

@@ -1,19 +1,20 @@
 module Importo
-  class ImportJobCallback
+  class ImportJobCallback < ActiveJob::Base
+    include Sidekiq::Batch::Callback
     include Rails.application.routes.url_helpers
 
-    def on_complete(_status, options)
-      options = options.deep_stringify_keys
-      import = Import.find(options["import_id"])
+    def perform(batch, params)
+      import = Import.find(batch.properties[:import_id])
+      complete_import(import)
+    end
+
+    def complete_import(import)
       if import.present?
         results_file = import.importer.results_file
-        if results_file.is_a?(StringIO)
-          import.result.attach(io: results_file, filename: import.importer.file_name("results"),
-                               content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-        else
-          import.result.attach(io: File.open(results_file), filename: import.importer.file_name("results"),
-                               content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-        end
+        results_file = results_file.is_a?(StringIO) ? results_file : File.open(results_file)
+
+        import.result.attach(io: results_file, filename: import.importer.file_name("results"),
+          content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
 
         ActiveRecord::Base.uncached do
           import.result_message = I18n.t("importo.importers.result_message",
@@ -27,5 +28,12 @@ module Importo
         end
       end
     end
+
+    def on_complete(status, options)
+      options = options.deep_stringify_keys
+      import = Import.find(options["import_id"])
+      complete_import(import)
+    end
+
   end
 end

data/app/jobs/importo/import_job.rb

@@ -1,40 +1,30 @@
 module Importo
-  class ImportJob
-    include Sidekiq::Job
-    sidekiq_options retry: 5
-    # queue_as :integration
+  class ImportJob < ApplicationJob
+    # No options here, gets added from the adapter
     queue_as Importo.config.queue_name
-
-    sidekiq_retries_exhausted do |msg, _e|
-      attributes = msg["args"][0]
-      index = msg["args"][1]
-      import_id = msg["args"][2]
-
-      execute_row(attributes, index, import_id, true, msg["bid"])
-    end
-
-    sidekiq_retry_in do |_count, exception, _jobhash|
-      case exception
-      when Importo::RetryError
-        exception.delay
-      end
-    end
+    include GoodJob::ActiveJobExtensions::Batches
 
     def perform(attributes, index, import_id)
-      self.class.execute_row(attributes, index, import_id, false, bid)
+      self.class.execute_row(attributes, index, import_id, false)
     end
 
-    def self.execute_row(attributes, index, import_id, last_attempt, bid)
+    def self.execute_row(attributes, index, import_id, last_attempt)
       attributes = JSON.load(attributes).deep_symbolize_keys if attributes.is_a?(String)
 
       import = Import.find(import_id)
-      record = import.importer.process_data_row(attributes, index, last_attempt: last_attempt)
-
-      batch = Importo::SidekiqBatchAdapter.find(bid)
-
-      if !import.completed? && import.can_complete? && batch.finished?
-        ImportJobCallback.new.on_complete(batch.status, {import_id: import.id})
-      end
+      import.importer.process_data_row(attributes, index, last_attempt: last_attempt)
+
+      # Between sidekiq and good job, there's a big difference:
+      # - Sidekiq calls on_complete callback when all jobs ran at least once.
+      # - GoodJob calls on_complete callback when all jobs are done (including retries).
+      # i.e. this logic is only needed for sidekiq
+      # if Importo.config.batch_adapter == Importo::SidekiqBatchAdapter
+      #   batch = Importo::SidekiqBatchAdapter.find(bid)
+
+      #   if !import.completed? && import.can_complete? && batch.finished?
+      #     ImportJobCallback.new.on_complete("success", {import_id: import.id})
+      #   end
+      # end
     end
   end
 end

data/app/jobs/importo/purge_import_job.rb

@@ -1,7 +1,9 @@
 module Importo
   class PurgeImportJob < ApplicationJob
-    def perform(owner, months)
+    def perform(owner, months,state = nil)
+
       imports = Import.where(importo_ownable: owner, created_at: ..months.months.ago.beginning_of_day)
+      imports = imports.where(state: state) if state
 
       imports.each do |import|
         import.original.purge

data/app/models/importo/import.rb

@@ -3,7 +3,7 @@
 module Importo
   class Import < Importo::ApplicationRecord
     # include ActiveStorage::Downloading
-
+    attr_accessor :checked_columns
     belongs_to :importo_ownable, polymorphic: true
 
     has_many :message_instances, as: :messagable
@@ -12,7 +12,6 @@ module Importo
     validates :kind, presence: true
     validates :original, presence: true
     validate :content_validator
-
     begin
       has_one_attached :original
       has_one_attached :result
@@ -20,7 +19,8 @@ module Importo
       # Weird loading sequence error, is fixed by the lib/importo/helpers
     end
 
-    state_machine :state, initial: :new do
+    state_machine :state, initial: :concept do
+      state :confirmed
       state :importing
       state :scheduled
       state :completed
@@ -35,11 +35,15 @@ module Importo
       after_transition any => :reverting, :do => :schedule_revert
 
       event :schedule do
-        transition new: :scheduled
+        transition confirmed: :scheduled
+      end
+
+      event :confirm do
+        transition concept: :confirmed
       end
 
       event :import do
-        transition new: :importing
+        transition confirmed: :importing
         transition scheduled: :importing
         transition failed: :importing
       end
@@ -111,7 +115,7 @@ module Importo
     private
 
     def schedule_import
-      ImportScheduleJob.perform_in(5.seconds, id)
+      ImportService.perform_later(import: self, checked_columns: self.checked_columns)
     end
 
     def schedule_revert

data/app/services/importo/import_context.rb

@@ -1,16 +1,8 @@
 # frozen_string_literal: true
 
 module Importo
-  if defined?(Servitium) && ApplicationContext < Servitium::Context
-    class ImportContext < ApplicationContext
-      input do
-        attribute :import, type: Import, typecaster: ->(value) { value.is_a?(Import) ? value : Import.find(value) }
-      end
-    end
-  else
-    class ImportContext < ApplicationContext
-      attribute :import, :model, class_name: "Importo::Import"
-    end
-
+  class ImportContext < ApplicationContext
+      attribute :import, type: Import, typecaster: ->(value) { value.is_a?(Import) ? value : Import.find(value) }
+      attribute :checked_columns, type: Array, typecaster: ->(value){ value.is_a?(Array) ? value : [] }
   end
 end

data/app/services/importo/import_service.rb

@@ -4,9 +4,10 @@ module Importo
   class ImportService < ApplicationService
     def perform
       context.import.import!
-      context.import.importer.import!
-    rescue
-        context.import.failure!
+      context.import.importer.import!(checked_columns: context.checked_columns )
+    rescue StandardError
+      context.import.failure!
+      context.fail!
     end
   end
 end

data/app/views/importo/imports/new.html.slim

@@ -4,7 +4,9 @@
       = f.input :kind, as: :hidden
       = sts.card :"importo_imports #{@import.kind}", title: t('.title'), icon: 'fad fa-file-spreadsheet' do |card|
         - card.with_action
-          = f.submit
+          = f.button 'preview', value: 'preview',  class: 'button secondary'
+        - card.with_action
+          = f.submit nil, value: 'upload'  
 
         .grid.grid-cols-12.gap-4
           .col-span-12

data/app/views/importo/imports/preview.html.slim

@@ -0,0 +1,36 @@
+= sts.form_with(model: false, url: upload_import_path) do |f|
+  = sts.card :"importo_imports #{@import.kind}", title: t('.title', kind: @import&.kind.capitalize), icon: 'fad fa-file-spreadsheet' do |card|
+    - card.with_action
+      / = f.button 'Cancel', value: 'cancel'
+      = f.submit 'Upload', value: 'upload'
+      = link_to(t('.cancel'), cancel_import_path(@import), method: :post, class: 'button')
+    .px-4.sm:px-6.lg:px-8
+      / .sm:flex.sm:items-center
+      /   .sm:flex-auto
+      /     h1.text-base.font-semibold.leading-6.text-gray-900
+      /       | Users
+      /     p.mt-2.text-sm.text-gray-700
+      /       | A list of all the users in your account including their name, title, email and role.
+      /   .mt-4.sm:ml-16.sm:mt-0.sm:flex-none
+      /     button.block.rounded-md.bg-indigo-600.px-3.py-2.text-center.text-sm.font-semibold.text-white.shadow-sm.hover:bg-indigo-500.focus-visible:outline.focus-visible:outline-2.focus-visible:outline-offset-2.focus-visible:outline-indigo-600[type="button"]
+      /       | Add user
+      .mt-8.flow-root
+        .-mx-4.-my-2.overflow-x-auto.sm:-mx-6.lg:-mx-8
+          .inline-block.min-w-full.py-2.align-middle.bg-white.dark:bg-gray-900.dark:border-gray-700
+            table.min-w-full.divide-y.divide-gray-20.border-2
+              thead.bg-gray-50
+                tr
+                  - @sheet_data.first.keys.each_with_index do |header, index|
+                    th.px-3.py-3.5.text-left.text-sm.font-semibold.border-2.text-gray-600[scope="col"]
+                      = f.check_box :selected_items, { multiple: true, checked: @check_header.include?(header), disabled: @check_header.exclude?(header) }, ActionController::Base.helpers.strip_tags(header)
+                tr
+                  - @sheet_data.first.keys.each_with_index do |header, index|
+                    th.px-3.py-3.5.text-left.text-sm.font-semibold.border-2.text-gray-600[scope="col"]
+                      / = f.check_box :active, checked: @check_header.include?(header)
+                      = ActionController::Base.helpers.strip_tags(header)
+              tbody.divide-y.divide-gray-200.bg-white
+                - @sheet_data.reject! { |h| h.keys == h.values }&.each do |h|
+                  tr
+                    - h.each do |key, value|
+                      td.whitespace-nowrap.px-3.py-4.text-sm.text-gray-500.border-2
+                        = value

data/config/locales/en.yml

@@ -2,7 +2,7 @@ en:
   helpers:
     submit:
       importo/import:
-        create: "Import"
+        create: "Upload"
   importo:
     sheet:
       results:
@@ -14,6 +14,8 @@ en:
         explanation: Purpose
         example: Example
     imports:
+      cancel:
+        success: "Import canceled for id %{id}, Redirecting to new"
       index:
         title: Import results
         card:
@@ -38,6 +40,15 @@ en:
           no_file: Import failed, please upload a file.
           error: Import failed, there were problems %{error}.
           success: "Import scheduled with id %{id}, you will get an email with the results."
+      preview:
+        no_file: Import failed, please upload valid file.
+        title: "%{kind} Import Preview"
+        cancel: Cancel
+      upload:
+        flash:
+          no_file: Import failed, please upload a file.
+          error: Import failed, there were problems.
+          success: "Import scheduled with id %{id}, you will get an email with the results."
     errors:
       parse_error: "We encountered a parse error: %{error}"
       structure_invalid: "The structure is invalid, these are the invalid headers: %{invalid_headers}"

data/config/routes.rb

@@ -4,10 +4,13 @@ Importo::Engine.routes.draw do
   resources :imports, except: %i[new] do
     member do
       post :undo
+      post :upload
+      post :cancel
     end
   end
-  get ":kind/new", to: "imports#new", as: :new_import
-  get ":kind/sample", to: "imports#sample", as: :sample_import
-  get ":kind/export", to: "imports#export", as: :export
-  root to: "imports#index"
+  get ':kind/new', to: 'imports#new', as: :new_import
+  get ':kind/sample', to: 'imports#sample', as: :sample_import
+  get ':kind/export', to: 'imports#export', as: :export
+  get ':kind/:id/preview', to: 'imports#preview', as: :preview
+  root to: 'imports#index'
 end

data/db/migrate/20251117132125_create_good_jobs.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class CreateGoodJobs < ActiveRecord::Migration[7.0]
+  def change
+    # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
+    # enable_extension 'pgcrypto'
+
+    create_table :good_jobs, id: :uuid do |t|
+      t.text :queue_name
+      t.integer :priority
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :performed_at
+      t.datetime :finished_at
+      t.text :error
+
+      t.timestamps
+
+      t.uuid :active_job_id
+      t.text :concurrency_key
+      t.text :cron_key
+      t.uuid :retried_good_job_id
+      t.datetime :cron_at
+    end
+
+    create_table :good_job_processes, id: :uuid do |t|
+      t.timestamps
+      t.jsonb :state
+    end
+
+    add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: :index_good_jobs_on_scheduled_at
+    add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
+    add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at
+    add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished
+    add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at
+    add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true
+    add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id
+    add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at
+  end
+end

data/db/migrate/20251117132126_create_good_job_settings.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateGoodJobSettings < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.table_exists?(:good_job_settings)
+      end
+    end
+
+    create_table :good_job_settings, id: :uuid do |t|
+      t.timestamps
+      t.text :key
+      t.jsonb :value
+      t.index :key, unique: true
+    end
+  end
+end

data/db/migrate/20251117132127_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateIndexGoodJobsJobsOnPriorityCreatedAtWhenUnfinished < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.index_name_exists?(:good_jobs, :index_good_jobs_jobs_on_priority_created_at_when_unfinished)
+      end
+    end
+
+    add_index :good_jobs, [:priority, :created_at], order: { priority: "DESC NULLS LAST", created_at: :asc },
+      where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished,
+      algorithm: :concurrently
+  end
+end

data/db/migrate/20251117132128_create_good_job_batches.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class CreateGoodJobBatches < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.table_exists?(:good_job_batches)
+      end
+    end
+
+    create_table :good_job_batches, id: :uuid do |t|
+      t.timestamps
+      t.text :description
+      t.jsonb :serialized_properties
+      t.text :on_finish
+      t.text :on_success
+      t.text :on_discard
+      t.text :callback_queue_name
+      t.integer :callback_priority
+      t.datetime :enqueued_at
+      t.datetime :discarded_at
+      t.datetime :finished_at
+    end
+
+    change_table :good_jobs do |t|
+      t.uuid :batch_id
+      t.uuid :batch_callback_id
+
+      t.index :batch_id, where: "batch_id IS NOT NULL"
+      t.index :batch_callback_id, where: "batch_callback_id IS NOT NULL"
+    end
+  end
+end

data/db/migrate/20251117132129_create_good_job_executions.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class CreateGoodJobExecutions < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.table_exists?(:good_job_executions)
+      end
+    end
+
+    create_table :good_job_executions, id: :uuid do |t|
+      t.timestamps
+
+      t.uuid :active_job_id, null: false
+      t.text :job_class
+      t.text :queue_name
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :finished_at
+      t.text :error
+
+      t.index [:active_job_id, :created_at], name: :index_good_job_executions_on_active_job_id_and_created_at
+    end
+
+    change_table :good_jobs do |t|
+      t.boolean :is_discrete
+      t.integer :executions_count
+      t.text :job_class
+    end
+  end
+end

data/db/migrate/20251117132130_create_good_jobs_error_event.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateGoodJobsErrorEvent < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_jobs, :error_event)
+      end
+    end
+
+    add_column :good_jobs, :error_event, :integer, limit: 2
+    add_column :good_job_executions, :error_event, :integer, limit: 2
+  end
+end

data/db/migrate/20251117132131_recreate_good_job_cron_indexes_with_conditional.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class RecreateGoodJobCronIndexesWithConditional < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_created_at_cond)
+          add_index :good_jobs, [:cron_key, :created_at], where: "(cron_key IS NOT NULL)",
+                    name: :index_good_jobs_on_cron_key_and_created_at_cond, algorithm: :concurrently
+        end
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_cron_at_cond)
+          add_index :good_jobs, [:cron_key, :cron_at], where: "(cron_key IS NOT NULL)", unique: true,
+                    name: :index_good_jobs_on_cron_key_and_cron_at_cond, algorithm: :concurrently
+        end
+
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_created_at)
+          remove_index :good_jobs, name: :index_good_jobs_on_cron_key_and_created_at
+        end
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_cron_at)
+          remove_index :good_jobs, name: :index_good_jobs_on_cron_key_and_cron_at
+        end
+      end
+
+      dir.down do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_created_at)
+          add_index :good_jobs, [:cron_key, :created_at],
+                    name: :index_good_jobs_on_cron_key_and_created_at, algorithm: :concurrently
+        end
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_cron_at)
+          add_index :good_jobs, [:cron_key, :cron_at], unique: true,
+                    name: :index_good_jobs_on_cron_key_and_cron_at, algorithm: :concurrently
+        end
+
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_created_at_cond)
+          remove_index :good_jobs, name: :index_good_jobs_on_cron_key_and_created_at_cond
+        end
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_cron_key_and_cron_at_cond)
+          remove_index :good_jobs, name: :index_good_jobs_on_cron_key_and_cron_at_cond
+        end
+      end
+    end
+  end
+end

data/db/migrate/20251117132132_create_good_job_labels.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateGoodJobLabels < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_jobs, :labels)
+      end
+    end
+
+    add_column :good_jobs, :labels, :text, array: true
+  end
+end

data/db/migrate/20251117132133_create_good_job_labels_index.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class CreateGoodJobLabelsIndex < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_labels)
+          add_index :good_jobs, :labels, using: :gin, where: "(labels IS NOT NULL)",
+            name: :index_good_jobs_on_labels, algorithm: :concurrently
+        end
+      end
+
+      dir.down do
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_labels)
+          remove_index :good_jobs, name: :index_good_jobs_on_labels
+        end
+      end
+    end
+  end
+end

data/db/migrate/20251117132134_remove_good_job_active_id_index.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class RemoveGoodJobActiveIdIndex < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_active_job_id)
+          remove_index :good_jobs, name: :index_good_jobs_on_active_job_id
+        end
+      end
+
+      dir.down do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_active_job_id)
+          add_index :good_jobs, :active_job_id, name: :index_good_jobs_on_active_job_id
+        end
+      end
+    end
+  end
+end

data/db/migrate/20251117132135_create_index_good_job_jobs_for_candidate_lookup.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateIndexGoodJobJobsForCandidateLookup < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.index_name_exists?(:good_jobs, :index_good_job_jobs_for_candidate_lookup)
+      end
+    end
+
+    add_index :good_jobs, [:priority, :created_at], order: { priority: "ASC NULLS LAST", created_at: :asc },
+      where: "finished_at IS NULL", name: :index_good_job_jobs_for_candidate_lookup,
+      algorithm: :concurrently
+  end
+end

data/db/migrate/20251117132136_create_good_job_execution_error_backtrace.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateGoodJobExecutionErrorBacktrace < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_job_executions, :error_backtrace)
+      end
+    end
+
+    add_column :good_job_executions, :error_backtrace, :text, array: true
+  end
+end

data/db/migrate/20251117132137_create_good_job_process_lock_ids.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateGoodJobProcessLockIds < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_jobs, :locked_by_id)
+      end
+    end
+
+    add_column :good_jobs, :locked_by_id, :uuid
+    add_column :good_jobs, :locked_at, :datetime
+    add_column :good_job_executions, :process_id, :uuid
+    add_column :good_job_processes, :lock_type, :integer, limit: 2
+  end
+end

data/db/migrate/20251117132138_create_good_job_process_lock_indexes.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class CreateGoodJobProcessLockIndexes < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked)
+          add_index :good_jobs, [:priority, :scheduled_at],
+                    order: { priority: "ASC NULLS LAST", scheduled_at: :asc },
+                    where: "finished_at IS NULL AND locked_by_id IS NULL",
+                    name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked,
+                    algorithm: :concurrently
+        end
+
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_locked_by_id)
+          add_index :good_jobs, :locked_by_id,
+                    where: "locked_by_id IS NOT NULL",
+                    name: :index_good_jobs_on_locked_by_id,
+                    algorithm: :concurrently
+        end
+
+        unless connection.index_name_exists?(:good_job_executions, :index_good_job_executions_on_process_id_and_created_at)
+          add_index :good_job_executions, [:process_id, :created_at],
+                    name: :index_good_job_executions_on_process_id_and_created_at,
+                    algorithm: :concurrently
+        end
+      end
+
+      dir.down do
+        remove_index(:good_jobs, name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked) if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked)
+        remove_index(:good_jobs, name: :index_good_jobs_on_locked_by_id) if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_locked_by_id)
+        remove_index(:good_job_executions, name: :index_good_job_executions_on_process_id_and_created_at) if connection.index_name_exists?(:good_job_executions, :index_good_job_executions_on_process_id_and_created_at)
+      end
+    end
+  end
+end

data/db/migrate/20251117132139_create_good_job_execution_duration.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateGoodJobExecutionDuration < ActiveRecord::Migration[7.0]
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_job_executions, :duration)
+      end
+    end
+
+    add_column :good_job_executions, :duration, :interval
+  end
+end

data/lib/importo/adapters/sidekiq_batch_adapter.rb

@@ -0,0 +1,79 @@
+if !defined?(Sidekiq::Batch)
+  module Sidekiq
+    class Batch
+      def self.new(*)
+        raise NotImplementedError, "Sidekiq::Batch is not available. Please install the sidekiq-pro or sidekiq-batch gem."
+      end
+    end
+  end
+end
+
+require_relative "../../../app/jobs/importo/import_job"
+
+module Importo
+  class SidekiqBatchAdapter
+    attr_reader :description
+    attr_accessor :properties
+    attr_writer :instance
+
+    delegate :description=, to: :@instance
+
+    def initialize
+      @instance = Sidekiq::Batch.new
+    end
+
+    def on_success(job)
+      @instance.on(:complete, job.constantize, properties)
+    end
+
+    def add
+      @instance.jobs do
+        yield
+      end
+    end
+
+    def finished?
+      status = Sidekiq::Batch::Status.new( @instance.bid)
+      status.complete?
+    end
+
+    class << self
+      def import_job_base_class
+        Object
+      end
+
+      def find(id)
+        instance = new
+        instance.instance = Sidekiq::Batch.new(id)
+        instance
+      end
+    end
+
+    module ImportJobIncludes
+      extend ActiveSupport::Concern
+
+      included do
+        queue_as Importo.config.queue_name
+        sidekiq_options retry: 5
+
+        sidekiq_retries_exhausted do |msg, _e|
+          attributes = msg["args"][0]
+          index = msg["args"][1]
+          import_id = msg["args"][2]
+
+          execute_row(attributes, index, import_id, true, msg["bid"])
+        end
+
+        sidekiq_retry_in do |_count, exception, _jobhash|
+          case exception
+          when Importo::RetryError
+            exception.delay
+          end
+        end
+      end
+    end
+  end
+end
+
+# Importo::ImportJob.send(:include, Sidekiq::Job)
+# Importo::ImportJob.send(:include, Importo::SidekiqBatchAdapter::ImportJobIncludes)

data/lib/importo/configuration.rb

@@ -3,9 +3,20 @@
 module Importo
   module Options
     module ClassMethods
-      def option(name, default: nil)
-        attr_accessor(name)
-        schema[name] = default
+      def option(name, default: nil, proc: false)
+        attr_writer(name)
+        schema[name] = {default: default, proc: proc}
+
+        if schema[name][:proc]
+          define_method(name) do |*params|
+            value = instance_variable_get(:"@#{name}")
+            instance_exec(*params, &value)
+          end
+        else
+          define_method(name) do
+            instance_variable_get(:"@#{name}")
+          end
+        end
       end
 
       def schema
@@ -14,8 +25,8 @@ module Importo
     end
 
     def set_defaults!
-      self.class.schema.each do |name, default|
-        instance_variable_set(:"@#{name}", default)
+      self.class.schema.each do |name, options|
+        instance_variable_set(:"@#{name}", options[:default])
       end
     end
 
@@ -34,6 +45,8 @@ module Importo
     option :base_service_context, default: "::ApplicationContext"
     option :current_import_owner, default: lambda {}
     option :queue_name, default: :import
+    # You can either use GoodJob::Batch or Importo::SidekiqBatchAdapter
+    option :batch_adapter, default: lambda { GoodJob::Batch }, proc: true
 
     option :admin_visible_imports, default: lambda { Importo::Import.where(importo_ownable: Importo.config.current_import_owner) }
     option(:admin_can_destroy,
@@ -41,6 +54,10 @@ module Importo
         false
       end)
 
+    # You can either use GoodJob::Batch or Importo::SidekiqBatchAdapter
+    option :batch_adapter, default: lambda { Importo::SidekiqBatchAdapter }, proc: true
+    option :import_job_base_class, default: "Object"
+
     # Extra links relevant for this import: { link_name: { icon: 'far fa-..', url: '...' } }
     option(:admin_extra_links,
       default: lambda do |import|

data/lib/importo/engine.rb

@@ -18,6 +18,9 @@ module Importo
       config.after_initialize do
         ActiveSupport.on_load(:active_record) do
           Importo::Import.include(ImportHelpers)
+
+          # For now put this here to ensure compatibility
+          require "importo/adapters/sidekiq_batch_adapter"
         end
       end
     end

data/lib/importo/test_helpers.rb

@@ -28,9 +28,12 @@ module Importo
 
       import.original.attach(io: sheet, filename: filename, content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", identify: false)
       import.save!
-
+      import.confirm
+      import.schedule
       ImportService.perform(import: import)
-      ImportJobCallback.new.on_complete("success", {import_id: import.id})
+      if Importo.config.batch_adapter == Importo::SidekiqBatchAdapter
+        ImportJobCallback.new.on_success(:success,{import_id: import.id})
+      end
       import
     end
   end

data/lib/importo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Importo
-  VERSION = "3.0.21"
+  VERSION = "3.0.23"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: importo
 version: !ruby/object:Gem::Version
-  version: 3.0.21
+  version: 3.0.23
 platform: ruby
 authors:
 - Andre Meij
 - Tom de Grunt
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-02 00:00:00.000000000 Z
+date: 2025-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: caxlsx
@@ -25,6 +25,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.1
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -81,20 +95,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: servitium
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: signum
   requirement: !ruby/object:Gem::Requirement
@@ -302,7 +302,6 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- app/adapters/importo/sidekiq_batch_adapter.rb
 - app/assets/config/importo_manifest.js
 - app/assets/javascripts/importo/application.js
 - app/assets/stylesheets/importo/application.css
@@ -337,6 +336,7 @@ files:
 - app/tables/importo/mensa_imports_table.rb
 - app/views/importo/imports/index.html.slim
 - app/views/importo/imports/new.html.slim
+- app/views/importo/imports/preview.html.slim
 - config/locales/en.yml
 - config/locales/nl.yml
 - config/routes.rb
@@ -345,6 +345,21 @@ files:
 - db/migrate/20190827093548_add_selected_fields_to_import.rb
 - db/migrate/20230510051447_remove_result_from_imports.rb
 - db/migrate/20230510083043_create_importo_results.rb
+- db/migrate/20251117132125_create_good_jobs.rb
+- db/migrate/20251117132126_create_good_job_settings.rb
+- db/migrate/20251117132127_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb
+- db/migrate/20251117132128_create_good_job_batches.rb
+- db/migrate/20251117132129_create_good_job_executions.rb
+- db/migrate/20251117132130_create_good_jobs_error_event.rb
+- db/migrate/20251117132131_recreate_good_job_cron_indexes_with_conditional.rb
+- db/migrate/20251117132132_create_good_job_labels.rb
+- db/migrate/20251117132133_create_good_job_labels_index.rb
+- db/migrate/20251117132134_remove_good_job_active_id_index.rb
+- db/migrate/20251117132135_create_index_good_job_jobs_for_candidate_lookup.rb
+- db/migrate/20251117132136_create_good_job_execution_error_backtrace.rb
+- db/migrate/20251117132137_create_good_job_process_lock_ids.rb
+- db/migrate/20251117132138_create_good_job_process_lock_indexes.rb
+- db/migrate/20251117132139_create_good_job_execution_duration.rb
 - lib/generators/importo/USAGE
 - lib/generators/importo/importer_generator.rb
 - lib/generators/importo/install_generator.rb
@@ -357,6 +372,7 @@ files:
 - lib/generators/satis/templates/config/initializers/satis.rb
 - lib/importo.rb
 - lib/importo/acts_as_import_owner.rb
+- lib/importo/adapters/sidekiq_batch_adapter.rb
 - lib/importo/configuration.rb
 - lib/importo/engine.rb
 - lib/importo/import_column.rb
@@ -367,7 +383,7 @@ homepage: https://github.com/entdec/importo
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -383,7 +399,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.10
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Rails engine allowing uploads and imports
 test_files: []

data/app/adapters/importo/sidekiq_batch_adapter.rb

@@ -1,35 +0,0 @@
-module Importo
-  class SidekiqBatchAdapter
-    attr_reader :description
-    attr_accessor :properties
-    attr_writer :instance
-
-    def initialize
-      @instance = Sidekiq::Batch.new
-    end
-
-    delegate :description=, :status, to: :@instance
-
-    def on_success(job)
-      @instance.on(:complete, job.constantize, properties)
-    end
-
-    def add
-      @instance.jobs do
-        yield
-      end
-    end
-
-    def finished?
-      @instance.status.complete?
-    end
-
-    class << self
-      def find(id)
-        instance = new
-        instance.instance = Sidekiq::Batch.new(id)
-        instance
-      end
-    end
-  end
-end