importo 3.0.21 → 3.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcca807f9c6f4ca09145b27128b1df2564b226bae7ff922618d399298cf8c231
-  data.tar.gz: b63912b0490807c5f99dc6928dd69b189cfcd0b92611ac6982d27c45a64b33e4
+  metadata.gz: d68cd0c105d04f8338c6eeecf55032d1f6a31cb6fa825787e0c4b43859a1fd6f
+  data.tar.gz: 6c64470c728960afac588aeeadd59f219062254bb07d14d1fd02751ec6271baf
 SHA512:
-  metadata.gz: 9b01d49b3e56b5823331572c42cccf441325aae495699bc7d52ecb1fb7aaf6468f685ffbfe3412ebc0f64c9d0c34b99f7340f21f420ce655d7804a0aad5beeaa
-  data.tar.gz: 553dadb41cbe13aae4857265964fd3cf449c641f0bc199938b22db259bc3ce897ea7a34b340f2bb4bf7380647ec5f6a216e4efc0a378d15714bb89f38669e0ce
+  metadata.gz: 716af97a98a787f7491a53df4099efba5d5eac4693ad375fbd36609c3dec86e930b7ac9dff38dc5d49afdc9d240ec581e7a7b0bb0ca7304b6a7b6d9f1eaddc4a
+  data.tar.gz: 04c57bf751390e916f996c5d3684277041f4f68d1e562a4aff0aee5595c0d1a07d86d93537ecac88eda996bc977bd4cd53374e1891fa4d646981f018ea30c0cc

data/app/importers/concerns/importable.rb

@@ -126,7 +126,7 @@ module Importable
   def import!
     raise ArgumentError, "Invalid data structure" unless structure_valid?
 
-    batch = Importo::SidekiqBatchAdapter.new
+    batch = Importo.config.batch_adapter.new
     batch.description = "#{import.original.filename} - #{import.kind}"
     batch.properties = {import_id: import.id}
     batch.on_success("Importo::ImportJobCallback")

data/app/importers/importo/import_job_callback.rb

@@ -7,13 +7,10 @@ module Importo
       import = Import.find(options["import_id"])
       if import.present?
         results_file = import.importer.results_file
-        if results_file.is_a?(StringIO)
-          import.result.attach(io: results_file, filename: import.importer.file_name("results"),
-                               content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-        else
-          import.result.attach(io: File.open(results_file), filename: import.importer.file_name("results"),
-                               content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-        end
+        results_file = results_file.is_a?(StringIO) ? results_file : File.open(results_file)
+
+        import.result.attach(io: results_file, filename: import.importer.file_name("results"),
+          content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
 
         ActiveRecord::Base.uncached do
           import.result_message = I18n.t("importo.importers.result_message",

data/app/jobs/importo/import_job.rb

@@ -1,24 +1,6 @@
 module Importo
-  class ImportJob
-    include Sidekiq::Job
-    sidekiq_options retry: 5
-    # queue_as :integration
-    queue_as Importo.config.queue_name
-
-    sidekiq_retries_exhausted do |msg, _e|
-      attributes = msg["args"][0]
-      index = msg["args"][1]
-      import_id = msg["args"][2]
-
-      execute_row(attributes, index, import_id, true, msg["bid"])
-    end
-
-    sidekiq_retry_in do |_count, exception, _jobhash|
-      case exception
-      when Importo::RetryError
-        exception.delay
-      end
-    end
+  class ImportJob < Importo.config.import_job_base_class.safe_constantize
+    # No options here, gets added from the adapter
 
     def perform(attributes, index, import_id)
       self.class.execute_row(attributes, index, import_id, false, bid)
@@ -28,12 +10,18 @@ module Importo
       attributes = JSON.load(attributes).deep_symbolize_keys if attributes.is_a?(String)
 
       import = Import.find(import_id)
-      record = import.importer.process_data_row(attributes, index, last_attempt: last_attempt)
-
-      batch = Importo::SidekiqBatchAdapter.find(bid)
-
-      if !import.completed? && import.can_complete? && batch.finished?
-        ImportJobCallback.new.on_complete(batch.status, {import_id: import.id})
+      import.importer.process_data_row(attributes, index, last_attempt: last_attempt)
+
+      # Between sidekiq and good job, there's a big difference:
+      # - Sidekiq calls on_complete callback when all jobs ran at least once.
+      # - GoodJob calls on_complete callback when all jobs are done (including retries).
+      # i.e. this logic is only needed for sidekiq
+      if Importo.config.batch_adapter == Importo::SidekiqBatchAdapter
+        batch = Importo::SidekiqBatchAdapter.find(bid)
+
+        if !import.completed? && import.can_complete? && batch.finished?
+          ImportJobCallback.new.on_complete("success", {import_id: import.id})
+        end
       end
     end
   end

data/lib/importo/adapters/sidekiq_batch_adapter.rb

@@ -0,0 +1,79 @@
+if !defined?(Sidekiq::Batch)
+  module Sidekiq
+    class Batch
+      def self.new(*)
+        raise NotImplementedError, "Sidekiq::Batch is not available. Please install the sidekiq-pro or sidekiq-batch gem."
+      end
+    end
+  end
+end
+
+require_relative "../../../app/jobs/importo/import_job"
+
+module Importo
+  class SidekiqBatchAdapter
+    attr_reader :description
+    attr_accessor :properties
+    attr_writer :instance
+
+    delegate :description=, to: :@instance
+
+    def initialize
+      @instance = Sidekiq::Batch.new
+    end
+
+    def on_success(job)
+      @instance.on(:complete, job.constantize, properties)
+    end
+
+    def add
+      @instance.jobs do
+        yield
+      end
+    end
+
+    def finished?
+      status = Sidekiq::Batch::Status.new( @instance.bid)
+      status.complete?
+    end
+
+    class << self
+      def import_job_base_class
+        Object
+      end
+
+      def find(id)
+        instance = new
+        instance.instance = Sidekiq::Batch.new(id)
+        instance
+      end
+    end
+
+    module ImportJobIncludes
+      extend ActiveSupport::Concern
+
+      included do
+        queue_as Importo.config.queue_name
+        sidekiq_options retry: 5
+
+        sidekiq_retries_exhausted do |msg, _e|
+          attributes = msg["args"][0]
+          index = msg["args"][1]
+          import_id = msg["args"][2]
+
+          execute_row(attributes, index, import_id, true, msg["bid"])
+        end
+
+        sidekiq_retry_in do |_count, exception, _jobhash|
+          case exception
+          when Importo::RetryError
+            exception.delay
+          end
+        end
+      end
+    end
+  end
+end
+
+Importo::ImportJob.send(:include, Sidekiq::Job)
+Importo::ImportJob.send(:include, Importo::SidekiqBatchAdapter::ImportJobIncludes)

data/lib/importo/configuration.rb

@@ -3,9 +3,19 @@
 module Importo
   module Options
     module ClassMethods
-      def option(name, default: nil)
-        attr_accessor(name)
-        schema[name] = default
+      def option(name, default: nil, proc: false)
+        attr_writer(name)
+        schema[name] = {default: default, proc: proc}
+        if schema[name][:proc]
+          define_method(name) do |*params|
+            value = instance_variable_get(:"@#{name}")
+            instance_exec(*params, &value)
+          end
+        else
+          define_method(name) do
+            instance_variable_get(:"@#{name}")
+          end
+        end
       end
 
       def schema
@@ -14,8 +24,8 @@ module Importo
     end
 
     def set_defaults!
-      self.class.schema.each do |name, default|
-        instance_variable_set(:"@#{name}", default)
+      self.class.schema.each do |name, options|
+        instance_variable_set(:"@#{name}", options[:default])
       end
     end
 
@@ -41,6 +51,10 @@ module Importo
         false
       end)
 
+    # You can either use GoodJob::Batch or Importo::SidekiqBatchAdapter
+    option :batch_adapter, default: lambda { Importo::SidekiqBatchAdapter }, proc: true
+    option :import_job_base_class, default: "Object"
+
     # Extra links relevant for this import: { link_name: { icon: 'far fa-..', url: '...' } }
     option(:admin_extra_links,
       default: lambda do |import|

data/lib/importo/engine.rb

@@ -18,6 +18,9 @@ module Importo
       config.after_initialize do
         ActiveSupport.on_load(:active_record) do
           Importo::Import.include(ImportHelpers)
+
+          # For now put this here to ensure compatibility
+          require "importo/adapters/sidekiq_batch_adapter"
         end
       end
     end

data/lib/importo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Importo
-  VERSION = "3.0.21"
+  VERSION = "3.0.22"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: importo
 version: !ruby/object:Gem::Version
-  version: 3.0.21
+  version: 3.0.22
 platform: ruby
 authors:
 - Andre Meij
 - Tom de Grunt
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-02 00:00:00.000000000 Z
+date: 2025-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: caxlsx
@@ -25,6 +25,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.1
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -302,7 +316,6 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- app/adapters/importo/sidekiq_batch_adapter.rb
 - app/assets/config/importo_manifest.js
 - app/assets/javascripts/importo/application.js
 - app/assets/stylesheets/importo/application.css
@@ -357,6 +370,7 @@ files:
 - lib/generators/satis/templates/config/initializers/satis.rb
 - lib/importo.rb
 - lib/importo/acts_as_import_owner.rb
+- lib/importo/adapters/sidekiq_batch_adapter.rb
 - lib/importo/configuration.rb
 - lib/importo/engine.rb
 - lib/importo/import_column.rb
@@ -367,7 +381,7 @@ homepage: https://github.com/entdec/importo
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -383,7 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.10
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Rails engine allowing uploads and imports
 test_files: []

data/app/adapters/importo/sidekiq_batch_adapter.rb

@@ -1,35 +0,0 @@
-module Importo
-  class SidekiqBatchAdapter
-    attr_reader :description
-    attr_accessor :properties
-    attr_writer :instance
-
-    def initialize
-      @instance = Sidekiq::Batch.new
-    end
-
-    delegate :description=, :status, to: :@instance
-
-    def on_success(job)
-      @instance.on(:complete, job.constantize, properties)
-    end
-
-    def add
-      @instance.jobs do
-        yield
-      end
-    end
-
-    def finished?
-      @instance.status.complete?
-    end
-
-    class << self
-      def find(id)
-        instance = new
-        instance.instance = Sidekiq::Batch.new(id)
-        instance
-      end
-    end
-  end
-end