background_job 0.0.1.rc1 → 0.0.1.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7073c14e1fd364a96f5f4ddc812c5d67921905307af97f85bc3d8a0173304093
-  data.tar.gz: 95f18748a479ffa52bf39ac94830080564a95223193c8655c59a36083b6eebc7
+  metadata.gz: 85008658f92ee38b57dc2dd7692c7b905ee97096907f97aef166f14a605e34b7
+  data.tar.gz: 19c2cba85992b54d9d1b7d5c080e884cb859a3c78b561d303145905850cab63a
 SHA512:
-  metadata.gz: aa43df351a913191022dc34d5773ec51ab80bfbdf575cad58c1d33845150fa75a443f3e394e00b46da3283f7e0e69a7ed86b4637ef6e2e960ff797021172758f
-  data.tar.gz: c78f2257f401f3d9585f210007cb314da943aaa971dd396cccee703e142d1c949e1c037a8fcbb20bb3e8a623b7db0e36b6feb4fd25f1b42338669913465dfe4b
+  metadata.gz: 02d6a213cc6a3d1f2e35806c4b9078eb5a8cbbf4e1cdbfbbad1f472ab8dd1ba17b4a37892f61ce0fccf63839b1c5e664b302ecd0acbbd30cedb6528a8b2a27cc
+  data.tar.gz: 0fd818b7983a7762a2b6f810b76cf07f4e01e0e5cfbeb9d63c3c130f2feb45c04c2384d00f944492c1cd29d7d526b3cc3e72ffba5262d51546c938455e0ce2c2

data/Gemfile.lock

@@ -1,8 +1,7 @@
 PATH
   remote: .
   specs:
-    background_job (0.0.1.rc1)
-      connection_pool
+    background_job (0.0.1.rc3)
       multi_json
       redis
 
@@ -22,7 +21,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     rake (12.3.3)
-    redis (5.2.0)
+    redis (5.3.0)
       redis-client (>= 0.22.0)
     redis-client (0.22.2)
       connection_pool

data/background_job.gemspec

@@ -34,6 +34,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'redis', '>= 0.0.0'
-  spec.add_dependency 'connection_pool', '>= 0.0.0'
   spec.add_dependency 'multi_json', '>= 0.0.0'
 end

data/docker-compose.yml

@@ -1,6 +1,5 @@
 services:
   redis:
     image: redis
-    command: redis-server
     ports:
       - 6379:6379

data/lib/background_job/configuration/sidekiq.rb

@@ -19,5 +19,17 @@ module BackgroundJob
       @redis_pool = nil
       @redis = value
     end
+
+    def update_queues!
+      redis_pool.with do |conn|
+        indexing_queues = jobs.values.map { |job| job[:queue] }.uniq
+        key = [namespace, 'queues'].compact.join(':')
+        conn.pipelined do |pipeline|
+          indexing_queues.each do |queue|
+            pipeline.sadd?(key, queue)
+          end
+        end
+      end
+    end
   end
 end

data/lib/background_job/configuration.rb

@@ -16,6 +16,14 @@ module BackgroundJob
   class Configuration
     attr_reader :redis
 
+    def reset!
+      @redis = nil
+      @redis_pool = nil
+      @services = nil
+      @faktory = nil
+      @sidekiq = nil
+    end
+
     def redis=(value)
       @redis_pool = nil
       @redis = value

data/lib/background_job/jobs/sidekiq.rb

@@ -26,18 +26,21 @@ module BackgroundJob
       #   * Otherwise enqueue for immediate execution
       #
       # @return [Hash] Payload that was sent to redis
-      def push
+      def push(**kwargs)
         normalize_before_push!
 
+        kwargs[:retry] ||= 3 # retry is a reserved keyword
         BackgroundJob.config.sidekiq.middleware.invoke(self, :sidekiq) do
-          # Optimization to enqueue something now that is scheduled to go out now or in the past
-          if (timestamp = payload.delete('at')) && (timestamp > Time.now.to_f)
-            redis_pool.with do |redis|
-              redis.zadd(scheduled_queue_name, timestamp.to_f.to_s, to_json(payload))
-            end
-          else
-            redis_pool.with do |redis|
-              redis.lpush(immediate_queue_name, to_json(payload))
+          retriable_connection(max_attempts: kwargs[:retry]) do |conn|
+            # Optimization to enqueue something now that is scheduled to go out now or in the past
+            if (timestamp = payload.delete('at')) && (timestamp > Time.now.to_f)
+              conn.zadd(scheduled_queue_name, timestamp.to_f.to_s, to_json(payload))
+            else
+              payload['enqueued_at'] = Time.now.to_f
+              conn.pipelined do |pipeline|
+                pipeline.sadd(queues_set_name, [queue_name])
+                pipeline.lpush(immediate_queue_name, to_json(payload))
+              end
             end
           end
           payload
@@ -48,7 +51,6 @@ module BackgroundJob
 
       def normalize_before_push!
         with_job_jid # Generate a unique job id
-        payload['enqueued_at'] = Time.now.to_f
       end
 
       def redis_pool
@@ -64,12 +66,34 @@ module BackgroundJob
       end
 
       def immediate_queue_name
-        [namespace, 'queue', payload.fetch('queue')].compact.join(':')
+        [namespace, 'queue', queue_name].compact.join(':')
+      end
+
+      def queues_set_name
+        [namespace, 'queues'].compact.join(':')
+      end
+
+      def queue_name
+        payload.fetch('queue')
       end
 
       def to_json(value)
         MultiJson.dump(value, mode: :compat)
       end
+
+      def retriable_connection(max_attempts: 3)
+        tries = 0
+        redis_pool.with do |conn|
+          yield conn
+        rescue Redis::CommandError => ex
+          if ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/ && tries < max_attempts
+            tries += 1
+            conn.close
+            retry
+          end
+          raise ex
+        end
+      end
     end
   end
 end

data/lib/background_job/mixin/faktory.rb

@@ -28,15 +28,16 @@ module BackgroundJob
       end
 
       class Builder < Module
-        def initialize(**options)
+        def initialize(native: false, **options)
+          @native = native
           @runtime_mod = Module.new do
             define_method(:background_job_user_options) { options }
           end
         end
 
         def extended(base)
-          base.include(::Faktory::Job) if defined?(::Faktory)
-          base.extend BackgroundJob::Mixin::SharedInterface
+          base.include(::Faktory::Job) if defined?(::Faktory::Job)
+          base.extend(BackgroundJob::Mixin::SharedInterface) unless @native
           base.extend ClassMethods
           base.extend @runtime_mod
         end

data/lib/background_job/mixin/sidekiq.rb

@@ -33,15 +33,20 @@ module BackgroundJob
       end
 
       class Builder < Module
-        def initialize(**options)
+        def initialize(native: false, **options)
+          @native = native
           @runtime_mod = Module.new do
             define_method(:background_job_user_options) { options }
           end
         end
 
         def extended(base)
-          base.include(::Sidekiq::Worker) if defined?(::Sidekiq)
-          base.extend BackgroundJob::Mixin::SharedInterface
+          if defined?(::Sidekiq::Job)
+            base.include(::Sidekiq::Job)
+          elsif defined?(::Sidekiq::Worker)
+            base.include(::Sidekiq::Worker)
+          end
+          base.extend(BackgroundJob::Mixin::SharedInterface) unless @native
           base.extend ClassMethods
           base.extend @runtime_mod
         end

data/lib/background_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BackgroundJob
-  VERSION = '0.0.1.rc1'
+  VERSION = '0.0.1.rc3'
 end

data/lib/background_job.rb

@@ -42,13 +42,21 @@ module BackgroundJob
     faktory: 'Faktory',
   }.freeze
 
-  SERVICES.each do |id, name|
-    define_singleton_method(id) do |job_name, **options|
-      Jobs.const_get(name).new(job_name, **options)
+  def self.job(service, job_name, **options)
+    service = service.to_sym
+    unless SERVICES.key?(service)
+      raise Error, "Service `#{service}' is not supported. Supported services are: #{SERVICES.keys.join(', ')}"
+    end
+    Jobs.const_get(SERVICES[service]).new(job_name, **options)
+  end
+
+  SERVICES.each do |service_name, _const_name|
+    define_singleton_method(service_name) do |job_name, **options|
+      job(service_name, job_name, **options)
     end
   end
 
-  def self.mixin(service, **options)
+  def self.mixin(service, native: false, **options)
     service = service.to_sym
     unless SERVICES.key?(service)
       raise Error, "Service `#{service}' is not supported. Supported services are: #{SERVICES.keys.join(', ')}"
@@ -58,7 +66,7 @@ module BackgroundJob
 
     module_name = service.to_s.split(/_/i).collect!{ |w| w.capitalize }.join
     mod = Mixin.const_get(module_name)
-    mod::Builder.new(**options)
+    mod::Builder.new(native: native, **options)
   end
 
   def self.jid

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: background_job
 version: !ruby/object:Gem::Version
-  version: 0.0.1.rc1
+  version: 0.0.1.rc3
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-14 00:00:00.000000000 Z
+date: 2024-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.0
-- !ruby/object:Gem::Dependency
-  name: connection_pool
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.0
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement