sidekiq-reliable-requeue 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 790e70ae377335b07a2dcbceb93d017dc869d4af
+  data.tar.gz: 908147212792aea48eeb93b3c49f8700a8dc0bfc
+SHA512:
+  metadata.gz: e34ffe5ba3b3a325aca30aa0dbaa22356b9252437e482ae364ce027e4acc4d3f12375dc122f1b35346ad626948f2e4ccc186576957b89faf69814936fb9cf282
+  data.tar.gz: bee0f73145968509c129c133cb17272110ea8646dc47a27bc5b17d9af20f7b5175f1d19784ff0eb5f138f69170591d5a6fd2534ad64cb84c54ca40d6889f03f1

data/lib/sidekiq-reliable-requeue/middleware.rb

@@ -0,0 +1,22 @@
+module SidekiqReliableRequeue
+  class Middleware
+
+    def call(worker, msg, queue)
+      if msg['reliable_requeue']
+        set_stale_info(msg)
+      end
+      yield
+    ensure
+      clean_stale_info
+    end
+
+    def set_stale_info(msg)
+      @redis_key_for_check = "#{msg['class']}:#{msg['jid']}"
+      RedisConnection.hset(SidekiqReliableStaleJobsKey, @redis_key_for_check, msg.to_json)
+    end
+
+    def clean_stale_info
+      RedisConnection.hdel(SidekiqReliableStaleJobsKey, @redis_key_for_check)
+    end
+  end
+end

data/lib/sidekiq-reliable-requeue/redis_connection.rb

@@ -0,0 +1,11 @@
+module SidekiqReliableRequeue
+  class RedisConnection
+    class << self
+      def method_missing(redis_method, *args)
+        Sidekiq.redis do |conn|
+          conn.send(redis_method, *args)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-reliable-requeue/worker.rb

@@ -0,0 +1,40 @@
+module SidekiqReliableRequeue
+  class Worker
+    include Sidekiq::Worker
+
+    def perform(key)
+      data = RedisConnection.hget(SidekiqReliableStaleJobsKey, key)
+      Sidekiq.logger.debug("Checking for data with key: #{key}")
+
+      unless data
+        Sidekiq.logger.debug("Job with key #{key} has been cleaned")
+        return
+      end
+
+      sidekiq_msg = JSON.parse(RedisConnection.hget(SidekiqReliableStaleJobsKey, key))
+      worker      = sidekiq_msg['class'].constantize
+
+      Sidekiq.logger.info("Requeuing #{sidekiq_msg}")
+      worker.client_push(sidekiq_msg.except('jid'))
+      RedisConnection.hdel(SidekiqReliableStaleJobsKey, key)
+    end
+
+    def self.requeue_stale_jobs
+      keys = RedisConnection.hkeys(SidekiqReliableStaleJobsKey)
+
+      if keys.empty?
+        Sidekiq.logger.info('No stale jobs to add')
+      end
+
+      keys.each do |key|
+        data = RedisConnection.hget(SidekiqReliableStaleJobsKey, key)
+        next unless data
+
+        sidekiq_msg = JSON.parse(RedisConnection.hget(SidekiqReliableStaleJobsKey, key))
+
+        Sidekiq.logger.info("Preparing job #{key} to be requeued in #{sidekiq_msg['requeue_timeout']}s")
+        self.perform_in(sidekiq_msg['requeue_timeout'], key)
+      end
+    end
+  end
+end

data/lib/sidekiq-reliable-requeue.rb

@@ -0,0 +1,30 @@
+require 'sidekiq'
+require 'sidekiq-reliable-requeue/redis_connection'
+require 'sidekiq-reliable-requeue/middleware'
+require 'sidekiq-reliable-requeue/worker'
+
+module SidekiqReliableRequeue
+  Sidekiq.default_worker_options = {
+      requeue_timeout:  20,
+      reliable_requeue: false
+  }
+
+  SidekiqReliableStaleJobsKey = 'sidekiq_tasks_being_performed'
+
+  def self.initialize(requeue_timeout: 20, reliable_requeue: false)
+    Sidekiq.default_worker_options = {
+        requeue_timeout:  requeue_timeout,
+        reliable_requeue: reliable_requeue
+    }
+
+    Sidekiq.configure_server do |config|
+      config.on(:startup) do
+        Worker.requeue_stale_jobs
+      end
+
+      config.server_middleware do |chain|
+        chain.add Middleware
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-reliable-requeue
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+platform: ruby
+authors:
+- Vincent Algayres
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-06-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sidekiq-reliable-requeue.rb
+- lib/sidekiq-reliable-requeue/middleware.rb
+- lib/sidekiq-reliable-requeue/redis_connection.rb
+- lib/sidekiq-reliable-requeue/worker.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Sidekiq middleware to handle requeuing of jobs if the worker gets badly terminated
+  or crashes
+test_files: []