sidekiq_cleaner 5.3.6

data/lib/sidekiq/util.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+require 'socket'
+require 'securerandom'
+require 'sidekiq/exception_handler'
+
+module Sidekiq
+  ##
+  # This module is part of Sidekiq core and not intended for extensions.
+  #
+  module Util
+    include ExceptionHandler
+
+    EXPIRY = 60 * 60 * 24
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, { context: last_words })
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current['sidekiq_label'] = name
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+
+    def redis(&block)
+      Sidekiq.redis(&block)
+    end
+
+    def hostname
+      ENV['DYNO'] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
+    end
+
+    def fire_event(event, options={})
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
+      arr = Sidekiq.options[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+          raise ex if reraise
+        end
+      end
+      arr.clear
+    end
+  end
+end

data/lib/sidekiq/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Sidekiq
+  VERSION = "5.3.6"
+end

data/lib/sidekiq/worker.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+require 'sidekiq/client'
+
+module Sidekiq
+
+  ##
+  # Include this module in your worker class and you can easily create
+  # asynchronous jobs:
+  #
+  #   class HardWorker
+  #     include Sidekiq::Worker
+  #
+  #     def perform(*args)
+  #       # do some work
+  #     end
+  #   end
+  #
+  # Then in your Rails app, you can do this:
+  #
+  #   HardWorker.perform_async(1, 2, 3)
+  #
+  # Note that perform_async is a class method, perform is an instance method.
+  module Worker
+    attr_accessor :jid
+
+    def self.included(base)
+      raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
+
+      base.extend(ClassMethods)
+      base.sidekiq_class_attribute :sidekiq_options_hash
+      base.sidekiq_class_attribute :sidekiq_retry_in_block
+      base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+
+    # This helper class encapsulates the set options for `set`, e.g.
+    #
+    #     SomeWorker.set(queue: 'foo').perform_async(....)
+    #
+    class Setter
+      def initialize(klass, opts)
+        @klass = klass
+        @opts = opts
+      end
+
+      def set(options)
+        @opts.merge!(options)
+        self
+      end
+
+      def perform_async(*args)
+        @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+
+        payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        payload.delete('at') if ts <= now
+        @klass.client_push(payload)
+      end
+      alias_method :perform_at, :perform_in
+    end
+
+    module ClassMethods
+      ACCESSOR_MUTEX = Mutex.new
+
+      def delay(*args)
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
+      end
+
+      def delay_for(*args)
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
+      end
+
+      def delay_until(*args)
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
+      end
+
+      def set(options)
+        Setter.new(self, options)
+      end
+
+      def perform_async(*args)
+        client_push('class' => self, 'args' => args)
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+
+        item = { 'class' => self, 'args' => args, 'at' => ts }
+
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        item.delete('at') if ts <= now
+
+        client_push(item)
+      end
+      alias_method :perform_at, :perform_in
+
+      ##
+      # Allows customization for this type of Worker.
+      # Legal options:
+      #
+      #   queue - use a named queue for this Worker, default 'default'
+      #   retry - enable the RetryJobs middleware for this Worker, *true* to use the default
+      #      or *Integer* count
+      #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
+      #      can be true, false or an integer number of lines to save, default *false*
+      #   pool - use the given Redis connection pool to push this type of job to a given shard.
+      #
+      # In practice, any option is allowed.  This is the main mechanism to configure the
+      # options for a specific job.
+      def sidekiq_options(opts={})
+        # stringify
+        self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
+      end
+
+      def sidekiq_retry_in(&block)
+        self.sidekiq_retry_in_block = block
+      end
+
+      def sidekiq_retries_exhausted(&block)
+        self.sidekiq_retries_exhausted_block = block
+      end
+
+      def get_sidekiq_options # :nodoc:
+        self.sidekiq_options_hash ||= Sidekiq.default_worker_options
+      end
+
+      def client_push(item) # :nodoc:
+        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
+        # stringify
+        item.keys.each do |key|
+          item[key.to_s] = item.delete(key)
+        end
+
+        Sidekiq::Client.new(pool).push(item)
+      end
+
+      def sidekiq_class_attribute(*attrs)
+        instance_reader = true
+        instance_writer = true
+
+        attrs.each do |name|
+          synchronized_getter = "__synchronized_#{name}"
+
+          singleton_class.instance_eval do
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          end
+
+          define_singleton_method(synchronized_getter) { nil }
+          singleton_class.class_eval do
+            private(synchronized_getter)
+          end
+
+          define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
+
+          ivar = "@#{name}"
+
+          singleton_class.instance_eval do
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          end
+          define_singleton_method("#{name}=") do |val|
+            singleton_class.class_eval do
+              ACCESSOR_MUTEX.synchronize do
+                undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
+                define_method(synchronized_getter) { val }
+              end
+            end
+
+            if singleton_class?
+              class_eval do
+                undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                define_method(name) do
+                  if instance_variable_defined? ivar
+                    instance_variable_get ivar
+                  else
+                    singleton_class.send name
+                  end
+                end
+              end
+            end
+            val
+          end
+
+          if instance_reader
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+            define_method(name) do
+              if instance_variable_defined?(ivar)
+                instance_variable_get ivar
+              else
+                self.class.public_send name
+              end
+            end
+          end
+
+          if instance_writer
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+            attr_writer name
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sidekiq.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require 'sidekiq/version'
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2')
+
+require 'sidekiq/logging'
+require 'sidekiq/client'
+require 'sidekiq/worker'
+require 'sidekiq/redis_connection'
+require 'sidekiq/delay'
+
+require 'json'
+
+module Sidekiq
+  NAME = 'Sidekiq'
+  LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
+
+  DEFAULTS = {
+    queues: [],
+    labels: [],
+    concurrency: 10,
+    require: '.',
+    environment: nil,
+    timeout: 8,
+    poll_interval_average: nil,
+    average_scheduled_poll_interval: 5,
+    error_handlers: [],
+    death_handlers: [],
+    lifecycle_events: {
+      startup: [],
+      quiet: [],
+      shutdown: [],
+      heartbeat: [],
+    },
+    dead_max_jobs: 10_000,
+    dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+    reloader: proc { |&block| block.call },
+  }
+
+  DEFAULT_WORKER_OPTIONS = {
+    'retry' => true,
+    'queue' => 'default'
+  }
+
+  FAKE_INFO = {
+    "redis_version" => "9.9.9",
+    "uptime_in_days" => "9999",
+    "connected_clients" => "9999",
+    "used_memory_human" => "9P",
+    "used_memory_peak_human" => "9P"
+  }
+
+  def self.❨╯°□°❩╯︵┻━┻
+    puts "Calm down, yo."
+  end
+
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+
+  def self.options=(opts)
+    @options = opts
+  end
+
+  ##
+  # Configuration for Sidekiq server, use like:
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #     end
+  #   end
+  def self.configure_server
+    yield self if server?
+  end
+
+  ##
+  # Configuration for Sidekiq client, use like:
+  #
+  #   Sidekiq.configure_client do |config|
+  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/0' }
+  #   end
+  def self.configure_client
+    yield self unless server?
+  end
+
+  def self.server?
+    defined?(Sidekiq::CLI)
+  end
+
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a replica, need
+        # to disconnect and reopen the socket to get back to the primary.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
+  end
+
+  def self.redis_info
+    redis do |conn|
+      begin
+        # admin commands can't go through redis-namespace starting
+        # in redis-namespace 2.0
+        if conn.respond_to?(:namespace)
+          conn.redis.info
+        else
+          conn.info
+        end
+      rescue Redis::CommandError => ex
+        #2850 return fake version when INFO command has (probably) been renamed
+        raise unless ex.message =~ /unknown command/
+        FAKE_INFO
+      end
+    end
+  end
+
+  def self.redis_pool
+    @redis ||= Sidekiq::RedisConnection.create
+  end
+
+  def self.redis=(hash)
+    @redis = if hash.is_a?(ConnectionPool)
+      hash
+    else
+      Sidekiq::RedisConnection.create(hash)
+    end
+  end
+
+  def self.client_middleware
+    @client_chain ||= Middleware::Chain.new
+    yield @client_chain if block_given?
+    @client_chain
+  end
+
+  def self.server_middleware
+    @server_chain ||= default_server_middleware
+    yield @server_chain if block_given?
+    @server_chain
+  end
+
+  def self.default_server_middleware
+    Middleware::Chain.new
+  end
+
+  def self.default_worker_options=(hash)
+    # stringify
+    @default_worker_options = default_worker_options.merge(Hash[hash.map{|k, v| [k.to_s, v]}])
+  end
+  def self.default_worker_options
+    defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
+  end
+
+  def self.default_retries_exhausted=(prok)
+    logger.info { "default_retries_exhausted is deprecated, please use `config.death_handlers << -> {|job, ex| }`" }
+    return nil unless prok
+    death_handlers << prok
+  end
+
+  ##
+  # Death handlers are called when all retries for a job have been exhausted and
+  # the job dies.  It's the notification to your application
+  # that this job will not succeed without manual intervention.
+  #
+  # Sidekiq.configure_server do |config|
+  #   config.death_handlers << ->(job, ex) do
+  #   end
+  # end
+  def self.death_handlers
+    options[:death_handlers]
+  end
+
+  def self.load_json(string)
+    JSON.parse(string)
+  end
+  def self.dump_json(object)
+    JSON.generate(object)
+  end
+
+  def self.logger
+    Sidekiq::Logging.logger
+  end
+  def self.logger=(log)
+    Sidekiq::Logging.logger = log
+  end
+
+  # How frequently Redis should be checked by a random Sidekiq process for
+  # scheduled and retriable jobs. Each individual process will take turns by
+  # waiting some multiple of this value.
+  #
+  # See sidekiq/scheduled.rb for an in-depth explanation of this value
+  def self.average_scheduled_poll_interval=(interval)
+    self.options[:average_scheduled_poll_interval] = interval
+  end
+
+  # Register a proc to handle any error which occurs within the Sidekiq process.
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+  #   end
+  #
+  # The default error handler logs errors to Sidekiq.logger.
+  def self.error_handlers
+    self.options[:error_handlers]
+  end
+
+  # Register a block to run at a point in the Sidekiq lifecycle.
+  # :startup, :quiet or :shutdown are valid events.
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.on(:shutdown) do
+  #       puts "Goodbye cruel world!"
+  #     end
+  #   end
+  def self.on(event, &block)
+    raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+    raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+    options[:lifecycle_events][event] << block
+  end
+
+  # We are shutting down Sidekiq but what about workers that
+  # are working on some long job?  This error is
+  # raised in workers that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  class Shutdown < Interrupt; end
+end
+
+require 'sidekiq/rails' if defined?(::Rails::Engine)

data/sidekiq_cleaner.gemspec

@@ -0,0 +1,21 @@
+require_relative 'lib/sidekiq/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Dan Belling"]
+  gem.email         = ["danhbelling@gmail.com"]
+  gem.summary       = "A sidekiq cleaner library."
+  gem.description   = "A cleaner library for the sidekiq job processor for use by the RL engineering team."
+  gem.homepage      = "https://github.com/reachlocal/sidekiq-cleaner"
+  gem.license       = "LGPL-3.0"
+
+  gem.executables   = ['sidekiq', 'sidekiqctl']
+  gem.files         = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.name          = "sidekiq_cleaner"
+  gem.version       = Sidekiq::VERSION
+  gem.required_ruby_version = ">= 2.2.2"
+
+  gem.add_dependency 'redis', '>= 3.3.5', '< 5'
+  gem.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
+  gem.add_dependency 'rack', '>= 1.5.0'
+  gem.add_dependency 'rack-protection', '>= 1.5.0'
+end