sidekiq-pauzer 1.0.0.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b38aee70ca050a4c5c0f8c41d81bae5b755bb4a73fd1b5ddb7b365fb61ba3f40
+  data.tar.gz: e233b4b60fc065a12c29f809ad196f763f05a54d7bec640c338dce6ec2c62984
+SHA512:
+  metadata.gz: 8e106b6c3e162eafddf56da121c6d3af3319928a145c1991f9c09ceeb54751eb68a15f716731e5bf2feb15fd785a7062ce19db5523a79b8a3be2e1889af39c94
+  data.tar.gz: 1db224ec249b5729e903a0ad8fc9f58235e8a39c5bc39f00a2e11a426c3a0ec20fb216faa5bf144ef20bbfa93f4dc94d8599546ab46052724142a89420e29a07

data/README.adoc

@@ -0,0 +1,67 @@
+= Sidekiq::Pauzer
+
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+    $ bundle add sidekiq-pauzer
+
+Or install it yourself as:
+
+    $ gem install sidekiq-pauzer
+
+
+== Usage
+
+[source, ruby]
+----
+require "sidekiq"
+require "sidekiq/pauzer"
+
+Sidekiq::Pauzer.configure do |config|
+  # Set redis key prefix.
+  # Default: nil
+  config.key_prefix = "my-app:"
+
+  # Set paused queues local cache refresh rate in seconds.
+  # Default: 10
+  config.refresh_rate = 15
+end
+
+Sidekiq.configure_server do |config|
+  config[:fetch_class] = Sidekiq::Pauzer::BasicFetch
+end
+----
+
+
+=== Adding Pause/Resume Button to the Queues Tab
+
+If you're not overriding `Sidekiq::Web.views` path, then you can override
+default queues tab with:
+
+[source, ruby]
+----
+require "sidekiq/web"
+require "sidekiq/pauzer/web"
+----
+
+NOTE: If you are using custom Sidekiq views path, then you will need to call
+  (after requiring `sidekiq/pauzer/web`): `Sidekiq::Pauzer.unpatch_views!`.
+
+
+== Development
+
+  scripts/update-gemfiles
+  scripts/run-rspec
+  bundle exec rubocop
+
+
+== Contributing
+
+* Fork sidekiq-pauzer
+* Make your changes
+* Ensure all tests pass (`bundle exec rake`)
+* Send a merge request
+* If we like them we'll merge them
+* If we've accepted a patch, feel free to ask for commit access!

data/lib/sidekiq/pauzer/adapters/redis.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Pauzer
+    module Adapters
+      # redis-rb adapter
+      module Redis
+        class << self
+          def adapts?(redis)
+            return true if defined?(::Redis) && redis.is_a?(::Redis)
+            return true if defined?(::Redis::Namespace) && redis.is_a?(::Redis::Namespace)
+
+            false
+          end
+
+          def pause!(redis, key, queue)
+            redis.sadd(key, queue)
+          end
+
+          def unpause!(redis, key, queue)
+            redis.srem(key, queue)
+          end
+
+          def paused_queues(redis, key)
+            # Cursor is not atomic, so there may be duplicates because of
+            # concurrent update operations
+            # See: https://redis.io/commands/scan/#scan-guarantees
+            redis.sscan_each(key).to_a.uniq.each(&:freeze)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/adapters/redis_client.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Pauzer
+    module Adapters
+      # redis-client adapter
+      module RedisClient
+        SIDEKIQ_SEVEN   = Gem::Version.new("7.0.0").freeze
+        SIDEKIQ_VERSION = Gem::Version.new(Sidekiq::VERSION).freeze
+
+        class << self
+          def adapts?(redis)
+            return true if SIDEKIQ_SEVEN <= SIDEKIQ_VERSION
+            return true if defined?(::RedisClient) && redis.is_a?(::RedisClient)
+            return true if defined?(::RedisClient::Decorator::Client) && redis.is_a?(::RedisClient::Decorator::Client)
+
+            false
+          end
+
+          def pause!(redis, key, queue)
+            redis.call("SADD", key, queue)
+          end
+
+          def unpause!(redis, key, queue)
+            redis.call("SREM", key, queue)
+          end
+
+          def paused_queues(redis, key)
+            # Cursor is not atomic, so there may be duplicates because of
+            # concurrent update operations
+            # See: https://redis.io/commands/scan/#scan-guarantees
+            redis.sscan(key).to_a.uniq.each(&:freeze)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/adapters.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "./adapters/redis"
+require_relative "./adapters/redis_client"
+
+module Sidekiq
+  module Pauzer
+    # @api internal
+    module Adapters
+      def self.[](redis)
+        return Adapters::RedisClient if Adapters::RedisClient.adapts?(redis)
+        return Adapters::Redis       if Adapters::Redis.adapts?(redis)
+
+        raise TypeError, "Unsupported redis client: #{redis.class}"
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/basic_fetch.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq/fetch"
+
+module Sidekiq
+  module Pauzer
+    # Default Sidekiq's BasicFetch infused with Pauzer
+    class BasicFetch < Sidekiq::BasicFetch
+      private
+
+      if Gem::Version.new("7.0.0") <= Gem::Version.new(Sidekiq::VERSION)
+        def queues_cmd
+          if @strictly_ordered_queues
+            @queues - Pauzer.paused_queues
+          else
+            permute = (@queues - Pauzer.paused_queues)
+            permute.shuffle!
+            permute.uniq!
+            permute
+          end
+        end
+      else
+        def queues_cmd
+          if @strictly_ordered_queues
+            *queues, timeout = @queues
+
+            (queues - Pauzer.paused_queues) << timeout
+          else
+            permute = (@queues - Pauzer.paused_queues)
+            permute.shuffle!
+            permute.uniq!
+            permute << { timeout: Sidekiq::BasicFetch::TIMEOUT }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/config.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Pauzer
+    class Config
+      REDIS_KEY = "sidekiq-pauzer"
+      private_constant :REDIS_KEY
+
+      # Default refresh rate
+      REFRESH_RATE = 10
+
+      # @return [String?]
+      attr_reader :key_prefix
+
+      # @return [Integer, Float]
+      attr_reader :refresh_rate
+
+      # Fully qualified Redis key
+      #
+      # @example Without key prefix (default)
+      #   config.redis_key # => "sidekiq-pauzer"
+      #
+      # @example With key prefix
+      #   config.key_prefix = "foobar:"
+      #   config.redis_key # => "foobar:sidekiq-pauzer"
+      #
+      # @return [String]
+      attr_reader :redis_key
+
+      def initialize
+        @key_prefix   = nil
+        @redis_key    = REDIS_KEY
+        @refresh_rate = REFRESH_RATE
+      end
+
+      # Set redis key prefix.
+      #
+      # @see redis_key
+      # @param value [String?] String that should be prepended to redis key
+      # @return [void]
+      def key_prefix=(value)
+        raise ArgumentError, "expected String, or nil; got #{value.class}" unless value.is_a?(String) || value.nil?
+
+        @redis_key  = [value, REDIS_KEY].compact.join.freeze
+        @key_prefix = value&.then(&:-@) # Don't freeze original String value if it was unfrozen
+      end
+
+      # Set paused queues local cache refresh rate in seconds.
+      #
+      # @param value [Float, Integer] refresh interval in seconds
+      # @return [void]
+      def refresh_rate=(value)
+        unless value.is_a?(Integer) || value.is_a?(Float)
+          raise ArgumentError, "expected Integer, or Float; got #{value.class}"
+        end
+
+        raise ArgumentError, "expected positive value; got #{value.inspect}" unless value.positive?
+
+        @refresh_rate = value
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/queues.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+require_relative "./adapters"
+
+module Sidekiq
+  module Pauzer
+    # @api internal
+    class Queues
+      include Enumerable
+
+      class Refresher < Concurrent::TimerTask; end
+
+      QUEUE_PREFIX = "queue:"
+
+      # @param config [Config]
+      def initialize(config)
+        @mutex     = Mutex.new
+        @queues    = []
+        @redis_key = config.redis_key
+        @refresher = initialize_refresher(config.refresh_rate)
+      end
+
+      def each(&block)
+        return to_enum __method__ unless block
+
+        @mutex.synchronize { @queues.dup }.each(&block)
+
+        self
+      end
+
+      def pause!(queue)
+        queue = normalize_queue_name(queue)
+
+        Sidekiq.redis { |conn| Adapters[conn].pause!(conn, @redis_key, queue) }
+
+        refresh
+      end
+
+      def unpause!(queue)
+        queue = normalize_queue_name(queue)
+
+        Sidekiq.redis { |conn| Adapters[conn].unpause!(conn, @redis_key, queue) }
+
+        refresh
+      end
+
+      def paused?(queue)
+        include?(normalize_queue_name(queue))
+      end
+
+      def start_refresher
+        @refresher.execute
+        nil
+      end
+
+      def stop_refresher
+        @refresher.shutdown
+        nil
+      end
+
+      def refresher_running?
+        @refresher.running?
+      end
+
+      private
+
+      def initialize_refresher(refresh_rate)
+        Refresher.new(execution_interval: refresh_rate, run_now: true) do
+          refresh
+        end
+      end
+
+      def refresh
+        @mutex.synchronize do
+          paused_queues = Sidekiq.redis do |conn|
+            Adapters[conn].paused_queues(conn, @redis_key)
+          end
+
+          @queues.replace(paused_queues)
+        end
+
+        self
+      end
+
+      def normalize_queue_name(queue)
+        queue.dup.delete_prefix(QUEUE_PREFIX)
+      end
+    end
+  end
+end

data/lib/sidekiq/pauzer/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Pauzer
+    VERSION = "1.0.0.alpha"
+  end
+end

data/lib/sidekiq/pauzer/web.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "erb"
+require "sidekiq"
+require "sidekiq/web"
+
+require_relative "../pauzer"
+
+module Sidekiq
+  module Pauzer
+    def self.unpatch_views!
+      WebAction.remove_method(:_erb_queues)
+    end
+  end
+
+  class WebApplication
+    @routes[Sidekiq::WebRouter::POST].delete_if do |web_route|
+      web_route.pattern == "/queues/:name"
+    end
+
+    post "/queues/:name" do
+      queue = Sidekiq::Queue.new(route_params[:name])
+
+      if params["pause"]
+        queue.pause!
+      elsif params["unpause"]
+        queue.unpause!
+      else
+        queue.clear
+      end
+
+      redirect "#{root_path}queues"
+    end
+  end
+
+  class WebAction
+    PAUZER_QUEUES_TEMPLATE =
+      ERB.new(File.read(File.expand_path("../../../web/views/queues.erb", __dir__))).src
+
+    class_eval <<-RUBY, __FILE__, __LINE__ + 1 # rubocop:disable Style/DocumentDynamicEvalDefinition
+      def _erb_queues
+        #{PAUZER_QUEUES_TEMPLATE}
+      end
+    RUBY
+  end
+end

data/lib/sidekiq/pauzer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "sidekiq"
+require "sidekiq/api"
+
+require_relative "./pauzer/basic_fetch"
+require_relative "./pauzer/config"
+require_relative "./pauzer/queues"
+require_relative "./pauzer/version"
+
+begin
+  require "sidekiq-ent/version"
+  raise "sidekiq-pauzer is incompatible with Sidekiq Enterprise"
+rescue LoadError
+  # All good - no compatibility issues
+end
+
+begin
+  require "sidekiq/pro/version"
+
+  raise "sidekiq-pauzer is incompatible with Sidekiq Pro"
+rescue LoadError
+  # All good - no compatibility issues
+end
+
+module Sidekiq
+  module Pauzer
+    MUTEX = Mutex.new
+
+    @config = Config.new
+    @queues = Queues.new(@config)
+
+    class << self
+      extend Forwardable
+
+      def_delegators :@queues, :pause!, :unpause!, :paused?
+
+      def paused_queues
+        @queues.map { |queue| "#{Queues::QUEUE_PREFIX}#{queue}" }
+      end
+
+      def configure
+        MUTEX.synchronize do
+          yield @config
+        ensure
+          start_refresher = @queues.refresher_running?
+          @queues.stop_refresher
+          @queues = Queues.new(@config)
+          @queues.start_refresher if start_refresher
+        end
+      end
+
+      def startup
+        MUTEX.synchronize { @queues.start_refresher }
+      end
+
+      def shutdown
+        MUTEX.synchronize { @queues.stop_refresher }
+      end
+    end
+  end
+
+  class Queue
+    remove_method :paused?
+
+    def paused?
+      Pauzer.paused?(name)
+    end
+
+    def pause!
+      Pauzer.pause!(name)
+    end
+
+    def unpause!
+      Pauzer.unpause!(name)
+    end
+  end
+
+  configure_server do |config|
+    config.on(:startup) { Pauzer.startup }
+    config.on(:shutdown) { Pauzer.shutdown }
+  end
+end

data/lib/sidekiq-pauzer.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "./sidekiq/pauzer"

data/web/views/queues.erb

@@ -0,0 +1,38 @@
+<div class="header-container">
+  <h1><%= t('Queues') %></h1>
+</div>
+
+<div class="table_container">
+  <table class="queues table table-hover table-bordered table-striped">
+    <thead>
+      <th><%= t('Queue') %></th>
+      <th><%= t('Size') %></th>
+      <th><%= t('Latency') %></th>
+      <th><%= t('Actions') %></th>
+    </thead>
+    <% @queues.each do |queue| %>
+      <tr>
+        <td>
+          <a href="<%= root_path %>queues/<%= CGI.escape(queue.name) %>"><%= h queue.name %></a>
+          <% if queue.paused? %>
+            <span class="label label-danger"><%= t('Paused') %></span>
+          <% end %>
+        </td>
+        <td><%= number_with_delimiter(queue.size) %> </td>
+        <td><% queue_latency = queue.latency %><%= number_with_delimiter(queue_latency.round(2)) %><%= (queue_latency < 60) ? '' : " (#{relative_time(Time.at(Time.now.to_f - queue_latency))})" %> </td>
+        <td class="delete-confirm">
+          <form action="<%=root_path %>queues/<%= CGI.escape(queue.name) %>" method="post">
+            <%= csrf_tag %>
+            <input class="btn btn-danger" type="submit" name="delete" title="This will delete all jobs within the queue, it will reappear if you push more jobs to it in the future." value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSureDeleteQueue', :queue => h(queue.name)) %>" />
+
+            <% if queue.paused? %>
+              <input class="btn btn-danger" type="submit" name="unpause" value="<%= t('Unpause') %>" />
+            <% else %>
+              <input class="btn btn-danger" type="submit" name="pause" value="<%= t('Pause') %>" />
+            <% end %>
+          </form>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+</div>

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-pauzer
+version: !ruby/object:Gem::Version
+  version: 1.0.0.alpha
+platform: ruby
+authors:
+- Alexey Zapparov
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-05-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.5'
+description:
+email:
+- alexey@zapparov.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.adoc
+- lib/sidekiq-pauzer.rb
+- lib/sidekiq/pauzer.rb
+- lib/sidekiq/pauzer/adapters.rb
+- lib/sidekiq/pauzer/adapters/redis.rb
+- lib/sidekiq/pauzer/adapters/redis_client.rb
+- lib/sidekiq/pauzer/basic_fetch.rb
+- lib/sidekiq/pauzer/config.rb
+- lib/sidekiq/pauzer/queues.rb
+- lib/sidekiq/pauzer/version.rb
+- lib/sidekiq/pauzer/web.rb
+- web/views/queues.erb
+homepage: https://gitlab.com/ixti/sidekiq-pauzer
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://gitlab.com/ixti/sidekiq-pauzer
+  source_code_uri: https://gitlab.com/ixti/sidekiq-pauzer/tree/v1.0.0.alpha
+  bug_tracker_uri: https://gitlab.com/ixti/sidekiq-pauzer/issues
+  changelog_uri: https://gitlab.com/ixti/sidekiq-pauzer/blob/v1.0.0.alpha/CHANGES.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Enhance Sidekiq with queue pausing
+test_files: []