sidekiq-debouncer 2.0.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78b258ab5ad21facbcf96f6aab96f30e38542ea24a7c5fd9e23ee99057b67e6d
-  data.tar.gz: 19db1ba7e392a4de9dcedfa8e236b8e8e334d7de56e71d8241aeb4e2d20b5e0a
+  metadata.gz: c98e0c985f240350757ce46298f39cb5586d7e01966f78bd3a886c8b4a78b7e0
+  data.tar.gz: e1f7ceda05cd4cc580fc5d318da59ccaf9a8a3f737020524c76a147add5fa5e3
 SHA512:
-  metadata.gz: 4c30876a9db412ee65a844c966fec6b4f2c84e33f5eb0393773c7db0ad9fd5d6c394b4c9a2f01e31178c93dcc5d5443e829e03f90a9c1db81814f594153af290
-  data.tar.gz: 44e2d19a313ca3d07daaed130a9fb8a21721a6e6ed17756ed21028017096dd528d6b6a2d482a624095822c23158a9777620ae6dbeb0e541b8e2c7acebd003f68
+  metadata.gz: 3175ce8da01db8a593290273a4a7c392133792de93c04a9091662ca96ab24caeea8a60e17e0ab7a7fd16e8b8273d184b62a421deb58374430e26f720f0708927
+  data.tar.gz: 2e42dedd11a4b6c05e9c4af5446803a1fab05daa7fe762b83ef3846b0f5c8cb253bf6f03e1bbbdc8e21311245087526c89098955c088d040c0c09606bba526da

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [3.0.0] - 2024-10-22
+- Complete rewrite of the library ([#25](https://github.com/paladinsoftware/sidekiq-debouncer/pull/25))
+- Read only Web UI ([#26](https://github.com/paladinsoftware/sidekiq-debouncer/pull/26))
+- Drop support for sidekiq 6.x and ruby 2.7 ([#28](https://github.com/paladinsoftware/sidekiq-debouncer/pull/28))
+- Respect sidekiq_options overridden by .set ([#27](https://github.com/paladinsoftware/sidekiq-debouncer/pull/27))
+
+## [2.0.2] - 2023-03-13
+- support Sidekiq::Testing
+
 ## [2.0.1] - 2023-03-04
 - don't remove debounce key in redis to avoid invalid debouncing
 

data/README.md

@@ -99,10 +99,22 @@ Sidekiq.configure_client do |config|
 end
 ```
 
+## Web UI
+Add `require 'sidekiq/debouncer/web'` after `require 'sidekiq/web'`.
+
 ## Testing
 
 In order to test the behavior of `sidekiq-debouncer` it is necessary to disable testing mode. It is the limitation of internal implementation.
 
+Sidekiq::Debouncer will not debounce the jobs in testing mode, instead they'll be executed like normal jobs (pushed to fake queue or executed inline depending on settings).
+Server middleware needs to be added to `Sidekiq::Testing` chain:
+
+```ruby 
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add Sidekiq::Debouncer::Middleware::Server
+end
+```
+
 ## License
 
 MIT Licensed. See LICENSE.txt for details.

data/lib/sidekiq/debouncer/enq.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "sidekiq/scheduled"
+
+module Sidekiq
+  module Debouncer
+    class Enq < ::Sidekiq::Scheduled::Enq
+      extend LuaCommands
+
+      LUA_ZPOPBYSCORE_WITHSCORE = File.read(File.expand_path("../lua/zpopbyscore_withscore.lua", __FILE__))
+      LUA_ZPOPBYSCORE_MULTI = File.read(File.expand_path("../lua/zpopbyscore_multi.lua", __FILE__))
+
+      define_lua_command(:zpopbyscore_withscore, LUA_ZPOPBYSCORE_WITHSCORE)
+      define_lua_command(:zpopbyscore_multi, LUA_ZPOPBYSCORE_MULTI)
+
+      def enqueue_jobs
+        redis do |conn|
+          while !@done && (job, score = zpopbyscore_withscore(conn, [Sidekiq::Debouncer::SET], [Time.now.to_f.to_s]))
+            job_args = zpopbyscore_multi(conn, [job], [score])
+
+            final_args = job_args.map { |elem| Sidekiq.load_json(elem.split("-", 2)[1]) }
+            job_class = job.split("/")[2]
+            klass = Object.const_get(job_class)
+
+            @client.push({"args" => final_args, "class" => klass, "debounce_key" => job})
+
+            logger.debug { "enqueued #{Sidekiq::Debouncer::SET}: #{job}" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer/job.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Debouncer
+    class Job
+      include Sidekiq::JobUtil
+
+      attr_reader :key, :score
+
+      def initialize(key, score)
+        @key = key
+        @score = score
+      end
+
+      def at
+        Time.at(score).utc
+      end
+
+      def args
+        @_args ||= Sidekiq.redis { |conn| conn.call("ZRANGE", key, "-inf", "+inf", "BYSCORE") }
+          .map { |elem| Sidekiq.load_json(elem.split("-", 2)[1]) }
+      end
+
+      def queue
+        normalized["queue"]
+      end
+
+      def klass
+        key.split("/")[2]
+      end
+
+      alias_method :display_class, :klass
+
+      private
+
+      def normalized
+        @_normalized ||= normalize_item({"args" => args, "class" => Object.const_get(klass), "debounce_key" => key})
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer/launcher.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Debouncer
+    module Launcher
+      def initialize(config, **kwargs)
+        @debounce_poller = Sidekiq::Debouncer::Poller.new(config)
+        super
+      end
+
+      def run
+        super
+        @debounce_poller.start
+      end
+
+      def quiet
+        super
+        @debounce_poller.terminate
+      end
+
+      def stop
+        @debounce_poller.terminate
+        super
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do
+  require "sidekiq/launcher"
+
+  ::Sidekiq::Launcher.prepend(Sidekiq::Debouncer::Launcher)
+end

data/lib/sidekiq/debouncer/locales/en.yml

@@ -0,0 +1,4 @@
+en:
+  Debounces: Debounces
+  DebounceKey: Debounce Key
+  NoDebouncesFound: No debounces found

data/lib/sidekiq/debouncer/lua/debounce.lua

@@ -1,22 +1,10 @@
 local set = KEYS[1]
 local debounce_key = KEYS[2]
 
-local job = ARGV[1]
+local args = ARGV[1]
 local time = ARGV[2]
 local ttl = ARGV[3]
 
-local existing_debounce = redis.call("GET", debounce_key)
-
-if existing_debounce then
-    redis.call("DEL", debounce_key)
-    -- skip if job wasn't found in schedule set
-    if redis.call("ZREM", set, existing_debounce) > 0 then
-        local new_args = cjson.decode(job)['args'][1]
-        local new_job = cjson.decode(existing_debounce)
-        table.insert(new_job['args'], new_args)
-        job = cjson.encode(new_job)
-    end
-end
-
-redis.call("SET", debounce_key, job, "EX", ttl)
-redis.call("ZADD", set, time, job)
+redis.call("ZADD", set, time, debounce_key)
+redis.call("ZADD", debounce_key, time, args)
+redis.call("EXPIRE", debounce_key, ttl)

data/lib/sidekiq/debouncer/lua/zpopbyscore_multi.lua

@@ -0,0 +1,5 @@
+local key, now = KEYS[1], ARGV[1]
+local jobs = redis.call("ZRANGE", key, "-inf", now, "BYSCORE")
+redis.call("ZREMRANGEBYSCORE", key, "-inf", now)
+
+return jobs

data/lib/sidekiq/debouncer/lua/zpopbyscore_withscore.lua

@@ -0,0 +1,6 @@
+local key, now = KEYS[1], ARGV[1]
+local jobs = redis.call("ZRANGE", key, "-inf", now, "BYSCORE", "LIMIT", 0, 1, "WITHSCORES")
+if jobs[1] then
+    redis.call("ZREM", key, jobs[1])
+    return {jobs[1], jobs[2]}
+end

data/lib/sidekiq/debouncer/lua_commands.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+
+module Sidekiq
+  module Debouncer
+    module LuaCommands
+      def define_lua_command(command, script)
+        sha = Digest::SHA1.hexdigest(script)
+        define_method(command) do |conn, keys, argv|
+          retryable = true
+          begin
+            conn.call("EVALSHA", sha, keys.size, *keys, *argv)
+          rescue RedisClient::CommandError => e
+            raise if !e.message.start_with?("NOSCRIPT") || !retryable
+
+            # upload script to redis cache and retry
+            conn.call("SCRIPT", "LOAD", script)
+            retryable = false
+            retry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer/middleware/client.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "digest/sha1"
+require "securerandom"
 
 module Sidekiq
   module Debouncer
@@ -11,10 +11,11 @@ module Sidekiq
       # job from schedule set when another debounce occurs.
       class Client
         include Sidekiq::ClientMiddleware
+        extend Sidekiq::Debouncer::LuaCommands
 
         LUA_DEBOUNCE = File.read(File.expand_path("../../lua/debounce.lua", __FILE__))
-        LUA_DEBOUNCE_SHA = Digest::SHA1.hexdigest(LUA_DEBOUNCE)
-        REDIS_ERROR_CLASS = defined?(RedisClient::CommandError) ? RedisClient::CommandError : Redis::CommandError
+
+        define_lua_command(:redis_debounce, LUA_DEBOUNCE)
 
         def initialize(options = {})
           @debounce_key_ttl = options.fetch(:ttl, 60 * 60 * 24) # 24 hours by default
@@ -28,9 +29,6 @@ module Sidekiq
           return job if !klass.get_sidekiq_options["debounce"] || job["debounce_key"]
 
           debounce(klass, job)
-
-          # prevent normal sidekiq flow
-          false
         end
 
         private
@@ -38,27 +36,31 @@ module Sidekiq
         def debounce(klass, job)
           raise NotSupportedError, "perform_at is not supported with debounce" if job.key?("at")
 
-          options = debounce_options(klass)
+          options = debounce_options(klass, job)
           key = debounce_key(klass, job, options)
           time = (options[:time].to_f + Time.now.to_f).to_s
 
-          job["debounce_key"] = key
-          job["args"] = [job["args"]]
-          job.delete("debounce")
+          return job.merge("args" => [job["args"]], "debounce_key" => key) if testing?
+
+          args_stringified = "#{SecureRandom.hex(12)}-#{Sidekiq.dump_json(job["args"])}"
 
           redis do |connection|
-            redis_debounce(connection, keys: ["schedule", key], argv: [Sidekiq.dump_json(job), time, @debounce_key_ttl])
+            redis_debounce(connection, [Sidekiq::Debouncer::SET, key], [args_stringified, time, @debounce_key_ttl])
           end
+
+          # prevent normal sidekiq flow
+          false
         end
 
         def debounce_key(klass, job, options)
           method = options[:by]
           result = method.is_a?(Symbol) ? klass.send(method, job["args"]) : method.call(job["args"])
-          "debounce/#{klass.name}/#{result}"
+          "debounce/v3/#{klass.name}/#{result}"
         end
 
-        def debounce_options(klass)
-          options = klass.get_sidekiq_options["debounce"].transform_keys(&:to_sym)
+        def debounce_options(klass, job)
+          options = job.fetch("debounce") { klass.get_sidekiq_options["debounce"] }
+          options = options.transform_keys(&:to_sym)
 
           raise MissingArgumentError, "'by' attribute not provided" unless options[:by]
           raise MissingArgumentError, "'time' attribute not provided" unless options[:time]
@@ -66,18 +68,8 @@ module Sidekiq
           options
         end
 
-        def redis_debounce(connection, keys:, argv:)
-          retryable = true
-          begin
-            connection.call("EVALSHA", LUA_DEBOUNCE_SHA, keys.size, *keys, *argv)
-          rescue REDIS_ERROR_CLASS => e
-            raise if !e.message.start_with?("NOSCRIPT") || !retryable
-
-            # upload script to redis cache and retry
-            connection.call("SCRIPT", "LOAD", LUA_DEBOUNCE)
-            retryable = false
-            retry
-          end
+        def testing?
+          defined?(Sidekiq::Testing) && Sidekiq::Testing.enabled?
         end
       end
     end

data/lib/sidekiq/debouncer/poller.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "sidekiq/scheduled"
+
+module Sidekiq
+  module Debouncer
+    class Poller < ::Sidekiq::Scheduled::Poller
+      def initialize(config)
+        super
+        @enq = Sidekiq::Debouncer::Enq.new(config)
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer/set.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Debouncer
+    class Set < Sidekiq::JobSet
+      def initialize
+        super Sidekiq::Debouncer::SET
+      end
+
+      def fetch_by_key(key)
+        score = Sidekiq.redis { |conn| conn.zscore(Sidekiq::Debouncer::SET, key) }
+        Job.new(key, score)
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer/version.rb

@@ -2,6 +2,6 @@
 
 module Sidekiq
   module Debouncer
-    VERSION = "2.0.1"
+    VERSION = "3.0.0"
   end
 end

data/lib/sidekiq/debouncer/views/index.html.erb

@@ -0,0 +1,48 @@
+<header class='row'>
+  <div class='col-sm-5 pull-left'>
+    <h3><%= t('Debounces') %></h3>
+  </div>
+  <div class='col-sm-7 pull-right'>
+    <%= erb :_paging, locals: { url: "#{root_path}debounces" } %>
+  </div>
+</header>
+
+<% if @debounces.size > 0 %>
+  <form action="<%= root_path %>debounces" method="post">
+    <%= csrf_tag %>
+    <div class="table_container">
+      <table class="table table-striped table-bordered table-hover">
+        <thead>
+          <tr>
+            <th class="table-checkbox checkbox-column">
+              <label>
+                <input type="checkbox" class="check_all" />
+              </label>
+            </th>
+            <th><%= t('When') %></th>
+            <th><%= t('DebounceKey') %></th>
+          </tr>
+        </thead>
+        <% @debounces.each do |entry| %>
+          <tr>
+            <td class="table-checkbox">
+              <label>
+                <input type='checkbox' name='key[]' value='<%= entry.key %>' class='shift_clickable' />
+              </label>
+            </td>
+            <td>
+              <a href="<%= root_path %>debounces/<%= ::Base64.urlsafe_encode64(entry.key) %>"><%= relative_time(entry.at) %></a>
+            </td>
+            <td>
+              <a href="<%= root_path %>debounces/<%= ::Base64.urlsafe_encode64(entry.key) %>"><%= entry.key %></a>
+            </td>
+          </tr>
+        <% end %>
+      </table>
+    </div>
+<!--    <input class="btn btn-danger pull-right flip" type="submit" name="delete" value="<%#= t('Delete') %>" />-->
+<!--    <input class="btn btn-danger pull-right flip" type="submit" name="add_to_queue" value="<%#= t('AddToQueue') %>" />-->
+  </form>
+<% else %>
+  <div class="alert alert-success"><%= t('NoDebouncesFound') %></div>
+<% end %>

data/lib/sidekiq/debouncer/views/show.html.erb

@@ -0,0 +1,38 @@
+<div class="header-container">
+  <h3><%= t('Job') %></h3>
+</div>
+
+<div class="table_container">
+  <table class="table table-bordered table-striped table-hover">
+    <tbody>
+      <tr>
+        <th><%= t('Queue') %></th>
+        <td><%= @job.queue %></td>
+      </tr>
+      <tr>
+        <th><%= t('When') %></th>
+        <td><%= relative_time(@job.at) %></td>
+      </tr>
+      <tr>
+        <th><%= t('DebounceKey') %></th>
+        <td>
+          <%= @job.key %>
+        </td>
+      </tr>
+      <tr>
+        <th><%= t('Job') %></th>
+        <td>
+          <%= @job.display_class %>
+        </td>
+      </tr>
+      <tr>
+        <th><%= t('Arguments') %></th>
+        <td>
+          <code class="code-wrap">
+            <div class="args-extended"><%= display_args(@job.args, nil) %></div>
+          </code>
+        </td>
+      </tr>
+    </tbody>
+  </table>
+</div>

data/lib/sidekiq/debouncer/web.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+if defined?(Sidekiq::Web)
+  require "sidekiq/debouncer/web_extension"
+
+  Sidekiq::Web.register Sidekiq::Debouncer::WebExtension
+  Sidekiq::Web.tabs["Debounces"] = "debounces"
+end

data/lib/sidekiq/debouncer/web_extension.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Sidekiq
+  module Debouncer
+    module WebExtension
+      def self.registered(app)
+        app.settings.locales << File.join(File.expand_path("..", __FILE__), "locales")
+
+        app.get "/debounces" do
+          view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+          @count = (params["count"] || 25).to_i
+          (@current_page, @total_size, @debounces) = page("debouncer", params["page"], @count)
+          @debounces = @debounces.map { |key, score| Sidekiq::Debouncer::Job.new(key, score) }
+
+          render(:erb, File.read(File.join(view_path, "index.html.erb")))
+        end
+
+        app.get "/debounces/:key" do
+          view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+          @job = Sidekiq::Debouncer::Set.new.fetch_by_key(Base64.urlsafe_decode64(route_params[:key]))
+
+          render(:erb, File.read(File.join(view_path, "show.html.erb")))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/debouncer.rb

@@ -2,11 +2,16 @@
 
 require "sidekiq/debouncer/version"
 require "sidekiq/debouncer/errors"
+require "sidekiq/debouncer/lua_commands"
 require "sidekiq/debouncer/middleware/client"
 require "sidekiq/debouncer/middleware/server"
+require "sidekiq/debouncer/job"
+require "sidekiq/debouncer/set"
 
 module Sidekiq
   module Debouncer
+    SET = "debouncer"
+
     def self.included(base)
       base.extend ClassMethods
     end
@@ -19,3 +24,9 @@ module Sidekiq
     end
   end
 end
+
+Sidekiq.configure_server do
+  require "sidekiq/debouncer/enq"
+  require "sidekiq/debouncer/poller"
+  require "sidekiq/debouncer/launcher"
+end

data/sidekiq-debouncer.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   DESCRIPTION
   gem.homepage = "https://github.com/paladinsoftware/sidekiq-debouncer"
   gem.license = "MIT"
-  gem.required_ruby_version = ">= 2.7.0"
+  gem.required_ruby_version = ">= 3.0.0"
 
   gem.files = Dir.glob("lib/**/*") + [
     "CHANGELOG.md",
@@ -27,11 +27,12 @@ Gem::Specification.new do |gem|
     "sidekiq-debouncer.gemspec"
   ]
 
-  gem.add_dependency "sidekiq", ">= 6.5", "< 8.0"
+  gem.add_dependency "sidekiq", ">= 7.0", "< 8.0"
 
   gem.add_development_dependency "rspec", "~> 3.12.0"
   gem.add_development_dependency "timecop", "~> 0.9.6"
   gem.add_development_dependency "simplecov", "~> 0.22.0"
   gem.add_development_dependency "parallel", "~> 1.22.1"
   gem.add_development_dependency "standard", "~> 1.24.3"
+  gem.add_development_dependency "rack-test", "~> 2.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-debouncer
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Sebastian Zuchmański
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-04 00:00:00.000000000 Z
+date: 2024-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,7 +17,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.5'
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '8.0'
@@ -27,7 +27,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.5'
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '8.0'
@@ -101,6 +101,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.24.3
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 description: |
   Sidekiq extension that adds the ability to debounce job execution.
   Worker will postpone its execution after `wait time` have elapsed since the last time it was invoked.
@@ -117,11 +131,24 @@ files:
 - README.md
 - lib/sidekiq-debouncer.rb
 - lib/sidekiq/debouncer.rb
+- lib/sidekiq/debouncer/enq.rb
 - lib/sidekiq/debouncer/errors.rb
+- lib/sidekiq/debouncer/job.rb
+- lib/sidekiq/debouncer/launcher.rb
+- lib/sidekiq/debouncer/locales/en.yml
 - lib/sidekiq/debouncer/lua/debounce.lua
+- lib/sidekiq/debouncer/lua/zpopbyscore_multi.lua
+- lib/sidekiq/debouncer/lua/zpopbyscore_withscore.lua
+- lib/sidekiq/debouncer/lua_commands.rb
 - lib/sidekiq/debouncer/middleware/client.rb
 - lib/sidekiq/debouncer/middleware/server.rb
+- lib/sidekiq/debouncer/poller.rb
+- lib/sidekiq/debouncer/set.rb
 - lib/sidekiq/debouncer/version.rb
+- lib/sidekiq/debouncer/views/index.html.erb
+- lib/sidekiq/debouncer/views/show.html.erb
+- lib/sidekiq/debouncer/web.rb
+- lib/sidekiq/debouncer/web_extension.rb
 - sidekiq-debouncer.gemspec
 homepage: https://github.com/paladinsoftware/sidekiq-debouncer
 licenses:
@@ -135,7 +162,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="