sidekiq-fairplay 0.0.1

data/lib/sidekiq/fairplay/redis.rb

@@ -0,0 +1,149 @@
+module Sidekiq
+  module Fairplay
+    class Redis
+      LUA = {}
+      SHAS = {}
+
+      LUA[:push_tenant_job] = <<~LUA
+        local queue_key = KEYS[1]
+        local counts_key = KEYS[2]
+
+        local tenant_id = ARGV[1]
+        local job_payload = ARGV[2]
+
+        redis.call('RPUSH', queue_key, job_payload)
+        redis.call('HINCRBY', counts_key, tenant_id, 1)
+
+        -- Clean up after a week of inactivity
+        redis.call('EXPIRE', queue_key, 604800)
+        redis.call('EXPIRE', counts_key, 604800)
+
+        return 1
+      LUA
+
+      LUA[:pop_tenant_job] = <<~LUA
+        local queue_key = KEYS[1]
+        local counts_key = KEYS[2]
+
+        local tenant_id = ARGV[1]
+
+        local popped = redis.call('LPOP', queue_key)
+        if not popped then
+          return 0
+        end
+
+        local newcount = redis.call('HINCRBY', counts_key, tenant_id, -1)
+        if newcount <= 0 then
+          redis.call('HDEL', counts_key, tenant_id)
+        end
+
+        return 1
+      LUA
+
+      LUA[:release_planner_lock] = <<~LUA
+        if redis.call('get', KEYS[1]) == ARGV[1] then
+          redis.call('del', KEYS[1])
+        end
+      LUA
+
+      def self.bootstrap_scripts
+        Sidekiq.redis do |conn|
+          LUA.each_with_object(SHAS) do |(name, lua), memo|
+            memo[name] = conn.call("SCRIPT", "LOAD", lua)
+          end
+        end
+      end
+
+      def push_tenant_job(job_class, tenant_id, payload)
+        script_call(
+          :push_tenant_job,
+          [tenant_queue_redis_key(job_class.name, tenant_id), tenant_counts_redis_key(job_class.name)],
+          [tenant_id.to_s, payload.to_s]
+        )
+      end
+
+      def pop_tenant_job(job_class, tenant_id)
+        script_call(
+          :pop_tenant_job,
+          [tenant_queue_redis_key(job_class.name, tenant_id), tenant_counts_redis_key(job_class.name)],
+          [tenant_id.to_s]
+        )
+      end
+
+      def peek_tenant(job_class, tenant_id)
+        redis_call(:lindex, tenant_queue_redis_key(job_class.name, tenant_id), 0)
+      end
+
+      def tenant_counts(job_class)
+        redis_call(:hgetall, tenant_counts_redis_key(job_class.name)) || {}
+      end
+
+      def with_planner_lock(job_class, jid)
+        return false unless try_acquire_planner_lock(job_class, jid)
+
+        begin
+          yield
+        ensure
+          release_planner_lock(job_class, jid)
+        end
+
+        true
+      end
+
+      def try_acquire_planner_lock(job_class, jid)
+        key = execute_lock_redis_key(job_class.name)
+        ttl = job_class.sidekiq_fairplay_options_hash[:planner_lock_ttl].to_i
+
+        !!redis_call(:set, key, jid.to_s, nx: true, ex: ttl)
+      end
+
+      def release_planner_lock(job_class, jid)
+        script_call(:release_planner_lock, [execute_lock_redis_key(job_class.name)], [jid.to_s])
+      end
+
+      def planner_enqueued_recently?(job_class)
+        key = enqueue_lock_redis_key(job_class.name)
+        window = job_class.sidekiq_fairplay_options_hash[:enqueue_interval].to_i
+
+        redis_call(:set, key, "1", nx: true, ex: window) ? false : true
+      end
+
+      private
+
+      def tenant_counts_redis_key(job_class_name)
+        ns("#{job_class_name.underscore}:tenant_counts")
+      end
+
+      def tenant_queue_redis_key(job_class_name, tenant_id)
+        ns("#{job_class_name.underscore}:tenant_queue:#{tenant_id}")
+      end
+
+      def enqueue_lock_redis_key(job_class_name)
+        ns("#{job_class_name.underscore}:enqueue_lock")
+      end
+
+      def execute_lock_redis_key(job_class_name)
+        ns("#{job_class_name.underscore}:execute_lock")
+      end
+
+      def ns(key)
+        "fairplay:#{key}"
+      end
+
+      def redis_call(command, *args, **kwargs)
+        Sidekiq.redis { |connection| connection.call(command.to_s.upcase, *args, **kwargs) }
+      end
+
+      def script_call(name, keys, args)
+        self.class.bootstrap_scripts if SHAS.length != LUA.length
+
+        redis_call(:evalsha, SHAS[name], keys.size, *keys, *args)
+      rescue RedisClient::CommandError => e
+        raise unless /NOSCRIPT/.match?(e.message)
+
+        self.class.bootstrap_scripts
+        retry
+      end
+    end
+  end
+end

data/lib/sidekiq/fairplay/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Fairplay
+    VERSION = "0.0.1"
+  end
+end

data/lib/sidekiq/fairplay.rb

@@ -0,0 +1,67 @@
+require "active_support"
+require "active_support/inflector"
+require "active_support/core_ext/string"
+require "active_support/configurable"
+require "active_support/core_ext/numeric/time"
+require "sidekiq"
+require "sidekiq/api"
+
+require "sidekiq/fairplay/version"
+
+module Sidekiq
+  module Fairplay
+    autoload :Config, "sidekiq/fairplay/config"
+    autoload :Redis, "sidekiq/fairplay/redis"
+    autoload :Middleware, "sidekiq/fairplay/middleware"
+    autoload :Planner, "sidekiq/fairplay/planner"
+
+    class << self
+      attr_writer :logger
+
+      def logger
+        @logger ||= Sidekiq.logger
+      end
+    end
+  end
+end
+
+module Sidekiq
+  module Fairplay
+    module Job
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def sidekiq_fairplay_options(opts = {})
+          opts = opts.compact
+
+          unless opts.key?(:enqueue_jobs) && opts.key?(:enqueue_interval)
+            raise ArgumentError, "You must specify how many jobs to enqueue and how often."
+          end
+
+          unless opts.key?(:tenant_key) && opts[:tenant_key].respond_to?(:call)
+            raise ArgumentError, "You must provide the tenant_key lambda."
+          end
+
+          @sidekiq_fairplay_options = default_fairplay_options.merge(opts)
+        end
+
+        def sidekiq_fairplay_options_hash
+          @sidekiq_fairplay_options || default_fairplay_options
+        end
+
+        private
+
+        def default_fairplay_options
+          {
+            latency_threshold: Sidekiq::Fairplay::Config.default_latency_threshold,
+            planner_queue: Sidekiq::Fairplay::Config.default_planner_queue,
+            planner_lock_ttl: Sidekiq::Fairplay::Config.default_planner_lock_ttl,
+            tenant_weights: Sidekiq::Fairplay::Config.default_tenant_weights
+          }
+        end
+      end
+    end
+  end
+end

data/sidekiq-fairplay.gemspec

@@ -0,0 +1,37 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "sidekiq/fairplay/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "sidekiq-fairplay"
+  spec.version = Sidekiq::Fairplay::VERSION
+  spec.authors = ["Alexander Baygeldin"]
+  spec.email = ["a.baygeldin@gmail.com"]
+  spec.summary = <<~SUMMARY
+    Make Sidekiq play fair — dynamic job prioritization for multi-tenant apps.
+  SUMMARY
+  spec.homepage = "http://github.com/baygeldin/sidekiq-fairplay"
+  spec.license = "MIT"
+
+  spec.files = `git ls-files -z`.split("\x0")
+  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 3.4.0"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "pry", "~> 0.15"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-sidekiq", "~> 5.0"
+  spec.add_development_dependency "standard", "~> 1.0"
+  spec.add_development_dependency "standard-performance", "~> 1.0"
+  spec.add_development_dependency "standard-rspec", "~> 0.3"
+  spec.add_development_dependency "simplecov", "~> 0.22"
+  spec.add_development_dependency "timecop", "~> 0.9"
+
+  spec.add_dependency "activesupport", "~> 7.0"
+  spec.add_runtime_dependency "sidekiq", "~> 7.0"
+
+  spec.metadata["rubygems_mfa_required"] = "true"
+end

data/spec/sidekiq/fairplay_spec.rb

@@ -0,0 +1,301 @@
+require "spec_helper"
+
+class RegularJob
+  include Sidekiq::Job
+
+  def perform(foo)
+  end
+end
+
+class FairplayJob
+  include Sidekiq::Job
+  include Sidekiq::Fairplay::Job
+
+  def perform(tenant_key, foo)
+  end
+end
+
+RSpec.describe Sidekiq::Fairplay do
+  before do
+    FairplayJob.sidekiq_fairplay_options \
+      enqueue_interval:,
+      enqueue_jobs:,
+      planner_queue:,
+      planner_lock_ttl:,
+      latency_threshold:,
+      tenant_key:,
+      tenant_weights:
+  end
+
+  let(:enqueue_interval) { 1 }
+  let(:enqueue_jobs) { 10 }
+  let(:planner_queue) { "default" }
+  let(:planner_lock_ttl) { 60 }
+  let(:latency_threshold) { 60 }
+  let(:tenant_key) { ->(tenant_key, *_args) { tenant_key } }
+  let(:tenant_weights) { ->(tenant_keys) { tenant_keys.to_h { |tid| [tid, 1] } } }
+
+  describe "fairness (probabilistic)" do
+    # Seed Ruby's PRNG for deterministic results
+    around do |example|
+      prev = srand(1234)
+      example.run
+    ensure
+      srand(prev)
+    end
+
+    let(:enqueue_jobs) { 1000 }
+    let(:tenant_weights) do
+      ->(tenant_ids) do
+        mapping = {"t1" => 1, "t2" => 3, "t3" => 6}
+        tenant_ids.to_h { |tid| [tid, mapping.fetch(tid, 1)] }
+      end
+    end
+
+    it "enqueues approximately proportional to weights" do
+      enqueue_jobs.times do |i|
+        FairplayJob.perform_async("t1", "a#{i}")
+        FairplayJob.perform_async("t2", "b#{i}")
+        FairplayJob.perform_async("t3", "c#{i}")
+      end
+
+      Sidekiq::Fairplay::Planner.new.perform("FairplayJob")
+
+      expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(enqueue_jobs)
+
+      jobs_per_tenant = FairplayJob.jobs.each_with_object(Hash.new(0)) do |job, memo|
+        memo[job["args"].first] += 1
+      end
+
+      expected_jobs_per_tenant = {"t1" => 100, "t2" => 300, "t3" => 600}
+      tolerance = 0.25 # 25% tolerance to avoid flakiness across Ruby versions
+
+      expected_jobs_per_tenant.each do |tid, exp|
+        low = (exp * (1 - tolerance)).floor
+        high = (exp * (1 + tolerance)).ceil
+
+        expect(jobs_per_tenant[tid]).to be_between(low, high).inclusive
+      end
+    end
+  end
+
+  describe "basic functionality" do
+    it "intercepts fairplay jobs and enqueues them later" do
+      FairplayJob.perform_async("t1", "a")
+      FairplayJob.perform_async("t2", "b")
+      FairplayJob.perform_async("t3", "c")
+
+      expect(FairplayJob).not_to have_enqueued_sidekiq_job
+      expect(Sidekiq::Fairplay::Planner)
+        .to have_enqueued_sidekiq_job("FairplayJob")
+        .exactly(1)
+        .immediately
+
+      Sidekiq::Fairplay::Planner.perform_one
+
+      expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(3)
+      expect(FairplayJob).to have_enqueued_sidekiq_job("t1", "a")
+      expect(FairplayJob).to have_enqueued_sidekiq_job("t2", "b")
+      expect(FairplayJob).to have_enqueued_sidekiq_job("t3", "c")
+    end
+
+    context "with custom planner queue" do
+      let(:planner_queue) { "whatever" }
+
+      it "enqueues the planner job on the configured queue" do
+        FairplayJob.perform_async("t1", "a")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(Sidekiq::Fairplay::Planner)
+          .to have_enqueued_sidekiq_job("FairplayJob")
+          .on(planner_queue)
+          .in(enqueue_interval.to_i)
+
+        expect(FairplayJob)
+          .to have_enqueued_sidekiq_job("t1", "a")
+          .on("default") # default queue for FairplayJob
+      end
+    end
+
+    context "when latency threshold exceeded" do
+      let(:queue) { instance_double(Sidekiq::Queue) }
+
+      before do
+        allow(Sidekiq::Queue).to receive(:new).and_return(queue)
+        allow(queue).to receive(:latency).and_return(latency_threshold.to_i + 1)
+      end
+
+      it "reschedules the planner without enqueuing jobs" do
+        FairplayJob.perform_async("t1", "a")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).not_to have_enqueued_sidekiq_job
+        expect(Sidekiq::Fairplay::Planner)
+          .to have_enqueued_sidekiq_job("FairplayJob")
+          .in(enqueue_interval.to_i)
+      end
+    end
+
+    context "with custom weights" do
+      let(:tenant_weights) do
+        ->(tenant_ids) do
+          tenant_ids.to_h do |tid|
+            [tid, (tid == "t1") ? 1 : 0]
+          end
+        end
+      end
+
+      it "uses weights to prefer specific tenant" do
+        FairplayJob.perform_async("t1", "a")
+        FairplayJob.perform_async("t2", "b")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(1)
+        expect(FairplayJob).to have_enqueued_sidekiq_job("t1", "a")
+        expect(FairplayJob).not_to have_enqueued_sidekiq_job("t2", "b")
+      end
+    end
+
+    context "when too many jobs in the queue" do
+      let(:enqueue_jobs) { 1 }
+
+      it "respects the enqueue_jobs limit" do
+        FairplayJob.perform_async("t1", "a")
+        FairplayJob.perform_async("t1", "b")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(1)
+        expect(FairplayJob)
+          .to have_enqueued_sidekiq_job("t1", "a")
+          .or have_enqueued_sidekiq_job("t1", "b")
+      end
+    end
+  end
+
+  describe "edge cases" do
+    it "ignores unknown job class" do
+      Sidekiq::Fairplay::Planner.new.perform("UnknownJob")
+
+      expect(Sidekiq::Fairplay::Planner).not_to have_enqueued_sidekiq_job
+      expect(FairplayJob).not_to have_enqueued_sidekiq_job
+    end
+
+    it "has no effect on regular jobs" do
+      RegularJob.perform_async("foo")
+
+      expect(RegularJob).to have_enqueued_sidekiq_job("foo")
+      expect(Sidekiq::Fairplay::Planner).not_to have_enqueued_sidekiq_job
+    end
+
+    it "has no effect on scheduled jobs" do
+      FairplayJob.perform_in(5, "t1", "a")
+
+      expect(FairplayJob).to have_enqueued_sidekiq_job("t1", "a").in(5)
+      expect(Sidekiq::Fairplay::Planner).not_to have_enqueued_sidekiq_job
+    end
+
+    context "with zero weights for all tenants" do
+      let(:tenant_weights) do
+        ->(tenant_ids) { tenant_ids.to_h { |tid| [tid, 0] } }
+      end
+
+      it "enqueues no jobs" do
+        FairplayJob.perform_async("t1", "a")
+        FairplayJob.perform_async("t2", "b")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).not_to have_enqueued_sidekiq_job
+      end
+    end
+  end
+
+  describe "errors" do
+    let(:tenant_key) { ->(_tid, *_args) {} }
+
+    it "raises when tenant key resolves to nil" do
+      tenant_key
+
+      expect { FairplayJob.perform_async("t1", "a") }
+        .to raise_error(ArgumentError, /tenant key cannot be nil/)
+    end
+  end
+
+  describe "implementation details" do
+    it "reschedules planning for the next interval" do
+      FairplayJob.perform_async("t1", "a")
+
+      Sidekiq::Fairplay::Planner.perform_one
+
+      expect(Sidekiq::Fairplay::Planner)
+        .to have_enqueued_sidekiq_job("FairplayJob")
+        .in(enqueue_interval.to_i)
+    end
+
+    context "when planner_lock_ttl is being held" do
+      let(:planner_lock_ttl) { 42 }
+
+      before do
+        redis = Sidekiq::Fairplay::Redis.new
+        redis.try_acquire_planner_lock(FairplayJob, "some_jid")
+      end
+
+      it "blocks planning until the TTL expires" do
+        FairplayJob.perform_async("t1", "a")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).not_to have_enqueued_sidekiq_job
+        expect(Sidekiq::Fairplay::Planner)
+          .to have_enqueued_sidekiq_job("FairplayJob")
+          .in(enqueue_interval.to_i)
+      end
+    end
+
+    context "when tenant_key and tenant_weights refer to class methods" do
+      before do
+        class << FairplayJob
+          def static_tenant_key(tid, *_args) = tid
+          def static_tenant_weights(tids) = tids.to_h { |tid| [tid, 1] }
+        end
+      end
+
+      let(:tenant_key) { ->(tid, *args) { static_tenant_key(tid, *args) } }
+      let(:tenant_weights) { ->(tids) { static_tenant_weights(tids) } }
+
+      it "works as expected" do
+        FairplayJob.perform_async("t1", "a")
+        FairplayJob.perform_async("t2", "b")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(2)
+        expect(FairplayJob).to have_enqueued_sidekiq_job("t1", "a")
+        expect(FairplayJob).to have_enqueued_sidekiq_job("t2", "b")
+      end
+    end
+
+    context "when using ActiveSupport::Duration" do
+      let(:enqueue_interval) { 1.minute }
+      let(:latency_threshold) { 1.hour }
+      let(:planner_lock_ttl) { 10.seconds }
+
+      it "handles durations correctly" do
+        FairplayJob.perform_async("t1", "a")
+
+        Sidekiq::Fairplay::Planner.perform_one
+
+        expect(Sidekiq::Fairplay::Planner)
+          .to have_enqueued_sidekiq_job("FairplayJob")
+          .in(enqueue_interval.to_i)
+
+        expect(FairplayJob).to have_enqueued_sidekiq_job.exactly(1)
+        expect(FairplayJob).to have_enqueued_sidekiq_job("t1", "a")
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,63 @@
+$LOAD_PATH << "." unless $LOAD_PATH.include?(".")
+
+require "rubygems"
+require "bundler/setup"
+require "timecop"
+require "simplecov"
+
+require "sidekiq"
+require "rspec-sidekiq"
+require "sidekiq/fairplay"
+require "pry"
+
+SimpleCov.start do
+  add_filter "spec"
+end
+
+Sidekiq::Fairplay.logger = nil
+
+Sidekiq.configure_client do |config|
+  config.redis = {db: 1}
+  config.logger = nil
+
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Fairplay::Middleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = {db: 1}
+  config.logger = nil
+
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Fairplay::Middleware
+  end
+end
+
+RSpec::Sidekiq.configure do |config|
+  config.clear_all_enqueued_jobs = true
+  config.warn_when_jobs_not_processed_by_sidekiq = false
+end
+
+RSpec.configure do |config|
+  config.order = :random
+  config.run_all_when_everything_filtered = true
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  config.before do
+    Sidekiq.redis do |conn|
+      keys = conn.call("KEYS", "fairplay*")
+      keys.each { |key| conn.call("DEL", key) }
+    end
+  end
+
+  config.before do
+    Timecop.freeze
+  end
+
+  config.after do
+    Timecop.return
+  end
+end
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")