sidekiq-robust-job 0.1.0

data/lib/sidekiq_robust_job/perform_missed_jobs_job.rb

@@ -0,0 +1,9 @@
+class SidekiqRobustJob
+  class PerformMissedJobsJob
+    include Sidekiq::Worker
+
+    def perform
+      SidekiqRobustJob::DependenciesContainer["missed_jobs"].invoke
+    end
+  end
+end

data/lib/sidekiq_robust_job/repository.rb

@@ -0,0 +1,50 @@
+class SidekiqRobustJob
+  class Repository
+    attr_reader :jobs_database, :clock
+    private     :jobs_database, :clock
+
+    def initialize(jobs_database:, clock:)
+      @jobs_database = jobs_database
+      @clock = clock
+    end
+
+    def transaction
+      jobs_database.transaction { yield }
+    end
+
+    def find(id)
+      jobs_database.find(id)
+    end
+
+    def save(record)
+      record.save!
+    end
+
+    def create(attributes)
+      jobs_database.create!(attributes)
+    end
+
+    def missed_jobs(missed_job_policy:)
+      jobs_database
+        .where(completed_at: nil, dropped_at: nil, failed_at: nil)
+        .select { |potentially_missed_job| missed_job_policy.call(potentially_missed_job) }
+    end
+
+    def unprocessed_for_digest(digest, exclude_id:)
+      jobs_database
+        .where(digest: digest)
+        .where(completed_at: nil)
+        .where(dropped_at: nil)
+        .where.not(id: exclude_id)
+    end
+
+    def drop_unprocessed_jobs_by_digest(dropped_by_job_id:, digest:, exclude_id:)
+      transaction do
+        unprocessed_for_digest(digest, exclude_id: exclude_id).lock!.find_each do |job|
+          job.drop(dropped_by_job_id: dropped_by_job_id)
+          save(job)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_robust_job/setter_proxy_job.rb

@@ -0,0 +1,70 @@
+class SidekiqRobustJob
+  class SetterProxyJob
+    def build(job_class, options)
+      Class.new(SimpleDelegator) do
+        attr_reader :job_class, :custom_options
+        private     :job_class, :custom_options
+
+        def initialize(job_class, custom_options = {})
+          super(job_class)
+
+          @job_class = job_class
+          @custom_options = custom_options
+        end
+
+        # rubocop:disable Naming/AccessorMethodName
+        def get_sidekiq_options
+          job_class.get_sidekiq_options.merge(custom_options.stringify_keys)
+        end
+        # rubocop:enable Naming/AccessorMethodName
+
+        def perform_async(*arguments)
+          SidekiqRobustJob.perform_async(self, *arguments)
+        end
+
+        def perform_in(interval, *arguments)
+          SidekiqRobustJob.perform_in(self, interval, *arguments)
+        end
+
+        def perform_at(time, *arguments)
+          SidekiqRobustJob.perform_at(self, time, *arguments)
+        end
+
+        def set(options = {})
+          SidekiqRobustJob.set(self, options)
+        end
+
+        def original_perform_in(*args)
+          call_sidekiq_method(:perform_in, *args)
+        end
+
+        def original_perform_at(*args)
+          call_sidekiq_method(:perform_at, *args)
+        end
+
+        def original_perform_async(*args)
+          call_sidekiq_method(:perform_async, *args)
+        end
+
+        def original_set(*args)
+          call_sidekiq_method(:set, *args)
+        end
+
+        # override to not fail on Sidekiq internal validation
+        def is_a?(val)
+          if val == Class
+            true
+          else
+            super
+          end
+        end
+
+        private
+
+        def call_sidekiq_method(name, *args)
+          Sidekiq::Worker::ClassMethods.instance_method(name).bind(self).call(*args)
+        end
+      end.new(job_class, options)
+    end
+  end
+end

data/lib/sidekiq_robust_job/sidekiq_job_extensions.rb

@@ -0,0 +1,34 @@
+class SidekiqRobustJob
+  module SidekiqJobExtensions
+    extend ActiveSupport::Concern
+
+    included do
+      class << self
+        alias_method :original_perform_async, :perform_async
+        alias_method :original_perform_in, :perform_in
+        alias_method :original_perform_at, :perform_at
+        alias_method :original_set, :set
+      end
+
+      def self.perform_async(*arguments)
+        SidekiqRobustJob.perform_async(self, *arguments)
+      end
+
+      def self.perform_in(interval, *arguments)
+        SidekiqRobustJob.perform_in(self, interval, *arguments)
+      end
+
+      def self.perform_at(time, *arguments)
+        SidekiqRobustJob.perform_at(self, time, *arguments)
+      end
+
+      def self.set(options = {})
+        SidekiqRobustJob.set(self, options)
+      end
+    end
+
+    def perform(job_id)
+      SidekiqRobustJob.perform(job_id)
+    end
+  end
+end

data/lib/sidekiq_robust_job/sidekiq_job_manager.rb

@@ -0,0 +1,80 @@
+class SidekiqRobustJob
+  class SidekiqJobManager
+    attr_reader :jobs_repository, :clock, :digest_generator, :memory_monitor
+    private     :jobs_repository, :clock, :digest_generator, :memory_monitor
+
+    def initialize(jobs_repository:, clock:, digest_generator:, memory_monitor:)
+      @jobs_repository = jobs_repository
+      @clock = clock
+      @digest_generator = digest_generator
+      @memory_monitor = memory_monitor
+    end
+
+    def perform_async(job_class, *arguments)
+      job = create_job(job_class, *arguments)
+      return if job.unprocessable?
+      job_class.original_perform_async(job.id).tap do |sidekiq_jid|
+        job.assign_sidekiq_data(execute_at: clock.now, sidekiq_jid: sidekiq_jid)
+        jobs_repository.save(job)
+      end
+    end
+
+    def perform_in(job_class, interval, *arguments)
+      job = create_job(job_class, *arguments)
+      return if job.unprocessable?
+      job_class.original_perform_in(interval, job.id).tap do |sidekiq_jid|
+        job.assign_sidekiq_data(execute_at: clock.now + interval, sidekiq_jid: sidekiq_jid)
+        jobs_repository.save(job)
+      end
+    end
+
+    def perform_at(job_class, time, *arguments)
+      job = create_job(job_class, *arguments)
+      return if job.unprocessable?
+      job_class.original_perform_at(time, job.id).tap do |sidekiq_jid|
+        job.assign_sidekiq_data(execute_at: time, sidekiq_jid: sidekiq_jid)
+        jobs_repository.save(job)
+      end
+    end
+
+    def set(job_class, options = {})
+      SidekiqRobustJob::DependenciesContainer["setter_proxy_job"].build(job_class, options)
+    end
+
+    def perform(job_id)
+      job = jobs_repository.find(job_id)
+      return if job.unprocessable?
+
+      job.started(memory_monitor: memory_monitor)
+      jobs_repository.save(job)
+      job.execute
+    end
+
+    private
+
+    def create_job(job_class, *arguments)
+      jobs_repository.create(
+        job_class: job_class,
+        arguments: Array.wrap(arguments),
+        enqueued_at: clock.now,
+        digest: digest_generator.generate(job_class, *arguments),
+        queue: job_class.get_sidekiq_options.fetch("queue", "default"),
+        uniqueness_strategy: job_class.get_sidekiq_options.fetch("uniqueness_strategy",
+          SidekiqRobustJob::UniquenessStrategy.no_uniqueness),
+        enqueue_conflict_resolution_strategy: job_class.get_sidekiq_options.fetch("enqueue_conflict_resolution_strategy",
+          SidekiqRobustJob::EnqueueConflictResolutionStrategy.do_nothing)
+      ).tap do |job|
+        jobs_repository.transaction do
+          resolve_potential_conflict_for_enqueueing(job)
+          jobs_repository.save(job)
+        end
+      end
+    end
+
+    def resolve_potential_conflict_for_enqueueing(job)
+      SidekiqRobustJob::DependenciesContainer["enqueue_conflict_resolution_resolver"]
+        .resolve(job.enqueue_conflict_resolution_strategy)
+        .execute(job)
+    end
+  end
+end

data/lib/sidekiq_robust_job/support/matchers/enqueue_sidekiq_robust_job.rb

@@ -0,0 +1,71 @@
+if Object.const_defined?("RSpec")
+  RSpec::Matchers.define :enqueue_sidekiq_robust_job do |job_class|
+    supports_block_expectations
+
+    match do |block|
+      robust_jobs_before = robust_jobs
+
+      block.call
+
+      robust_jobs_before.empty? && expected_job_after_execution_exists?
+    end
+
+    failure_message do
+      "expected #{job_class} to have enqueued job with #{@args} to be executed at: #{execution_time}. " +
+          "All enqueued jobs of this type are: #{formatted_sidekiq_jobs}. Perhaps the job was enqueued before or execution time didn't match?"
+    end
+
+    define_method :jobs do
+      SidekiqRobustJob.configuration.sidekiq_job_model.all.to_a
+    end
+
+    define_method :sidekiq_jobs do
+      job_class.jobs
+    end
+
+    define_method :formatted_sidekiq_jobs do
+      sidekiq_jobs.map do |job|
+        arguments = SidekiqRobustJob.configuration.sidekiq_job_model.find(job["args"].first).arguments
+        "arguments: #{arguments}, at: #{job['at']}"
+      end.join(";")
+    end
+
+    define_method :execution_time do
+      raise "cannot use both :in and :at!" if @interval && @at
+
+      if @interval
+        Time.now.to_i + @interval
+      elsif @at
+        @at.to_time.to_i
+      else
+        Time.now.to_i
+      end
+    end
+
+    define_method :expected_job_after_execution_exists? do
+      robust_jobs.any? do |robust_job|
+        sidekiq_jobs.any? do |sidekiq_job|
+          sidekiq_job_at = sidekiq_job["at"]
+          sidekiq_job["args"] == [robust_job.id] && (sidekiq_job_at.nil? || execution_time.to_i == Time.at(sidekiq_job_at).to_i)
+        end
+      end
+    end
+
+    define_method :robust_jobs do
+      jobs.select { |job| job.job_class == job_class.to_s && job.arguments == @args }
+    end
+
+    chain :with do |*args|
+      @args = Array.wrap(args)
+    end
+
+    chain :in do |interval|
+      @interval = interval
+    end
+
+    chain :at do |at_time|
+      @at = at_time
+    end
+  end
+  RSpec::Matchers.define_negated_matcher :not_enqueue_sidekiq_robust_job, :enqueue_sidekiq_robust_job
+end

data/lib/sidekiq_robust_job/uniqueness_strategy.rb

@@ -0,0 +1,66 @@
+class SidekiqRobustJob
+  class UniquenessStrategy
+    def self.no_uniqueness
+      :no_uniqueness
+    end
+
+    def self.until_executing
+      :until_executing
+    end
+
+    def self.until_executed
+      :until_executed
+    end
+
+    attr_reader :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
+    private     :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
+
+    def initialize(locker:, lock_ttl_proc:, jobs_repository:, memory_monitor:)
+      @locker = locker
+      @lock_ttl_proc = lock_ttl_proc
+      @jobs_repository = jobs_repository
+      @memory_monitor = memory_monitor
+    end
+
+    def resolve(strategy)
+      case strategy.to_sym
+      when SidekiqRobustJob::UniquenessStrategy.no_uniqueness
+        SidekiqRobustJob::UniquenessStrategy::NoUniqueness.new(
+          locker: locker,
+          lock_ttl_proc: lock_ttl_proc,
+          jobs_repository: jobs_repository,
+          memory_monitor: memory_monitor
+        )
+      when SidekiqRobustJob::UniquenessStrategy.until_executing
+        SidekiqRobustJob::UniquenessStrategy::UntilExecuting.new(
+          locker: locker,
+          lock_ttl_proc: lock_ttl_proc,
+          jobs_repository: jobs_repository,
+          memory_monitor: memory_monitor
+        )
+      when SidekiqRobustJob::UniquenessStrategy.until_executed
+        SidekiqRobustJob::UniquenessStrategy::UntilExecuted.new(
+          locker: locker,
+          lock_ttl_proc: lock_ttl_proc,
+          jobs_repository: jobs_repository,
+          memory_monitor: memory_monitor
+        )
+      else
+        raise UnknownStrategyError.new(strategy)
+      end
+    end
+
+    class UnknownStrategyError < StandardError
+      attr_reader :strategy
+      private     :strategy
+
+      def initialize(strategy)
+        @strategy = strategy
+      end
+
+      def message
+        "unknown uniqueness strategy: #{strategy}"
+      end
+    end
+  end
+end

data/lib/sidekiq_robust_job/uniqueness_strategy/base.rb

@@ -0,0 +1,46 @@
+class SidekiqRobustJob
+  class UniquenessStrategy
+    class Base
+      attr_reader :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
+      private     :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
+
+      def initialize(locker:, lock_ttl_proc:,  jobs_repository:, memory_monitor:)
+        @locker = locker
+        @lock_ttl_proc = lock_ttl_proc
+        @jobs_repository = jobs_repository
+        @memory_monitor = memory_monitor
+      end
+
+      def execute(_job)
+        raise "implement me"
+      end
+
+      private
+
+      def perform_job_and_finalize(job)
+        begin
+          job.call
+        rescue StandardError => error
+          job.failed(error)
+          jobs_repository.save(job)
+          raise
+        end
+
+        job.completed(memory_monitor: memory_monitor)
+        jobs_repository.save(job)
+      end
+
+      def drop_unprocessed_jobs(job)
+        jobs_repository.drop_unprocessed_jobs_by_digest(
+          dropped_by_job_id: job.id,
+          digest: job.digest,
+          exclude_id: job.id
+        )
+      end
+
+      def lock(job)
+        locker.lock(job.digest, lock_ttl_proc.call(job)) { |locked| yield locked }
+      end
+    end
+  end
+end

data/lib/sidekiq_robust_job/uniqueness_strategy/no_uniqueness.rb

@@ -0,0 +1,9 @@
+class SidekiqRobustJob
+  class UniquenessStrategy
+    class NoUniqueness < SidekiqRobustJob::UniquenessStrategy::Base
+      def execute(job)
+        perform_job_and_finalize(job)
+      end
+    end
+  end
+end

data/lib/sidekiq_robust_job/uniqueness_strategy/until_executed.rb

@@ -0,0 +1,16 @@
+class SidekiqRobustJob
+  class UniquenessStrategy
+    class UntilExecuted < SidekiqRobustJob::UniquenessStrategy::Base
+      def execute(job)
+        lock(job) do |locked|
+          if locked
+            perform_job_and_finalize(job)
+            drop_unprocessed_jobs(job)
+          else
+            job.reschedule and return
+          end
+        end
+      end
+    end
+  end
+end