canvas_sync 0.21.1.beta1 → 0.22.0.beta1

data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb

@@ -0,0 +1,135 @@
+
+module CanvasSync::JobUniqueness
+  module Compat
+    module Sidekiq
+      module WorkerExtension
+        extend ActiveSupport::Concern
+        include UniqueJobCommon
+
+        class_methods do
+          def ensure_uniqueness(**kwargs)
+            super(**kwargs)
+            if !(defined?(@@validated_config) && @@validated_config)
+              Compat::Sidekiq.validate_middleware_placement!()
+              @@validated_config = true
+            end
+          end
+        end
+      end
+
+      class SidekiqLockContext < LockContext
+        def job_scheduled_at
+          @job_instance["at"]
+        end
+
+        def reenqueue(schedule_in:)
+          job_class.set(queue: job_queue.to_sym).perform_in(schedule_in, *@job_instance["args"])
+        end
+      end
+
+      class CommonMiddleware
+        def lock_context(msg)
+          opts = worker_uniqueness(msg)
+          return nil unless opts
+
+          SidekiqLockContext.new({
+            job_clazz: msg['class'],
+            jid: msg['jid'],
+            queue: msg['queue'],
+            args: msg['args'],
+            # kwargs: msg['kwargs'],
+            **(msg['uniqueness_cache_data'] || {}),
+          }, job_instance: msg)
+        end
+
+        def worker_uniqueness(msg)
+          return nil if Compat::Sidekiq.is_activejob_job?(msg)
+
+          worker_class = msg['class'].constantize
+          return nil unless worker_class.respond_to?(:unique_job_options)
+
+          worker_class.unique_job_options
+        end
+      end
+
+      class ClientMiddleware < CommonMiddleware
+        include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
+
+        def call(_worker, msg, _queue, _redis_pool = nil, &blk)
+          ctx = lock_context(msg)
+          return blk.call unless ctx
+          msg['uniqueness_cache_data'] = ctx.cache_data
+          ctx.handle_lifecycle!(:enqueue, &blk)
+        end
+      end
+
+      class ServerMiddleware < CommonMiddleware
+        include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
+
+        def call(_worker, msg, _queue, &blk)
+          ctx = lock_context(msg)
+          return blk.call unless ctx
+          ctx.handle_lifecycle!(:perform, &blk)
+        end
+      end
+
+      def self.is_activejob_job?(msg)
+        return false unless defined?(::ActiveJob)
+
+        msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < Compat::ActiveJob::UniqueJobExtension
+      end
+
+      def self.validate_middleware_order(chain, order)
+        chain_classes = chain.entries.map(&:klass)
+        filtered = chain_classes.select { |klass| order.include?(klass) }
+        raise "Middleware chain does not contain all required middleware: #{order - filtered}" unless order.all? { |klass| filtered.include?(klass) }
+        raise "Middleware must be in order: #{order.inspect}" if filtered != order
+      end
+
+      def self.sidekiq_middleware(placement, &blk)
+        install_middleware = ->(config) do
+          config.send("#{placement}_middleware") do |chain|
+            blk.call(chain)
+          end
+        end
+
+        ::Sidekiq.configure_client(&install_middleware) if placement == :client
+        ::Sidekiq.configure_server(&install_middleware)
+      end
+
+      def self.validate_middleware_placement!
+        sidekiq_middleware(:client) do |chain|
+          # Unique middleware must come _before_ the Batch middleware so that the uniqueness middleware can wrap the job in a batch
+          validate_middleware_order(chain, [
+            CanvasSync::JobUniqueness::Compat::Sidekiq::ClientMiddleware,
+            CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware,
+          ])
+        end
+
+        sidekiq_middleware(:server) do |chain|
+          # Unique middleware must com _after_ the Batch middleware so that the Batch is loaded before reaching the uniqueness middleware
+          validate_middleware_order(chain, [
+            CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware,
+            CanvasSync::JobUniqueness::Compat::Sidekiq::ServerMiddleware,
+          ])
+        end
+      end
+
+      def self.configure
+        sidekiq_middleware(:client) do |chain|
+          chain.insert_before CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware, Compat::Sidekiq::ClientMiddleware
+        end
+
+        sidekiq_middleware(:server) do |chain|
+          chain.insert_after CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware, Compat::Sidekiq::ServerMiddleware
+        end
+
+        ::Sidekiq::Worker.extend(ActiveSupport::Concern) unless ::Sidekiq::Worker < ActiveSupport::Concern
+
+        ::Sidekiq::Worker.send(:include, Compat::Sidekiq::WorkerExtension)
+      end
+    end
+  end
+end
+
+# require_relative 'sidekiq/web'

data/lib/canvas_sync/job_uniqueness/compat.rb

@@ -0,0 +1,20 @@
+
+module CanvasSync::JobUniqueness
+  module Compat
+    def self.load_compat(name)
+      name = name.to_s
+      begin
+        require name
+      rescue LoadError
+      end
+
+      if name.classify.safe_constantize
+        require_relative "./compat/#{name}"
+        "#{self.name}::#{name.classify}".constantize.configure
+      end
+    end
+
+    load_compat(:active_job)
+    load_compat(:sidekiq)
+  end
+end

data/lib/canvas_sync/job_uniqueness/configuration.rb

@@ -0,0 +1,25 @@
+module CanvasSync::JobUniqueness
+  class Configuration
+    include ActiveSupport::Configurable
+
+    config_accessor(:lock_ttl) { 14.days.to_i }
+    config_accessor(:lock_prefix) { 'uniquejob' }
+    config_accessor(:on_conflict) { :raise }
+
+    config_accessor(:lock_strategies) { {} }
+    config_accessor(:conflict_strategies) { {} }
+
+    config_accessor(:redis_version) { nil }
+    def on_conflict=(action)
+      validate_on_conflict_action!(action)
+
+      config.on_conflict = action
+    end
+
+    def validate_on_conflict_action!(action)
+      return if action.nil? || %i[log raise].include?(action) || action.respond_to?(:call)
+
+      raise ActiveJob::Uniqueness::InvalidOnConflictAction, "Unexpected '#{action}' action on conflict"
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb

@@ -0,0 +1,47 @@
+
+require 'canvas_sync/job_uniqueness/lock_context'
+require 'canvas_sync/job_uniqueness/unique_job_common'
+
+require 'canvas_sync/job_uniqueness/strategy'
+require 'canvas_sync/job_uniqueness/on_conflict'
+require 'canvas_sync/job_uniqueness/compat'
+
+module CanvasSync::JobUniqueness
+  extend ActiveSupport::Autoload
+
+  autoload :Locksmith
+  autoload :Configuration
+
+  class Conflict < StandardError
+    attr_reader :lock_context
+
+    def initialize(lock_context)
+      super()
+      @lock_context = lock_context
+    end
+  end
+
+  class CouldNotLockError < StandardError
+    attr_reader :lock_context, :source
+
+    def initialize(lock_context, source:)
+      super()
+      @lock_context = lock_context
+      @source = source
+    end
+  end
+
+  class << self
+    def configure
+      yield config
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def logger
+      CanvasSync.logger
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/lock_context.rb

@@ -0,0 +1,171 @@
+
+module CanvasSync::JobUniqueness
+  class LockContext
+    def self.from_serialized(data, **kwargs)
+      context_class = data[:clazz]&.constantize || self
+      context_class.new(data, **kwargs)
+    end
+
+    # { job_clazz, jid, queue, args?, kwargs?, base_key? }
+    def initialize(data, job_instance: nil, config: nil)
+      @base_key = data[:base_key]
+      @context_data = data
+      @job_instance = job_instance
+      @config = config || @context_data[:config]
+    end
+
+    # This is primarily for rehydrating in a Batch Callback, so it is unlikely that args and kwargs are needed.
+    # Honestly, base_key and job_clazz are probably the only needed values
+    def serialize
+      {
+        clazz: self.class.to_s,
+        job_clazz: @context_data[:job_clazz].to_s,
+        jid: @context_data[:jid],
+        queue: @context_data[:queue],
+        **cache_data,
+      }
+    end
+
+    # Properties to cache on the serialized Job object to prevent issues arising from code changes between enqueue and perform
+    def cache_data
+      {
+        base_key: base_key,
+        job_score: job_score,
+        # TODO Should config also be cached on the Job at time of enqueue?
+        # config: config,
+      }
+    end
+
+    def debug_data
+      {
+        context_class: self.class.to_s,
+        job_class: job_class.to_s,
+        queue: job_queue,
+        limit: config[:limit],
+        timeout: config[:timeout],
+        ttl: config[:ttl],
+        strategy: config[:strategy],
+        time: Time.now.to_f,
+        at: job_scheduled_at,
+      }
+    end
+
+    def handle_lifecycle!(stage, *args, **kwargs, &blk)
+      lock_strategy.send(:"on_#{stage}", *args, **kwargs, &blk)
+    rescue CouldNotLockError => e
+      call_conflict_strategy(stage)
+    end
+
+    def lock_strategy
+      return @lock_strategy if defined?(@lock_strategy)
+
+      strat_name = config[:strategy]
+      @lock_strategy = Strategy.lookup(strat_name).new(self)
+    end
+
+    def config
+      @config ||= job_class.unique_job_options
+    end
+
+    def job_class
+      @job_class ||= begin
+        if (job_clazz = @context_data[:job_clazz]).is_a?(String)
+          job_clazz.constantize
+        else
+          job_clazz
+        end
+      end
+    end
+
+    def job_id
+      @context_data[:jid]
+    end
+
+    def job_queue
+      @context_data[:queue]
+    end
+
+    def job_scheduled_at
+      nil
+    end
+
+    def job_score
+      @context_data[:job_score] || job_scheduled_at.to_s
+    end
+
+    def base_key(any_hash: false)
+      @base_key ||= begin
+        queue = @context_data[:queue] || "default"
+
+        base_key = [
+          CanvasSync::JobUniqueness.config.lock_prefix.presence,
+        ].compact
+
+        scope = config[:scope]
+        if scope.is_a?(Proc)
+          base_key << scope.call(queue: queue)
+        elsif scope == :global
+          base_key << job_class.name
+        elsif scope == :per_queue
+          base_key << job_class.name
+          base_key << queue
+        else
+          base_key << scope
+        end
+
+        args = @context_data[:args] || []
+        kwargs = @context_data[:kwargs] || {}
+        if config[:hash].is_a?(Proc)
+          base_key << config[:hash].call(*args, **kwargs)
+        elsif config[:hash].nil?
+          base_key << OpenSSL::Digest::MD5.hexdigest(JSON.dump([ args, kwargs.sort]))
+        else
+          base_key << config[:hash]
+        end
+
+        base_key.join(":")
+      end
+    end
+
+    def reenqueue
+      raise NotImplementedError, "needs to be implemented in child class"
+    end
+
+    protected
+
+    attr_reader :job_instance
+
+    #
+    # Call whatever strategy that has been configured
+    #
+    # @param [Symbol] origin the origin `:client` or `:server`
+    #
+    # @return [void] the return value is irrelevant
+    #
+    # @yieldparam [void] if a new job id was set and a block is given
+    # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+    #   one specific situation to yield back to the middleware.
+    def call_conflict_strategy(origin)
+      new_job_id = nil
+      strategy = conflict_strategy_for(origin)
+
+      strategy.call
+    end
+
+    def conflict_strategy_for(origin)
+      raise ArgumentError, "#origin needs to be either `:enqueue` or `:perform`" unless %i[enqueue perform].include?(origin)
+
+      @conflict_strategies ||= {}
+      @conflict_strategies[origin] ||= begin
+        cfg = config[:on_conflict]
+        cfg = cfg[origin] if cfg.is_a?(Hash)
+
+        cstrat_cls = OnConflict.lookup(cfg || :null_strategy)
+        cstrat = cstrat_cls.new(self) # TODO Pass redis_pool?
+      end
+    end
+
+    private
+
+  end
+end

data/lib/canvas_sync/job_uniqueness/locksmith.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "sidekiq_unique_jobs"
+
+module CanvasSync::JobUniqueness
+  # This class is intended to be the complete translation layer between CanvasSync::JobUniqueness and SidekiqUniqueJobs.
+  # In other words, you could consider it the "locking backend" and thus could potentially swap out SUJ for a more succinct solution.
+  #
+  # SUJ's implementation is somewhat complex, but is somewhat pre-tailored over (eg) https://github.com/leandromoreira/redlock-rb.
+  # Mainly SUJ tracks the JID so that if a process dies, another can pick up the job without having to figure out how to unlock it.
+  # SUJ also handles the integration into Sidekiq Web, which is a nice bonus.
+  class Locksmith < SidekiqUniqueJobs::Locksmith
+    attr_reader :lock_context
+
+    def initialize(key, lock_context, redis_pool = nil)
+      @lock_context = lock_context
+      @job_id = lock_context.job_id
+      @item = lock_context
+      @key = SidekiqUniqueJobs::Key.new(key)
+
+      lcfg = lock_context.config
+      @config = OpenStruct.new({
+        :"type" => lcfg[:strategy],
+        :"pttl" => lcfg[:ttl] * 1000,
+        :"timeout" => lcfg[:timeout],
+        :"wait_for_lock?" => lcfg[:ttl]&.positive?,
+        :"lock_info" => false,
+        :"limit" => lcfg[:limit],
+      })
+
+      @redis_pool  = redis_pool
+    end
+
+    def locked_jids
+      SidekiqUniqueJobs::Lock.new(@key).locked_jids
+    end
+
+    def swap_locks(old_jid)
+      olimit = lock_context.config[:limit]
+      new_jid = @job_id
+      return if old_jid == new_jid
+
+      # NB This is quite hacky, but should work
+      #
+      # Ideally the unlock(old) and lock(new) would be atomic, but that increases the amount of coupling with Sidekiq-Unique-Jobs - right now,
+      #   we're using fairly stable (though still internal) SUJ APIs; I fear that writing custom Lua will be significantly more brittle
+      #
+      # In the general case, we'd only bump limit by 1, but that leaves a potential race-condition when limit is configured > 1:
+      # (Assuming until_and_while_executing, reschedule, limit = 2):
+      #   (Workers are performing 2 Jobs, RUN lock count = 2)
+      #   Worker 1 pulls Job A
+      #   Worker 2 pulls Job B
+      #   W1 and W2 both fail to get the runtime lock
+      #   W1 and W2 call swap_locks
+      #   W1 calls lock(limit+1), lock is granted, lock count becomes limit+1
+      #   W2 calls lock(limit+1), lock is denied because count would be limit+2
+      #   W1 calls unlock(old_jid)
+
+      # Force creation of another lock, ignoring the limit
+      @config.limit = olimit + 100
+      result = lock
+
+      # Release the old lock, bringing us back within the limit
+      @job_id = old_jid
+      unlock
+
+      result
+    ensure
+      @config.limit = olimit
+      @job_id = new_jid
+    end
+
+    private
+
+    def lock_score
+      lock_context.job_score
+    end
+
+    def lock_info
+      @lock_info = JSON.dump(lock_context.debug_data)
+    end
+
+    # def taken?(conn)
+    #   v = conn.hexists(key.locked, job_id)
+    #   v.is_a?(Numeric) ? v != 0 : v
+    # end
+
+    def redis_version
+      @redis_version ||= CanvasSync::JobUniqueness.config.redis_version || SidekiqUniqueJobs.fetch_redis_version
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb

@@ -0,0 +1,32 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class Base
+      attr_reader :lock_context
+      attr_reader :redis_pool
+
+      class_attribute :_valid_for, instance_writer: false
+
+      def self.valid_for(*origins)
+        if origins.present?
+          orgins = Array(origins).map(&:to_sym)
+          self._valid_for = origins
+        else
+          self._valid_for || [:enqueue, :perform]
+        end
+      end
+
+      def initialize(lock_context, redis_pool = nil)
+        @lock_context = lock_context
+        @redis_pool = redis_pool
+      end
+
+      def call
+        raise NotImplementedError, "needs to be implemented in child class"
+      end
+
+      def replace?
+        is_a?(Replace)
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb

@@ -0,0 +1,13 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class Log < Base
+      valid_for :enqueue, :perform
+
+      def call
+        CanvasSync::JobUniqueness.logger.info(<<~MESSAGE.chomp)
+          Skipping job with id (#{lock_context.job_id}) because key: (#{lock_context.base_key}) is locked
+        MESSAGE
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb

@@ -0,0 +1,9 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class NullStrategy < Base
+      valid_for :enqueue, :perform
+
+      def call; end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb

@@ -0,0 +1,11 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class Raise < Base
+      valid_for :enqueue, :perform
+
+      def call
+        raise Conflict, lock_context
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb

@@ -0,0 +1,21 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class Reject < Base
+      valid_for :perform
+
+      def call
+        # TODO Allow this to work on Sidekiq-backed ActiveJob
+        unless lock_context.is_a?(CanvasSync::JobUniqueness::Compat::Sidekiq::SidekiqLockContext)
+          CanvasSync::JobUniqueness.logger.error(":reject conflict strategy is not supported for non-Sidekiq-backed jobs")
+          return
+        end
+
+        kwargs = {}
+        kwargs[:notify_failure] = false if Sidekiq::DeadSet.instance_method(:kill).arity > 1
+
+        sidekiq_message = lock_context.instance_variable_get(:@job_instance)
+        Sidekiq::DeadSet.new.kill(JSON.dump(sidekiq_message), **kwargs)
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb

@@ -0,0 +1,20 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    class Reschedule < OnConflict::Base
+      valid_for :perform
+
+      def call
+        Thread.current[:unique_jobs_previous_jid] = lock_context.job_id
+        rescheduled = lock_context.reenqueue(
+          schedule_in: schedule_in,
+        )
+      ensure
+        Thread.current[:unique_jobs_previous_jid] = nil
+      end
+
+      def schedule_in
+        lock_context.config[:reschedule_in] || 5
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_uniqueness/on_conflict.rb

@@ -0,0 +1,41 @@
+module CanvasSync::JobUniqueness
+  module OnConflict
+    extend ActiveSupport::Autoload
+
+    autoload :Base
+
+    autoload :Log
+    autoload :NullStrategy
+    autoload :Raise
+    autoload :Reject
+    # autoload :Replace
+    autoload :Reschedule
+
+    class << self
+      def lookup(strategy)
+        matching_strategy(strategy.to_s.camelize) ||
+          CanvasSync::JobUniqueness.config.conflict_strategies[strategy] ||
+          raise(StrategyNotFound, "on_conflict: #{strategy} is not found. Is it declared in the configuration?")
+      end
+
+      def validate!(on_conflict, lock_strategy)
+        on_conflict = { enqueue: on_conflict, perform: on_conflict } unless on_conflict.is_a?(Hash)
+
+        lock_strategy = Strategy.lookup(lock_strategy) if lock_strategy.is_a?(Symbol)
+        on_conflict.each do |origin, strategy|
+          strategy = OnConflict.lookup(strategy) if strategy.is_a?(Symbol)
+
+          if lock_strategy.locks_on.include?(origin) && !strategy.valid_for.include?(origin)
+            raise ArgumentError, "(#{origin.to_s.titleize}) conflict strategy #{strategy.name.underscore} is not valid for lock strategy #{lock_strategy.name.underscore}"
+          end
+        end
+      end
+
+      private
+
+      def matching_strategy(const)
+        const_get(const, false) if const_defined?(const, false)
+      end
+    end
+  end
+end