canvas_sync 0.23.5 → 0.24.0.beta1

data/lib/canvas_sync/job_batches/redis_script.rb

@@ -1,161 +0,0 @@
-require 'pathname'
-require 'digest/sha1'
-require 'erb'
-
-# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
-
-module CanvasSync::JobBatches
-  # {RedisScript} represents a lua script in the filesystem. It loads the script
-  # from disk and handles talking to redis to execute it. Error handling
-  # is handled by {LuaError}.
-  class RedisScript
-
-    # Loads the script file from disk and calculates its +SHA1+ sum.
-    #
-    # @param file [Pathname] the full path to the indicated file
-    def initialize(file)
-      @file = Pathname.new(file)
-    end
-
-    # Passes the script and supplied arguments to redis for evaulation.
-    # It first attempts to use a script redis has already cached by using
-    # the +EVALSHA+ command, but falls back to providing the full script
-    # text via +EVAL+ if redis has not seen this script before. Future
-    # invocations will then use +EVALSHA+ without erroring.
-    #
-    # @param redis [Redis] the redis connection to run against
-    # @param args [*Objects] the arguments to the script
-    # @return [Object] the value passed back by redis after script execution
-    # @raise [LuaError] if the script failed to compile of encountered a
-    #   runtime error
-    def call(redis, *args)
-      t = Time.now
-      begin
-        redis.evalsha(digest, *args)
-      rescue => e
-        e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
-      end
-    rescue => e
-      if LuaError.intercepts?(e)
-        raise LuaError.new(e, @file, content)
-      else
-        raise
-      end
-    end
-
-    def content
-      @content ||= load_lua(@file)
-    end
-
-    def digest
-      @digest ||= Digest::SHA1.hexdigest content
-    end
-
-    private
-
-    def script_path
-      Rails.root + 'app/redis_lua'
-    end
-
-    def relative_path
-      @path ||= @file.relative_path_from(script_path)
-    end
-
-    def load_lua(file)
-      TemplateContext.new(script_path).template(script_path + file)
-    end
-
-    class TemplateContext
-      def initialize(script_path)
-        @script_path = script_path
-      end
-
-      def template(pathname)
-        @partial_templates ||= {}
-        ERB.new(File.read(pathname)).result binding
-      end
-
-      # helper method to include a lua partial within another lua script
-      #
-      # @param relative_path [String] the relative path to the script from
-      #     `script_path`
-      def include_partial(relative_path)
-        unless @partial_templates.has_key? relative_path
-          @partial_templates[relative_path] = nil
-          template( Pathname.new("#{@script_path}/#{relative_path}") )
-        end
-      end
-    end
-
-    # Reformats errors raised by redis representing failures while executing
-    # a lua script. The default errors have confusing messages and backtraces,
-    # and a type of +RuntimeError+. This class improves the message and
-    # modifies the backtrace to include the lua script itself in a reasonable
-    # way.
-    class LuaError < StandardError
-      PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
-      WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
-      CONTEXT_LINE_NUMBER = 2
-
-      attr_reader :error, :file, :content
-
-      # Is this error one that should be reformatted?
-      #
-      # @param error [StandardError] the original error raised by redis
-      # @return [Boolean] is this an error that should be reformatted?
-      def self.intercepts? error
-        error.message =~ PATTERN
-      end
-
-      # Initialize a new {LuaError} from an existing redis error, adjusting
-      # the message and backtrace in the process.
-      #
-      # @param error [StandardError] the original error raised by redis
-      # @param file [Pathname] full path to the lua file the error ocurred in
-      # @param content [String] lua file content the error ocurred in
-      def initialize error, file, content
-        @error = error
-        @file = file
-        @content = content
-
-        @error.message =~ PATTERN
-        _stage, line_number, message = $1, $2, $3
-        error_context = generate_error_context(content, line_number.to_i)
-
-        super "#{message}\n\n#{error_context}\n\n"
-        set_backtrace generate_backtrace file, line_number
-      end
-
-      private
-
-      def generate_error_context(content, line_number)
-        lines = content.lines.to_a
-        beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
-        ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
-        line_number_width = ending_line_number.to_s.length
-
-        (beginning_line_number..ending_line_number).map do |number|
-          indicator = number == line_number ? '=>' : '  '
-          formatted_number = "%#{line_number_width}d" % number
-          " #{indicator} #{formatted_number}: #{lines[number - 1]}"
-        end.join.chomp
-      end
-
-      def generate_backtrace(file, line_number)
-        pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
-        index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
-        pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
-        pre_wolverine.unshift("#{file}:#{line_number}")
-        pre_wolverine
-      end
-
-      def backtrace_before_entering_wolverine(backtrace)
-        backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
-      end
-
-      def line_from_wolverine(line)
-        line.split(':').first.include?(WOLVERINE_LIB_PATH)
-      end
-    end
-  end
-end

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -1,14 +0,0 @@
-
-local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
-redis.call('HSET', KEYS[1], ARGV[1], 'true')
-
-if previously_scheduled ~= 'true' then
-    local pcb_key = KEYS[1] .. '-pending_callbacks'
-    redis.call('SADD', pcb_key, ARGV[1] .. '-finalize')
-    if ARGV[2] == 'true' then
-        redis.call('SADD', pcb_key, ARGV[1])
-    end
-    redis.call('EXPIRE', pcb_key, ARGV[3])
-end
-
-return previously_scheduled

data/lib/canvas_sync/job_batches/status.rb

@@ -1,89 +0,0 @@
-module CanvasSync::JobBatches
-  class Batch
-    class Status
-      attr_reader :bid
-
-      def initialize(bid)
-        bid = bid.bid if bid.is_a?(Batch)
-        @bid = bid
-      end
-
-      def join
-        raise "Not supported"
-      end
-
-      def pending
-        Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
-      end
-
-      def failures
-        Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
-      end
-
-      def dead
-        Batch.redis { |r| r.scard("BID-#{bid}-dead") }.to_i
-      end
-
-      def completed_count
-        job_count - pending
-      end
-
-      def job_count
-        Batch.redis { |r| r.hget("BID-#{bid}", "job_count") }.to_i
-      end
-
-      def created_at
-        Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
-      end
-
-      def parent_bid
-        Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
-      end
-
-      def failure_info
-        Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
-      end
-
-      def complete?
-        'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
-      end
-
-      def success?
-        'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
-      end
-
-      def child_count
-        Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
-      end
-
-      def completed_children_count
-        Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
-      end
-
-      def successful_children_count
-        Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
-      end
-
-      def failed_children_count
-        Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
-      end
-
-      def data
-        {
-          bid: bid,
-          failures: failures,
-          pending: pending,
-          created_at: created_at,
-          complete: complete?,
-          success: success?,
-          failure_info: failure_info,
-          parent_bid: parent_bid,
-          child_count: child_count,
-          completed_children_count: completed_children_count,
-          successful_children_count: successful_children_count,
-          failed_children_count: failed_children_count,
-        }
-      end
-    end
-  end
-end

data/lib/canvas_sync/job_uniqueness/compat/active_job.rb

@@ -1,75 +0,0 @@
-
-module CanvasSync::JobUniqueness
-  module Compat
-    module ActiveJob
-
-      class ActiveJobLockContext < LockContext
-        def job_scheduled_at
-          job_instance&.scheduled_at
-        end
-
-        def reenqueue(schedule_in:)
-          job_class.set(
-            queue: job_queue.to_sym,
-            wait: schedule_in,
-            priortity: job_instance.priority,
-          ).perform_later(*job_instance.arguments)
-        end
-      end
-
-      module UniqueJobExtension
-        extend ActiveSupport::Concern
-
-        included do
-          set_callback(:enqueue, :around, prepend: true) do |job, block|
-            ctx = uniqueness_lock_context
-            @uniqueness_cache_data = ctx.cache_data
-            ctx.handle_lifecycle!(:enqueue, &block)
-          end
-
-          around_perform do |job, block|
-            ctx = uniqueness_lock_context
-            ctx.handle_lifecycle!(:perform, &block)
-          end
-        end
-
-        def serialize
-          super.tap do |data|
-            data['uniqueness_cache_data'] = @uniqueness_cache_data.stringify_keys
-          end
-        end
-
-        def deserialize(data)
-          super
-          @uniqueness_cache_data = data['uniqueness_cache_data']&.symbolize_keys
-        end
-
-        def uniqueness_lock_context
-          ActiveJobLockContext.new({
-            job_clazz: self.class,
-            jid: self.job_id,
-            args: self.arguments,
-            queue: self.queue_name,
-            **(@uniqueness_cache_data || {})
-          }, job_instance: self)
-        end
-      end
-
-      module JobExtension
-        extend ActiveSupport::Concern
-        include UniqueJobCommon
-
-        class_methods do
-          def ensure_uniqueness(**kwargs)
-            super(**kwargs)
-            include UniqueJobExtension
-          end
-        end
-      end
-
-      def self.configure
-        ::ActiveJob::Base.include JobExtension
-      end
-    end
-  end
-end

data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb

@@ -1,135 +0,0 @@
-
-module CanvasSync::JobUniqueness
-  module Compat
-    module Sidekiq
-      module WorkerExtension
-        extend ActiveSupport::Concern
-        include UniqueJobCommon
-
-        class_methods do
-          def ensure_uniqueness(**kwargs)
-            super(**kwargs)
-            if !(defined?(@@validated_config) && @@validated_config)
-              Compat::Sidekiq.validate_middleware_placement!()
-              @@validated_config = true
-            end
-          end
-        end
-      end
-
-      class SidekiqLockContext < LockContext
-        def job_scheduled_at
-          @job_instance&.[]("at")
-        end
-
-        def reenqueue(schedule_in:)
-          job_class.set(queue: job_queue.to_sym).perform_in(schedule_in, *@job_instance["args"])
-        end
-      end
-
-      class CommonMiddleware
-        def lock_context(msg)
-          opts = worker_uniqueness(msg)
-          return nil unless opts
-
-          SidekiqLockContext.new({
-            job_clazz: msg['class'],
-            jid: msg['jid'],
-            queue: msg['queue'],
-            args: msg['args'],
-            # kwargs: msg['kwargs'],
-            **(msg['uniqueness_cache_data']&.symbolize_keys || {}),
-          }, job_instance: msg)
-        end
-
-        def worker_uniqueness(msg)
-          return nil if Compat::Sidekiq.is_activejob_job?(msg)
-
-          worker_class = msg['class'].constantize
-          return nil unless worker_class.respond_to?(:unique_job_options)
-
-          worker_class.unique_job_options
-        end
-      end
-
-      class ClientMiddleware < CommonMiddleware
-        include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
-
-        def call(_worker, msg, _queue, _redis_pool = nil, &blk)
-          ctx = lock_context(msg)
-          return blk.call unless ctx
-          msg['uniqueness_cache_data'] = ctx.cache_data.stringify_keys
-          ctx.handle_lifecycle!(:enqueue, &blk)
-        end
-      end
-
-      class ServerMiddleware < CommonMiddleware
-        include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
-
-        def call(_worker, msg, _queue, &blk)
-          ctx = lock_context(msg)
-          return blk.call unless ctx
-          ctx.handle_lifecycle!(:perform, &blk)
-        end
-      end
-
-      def self.is_activejob_job?(msg)
-        return false unless defined?(::ActiveJob)
-
-        msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < Compat::ActiveJob::UniqueJobExtension
-      end
-
-      def self.validate_middleware_order(chain, order)
-        chain_classes = chain.entries.map(&:klass)
-        filtered = chain_classes.select { |klass| order.include?(klass) }
-        raise "Middleware chain does not contain all required middleware: #{order - filtered}" unless order.all? { |klass| filtered.include?(klass) }
-        raise "Middleware must be in order: #{order.inspect}" if filtered != order
-      end
-
-      def self.sidekiq_middleware(placement, &blk)
-        install_middleware = ->(config) do
-          config.send("#{placement}_middleware") do |chain|
-            blk.call(chain)
-          end
-        end
-
-        ::Sidekiq.configure_client(&install_middleware) if placement == :client
-        ::Sidekiq.configure_server(&install_middleware)
-      end
-
-      def self.validate_middleware_placement!
-        sidekiq_middleware(:client) do |chain|
-          # Unique middleware must come _before_ the Batch middleware so that the uniqueness middleware can wrap the job in a batch
-          validate_middleware_order(chain, [
-            CanvasSync::JobUniqueness::Compat::Sidekiq::ClientMiddleware,
-            CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware,
-          ])
-        end
-
-        sidekiq_middleware(:server) do |chain|
-          # Unique middleware must com _after_ the Batch middleware so that the Batch is loaded before reaching the uniqueness middleware
-          validate_middleware_order(chain, [
-            CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware,
-            CanvasSync::JobUniqueness::Compat::Sidekiq::ServerMiddleware,
-          ])
-        end
-      end
-
-      def self.configure
-        sidekiq_middleware(:client) do |chain|
-          chain.insert_before CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware, Compat::Sidekiq::ClientMiddleware
-        end
-
-        sidekiq_middleware(:server) do |chain|
-          chain.insert_after CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware, Compat::Sidekiq::ServerMiddleware
-        end
-
-        ::Sidekiq::Worker.extend(ActiveSupport::Concern) unless ::Sidekiq::Worker < ActiveSupport::Concern
-
-        ::Sidekiq::Worker.send(:include, Compat::Sidekiq::WorkerExtension)
-      end
-    end
-  end
-end
-
-# require_relative 'sidekiq/web'

data/lib/canvas_sync/job_uniqueness/compat.rb

@@ -1,20 +0,0 @@
-
-module CanvasSync::JobUniqueness
-  module Compat
-    def self.load_compat(name)
-      name = name.to_s
-      begin
-        require name
-      rescue LoadError
-      end
-
-      if name.classify.safe_constantize
-        require_relative "./compat/#{name}"
-        "#{self.name}::#{name.classify}".constantize.configure
-      end
-    end
-
-    load_compat(:active_job)
-    load_compat(:sidekiq)
-  end
-end

data/lib/canvas_sync/job_uniqueness/configuration.rb

@@ -1,25 +0,0 @@
-module CanvasSync::JobUniqueness
-  class Configuration
-    include ActiveSupport::Configurable
-
-    config_accessor(:lock_ttl) { 14.days.to_i }
-    config_accessor(:lock_prefix) { 'uniquejob' }
-    config_accessor(:on_conflict) { :raise }
-
-    config_accessor(:lock_strategies) { {} }
-    config_accessor(:conflict_strategies) { {} }
-
-    config_accessor(:redis_version) { nil }
-    def on_conflict=(action)
-      validate_on_conflict_action!(action)
-
-      config.on_conflict = action
-    end
-
-    def validate_on_conflict_action!(action)
-      return if action.nil? || %i[log raise].include?(action) || action.respond_to?(:call)
-
-      raise ActiveJob::Uniqueness::InvalidOnConflictAction, "Unexpected '#{action}' action on conflict"
-    end
-  end
-end

data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb

@@ -1,47 +0,0 @@
-
-require 'canvas_sync/job_uniqueness/lock_context'
-require 'canvas_sync/job_uniqueness/unique_job_common'
-
-require 'canvas_sync/job_uniqueness/strategy'
-require 'canvas_sync/job_uniqueness/on_conflict'
-require 'canvas_sync/job_uniqueness/compat'
-
-module CanvasSync::JobUniqueness
-  extend ActiveSupport::Autoload
-
-  autoload :Locksmith
-  autoload :Configuration
-
-  class Conflict < StandardError
-    attr_reader :lock_context
-
-    def initialize(lock_context)
-      super()
-      @lock_context = lock_context
-    end
-  end
-
-  class CouldNotLockError < StandardError
-    attr_reader :lock_context, :source
-
-    def initialize(lock_context, source:)
-      super()
-      @lock_context = lock_context
-      @source = source
-    end
-  end
-
-  class << self
-    def configure
-      yield config
-    end
-
-    def config
-      @config ||= Configuration.new
-    end
-
-    def logger
-      CanvasSync.logger
-    end
-  end
-end