canvas_sync 0.21.1 → 0.22.0.beta1

data/lib/canvas_sync/job_batches/redis_script.rb

@@ -4,159 +4,157 @@ require 'erb'
 
 # Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
 
-module CanvasSync
-  module JobBatches
-    # {RedisScript} represents a lua script in the filesystem. It loads the script
-    # from disk and handles talking to redis to execute it. Error handling
-    # is handled by {LuaError}.
-    class RedisScript
-
-      # Loads the script file from disk and calculates its +SHA1+ sum.
-      #
-      # @param file [Pathname] the full path to the indicated file
-      def initialize(file)
-        @file = Pathname.new(file)
-      end
+module CanvasSync::JobBatches
+  # {RedisScript} represents a lua script in the filesystem. It loads the script
+  # from disk and handles talking to redis to execute it. Error handling
+  # is handled by {LuaError}.
+  class RedisScript
+
+    # Loads the script file from disk and calculates its +SHA1+ sum.
+    #
+    # @param file [Pathname] the full path to the indicated file
+    def initialize(file)
+      @file = Pathname.new(file)
+    end
 
-      # Passes the script and supplied arguments to redis for evaulation.
-      # It first attempts to use a script redis has already cached by using
-      # the +EVALSHA+ command, but falls back to providing the full script
-      # text via +EVAL+ if redis has not seen this script before. Future
-      # invocations will then use +EVALSHA+ without erroring.
-      #
-      # @param redis [Redis] the redis connection to run against
-      # @param args [*Objects] the arguments to the script
-      # @return [Object] the value passed back by redis after script execution
-      # @raise [LuaError] if the script failed to compile of encountered a
-      #   runtime error
-      def call(redis, *args)
-        t = Time.now
-        begin
-          redis.evalsha(digest, *args)
-        rescue => e
-          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
-        end
+    # Passes the script and supplied arguments to redis for evaulation.
+    # It first attempts to use a script redis has already cached by using
+    # the +EVALSHA+ command, but falls back to providing the full script
+    # text via +EVAL+ if redis has not seen this script before. Future
+    # invocations will then use +EVALSHA+ without erroring.
+    #
+    # @param redis [Redis] the redis connection to run against
+    # @param args [*Objects] the arguments to the script
+    # @return [Object] the value passed back by redis after script execution
+    # @raise [LuaError] if the script failed to compile of encountered a
+    #   runtime error
+    def call(redis, *args)
+      t = Time.now
+      begin
+        redis.evalsha(digest, *args)
       rescue => e
-        if LuaError.intercepts?(e)
-          raise LuaError.new(e, @file, content)
-        else
-          raise
-        end
+        e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
       end
-
-      def content
-        @content ||= load_lua(@file)
+    rescue => e
+      if LuaError.intercepts?(e)
+        raise LuaError.new(e, @file, content)
+      else
+        raise
       end
+    end
 
-      def digest
-        @digest ||= Digest::SHA1.hexdigest content
-      end
+    def content
+      @content ||= load_lua(@file)
+    end
 
-      private
+    def digest
+      @digest ||= Digest::SHA1.hexdigest content
+    end
 
-      def script_path
-        Rails.root + 'app/redis_lua'
-      end
+    private
 
-      def relative_path
-        @path ||= @file.relative_path_from(script_path)
-      end
+    def script_path
+      Rails.root + 'app/redis_lua'
+    end
 
-      def load_lua(file)
-        TemplateContext.new(script_path).template(script_path + file)
-      end
+    def relative_path
+      @path ||= @file.relative_path_from(script_path)
+    end
 
-      class TemplateContext
-        def initialize(script_path)
-          @script_path = script_path
-        end
+    def load_lua(file)
+      TemplateContext.new(script_path).template(script_path + file)
+    end
 
-        def template(pathname)
-          @partial_templates ||= {}
-          ERB.new(File.read(pathname)).result binding
-        end
+    class TemplateContext
+      def initialize(script_path)
+        @script_path = script_path
+      end
 
-        # helper method to include a lua partial within another lua script
-        #
-        # @param relative_path [String] the relative path to the script from
-        #     `script_path`
-        def include_partial(relative_path)
-          unless @partial_templates.has_key? relative_path
-            @partial_templates[relative_path] = nil
-            template( Pathname.new("#{@script_path}/#{relative_path}") )
-          end
-        end
+      def template(pathname)
+        @partial_templates ||= {}
+        ERB.new(File.read(pathname)).result binding
       end
 
-      # Reformats errors raised by redis representing failures while executing
-      # a lua script. The default errors have confusing messages and backtraces,
-      # and a type of +RuntimeError+. This class improves the message and
-      # modifies the backtrace to include the lua script itself in a reasonable
-      # way.
-      class LuaError < StandardError
-        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
-        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
-        CONTEXT_LINE_NUMBER = 2
-
-        attr_reader :error, :file, :content
-
-        # Is this error one that should be reformatted?
-        #
-        # @param error [StandardError] the original error raised by redis
-        # @return [Boolean] is this an error that should be reformatted?
-        def self.intercepts? error
-          error.message =~ PATTERN
+      # helper method to include a lua partial within another lua script
+      #
+      # @param relative_path [String] the relative path to the script from
+      #     `script_path`
+      def include_partial(relative_path)
+        unless @partial_templates.has_key? relative_path
+          @partial_templates[relative_path] = nil
+          template( Pathname.new("#{@script_path}/#{relative_path}") )
         end
+      end
+    end
 
-        # Initialize a new {LuaError} from an existing redis error, adjusting
-        # the message and backtrace in the process.
-        #
-        # @param error [StandardError] the original error raised by redis
-        # @param file [Pathname] full path to the lua file the error ocurred in
-        # @param content [String] lua file content the error ocurred in
-        def initialize error, file, content
-          @error = error
-          @file = file
-          @content = content
-
-          @error.message =~ PATTERN
-          _stage, line_number, message = $1, $2, $3
-          error_context = generate_error_context(content, line_number.to_i)
-
-          super "#{message}\n\n#{error_context}\n\n"
-          set_backtrace generate_backtrace file, line_number
-        end
+    # Reformats errors raised by redis representing failures while executing
+    # a lua script. The default errors have confusing messages and backtraces,
+    # and a type of +RuntimeError+. This class improves the message and
+    # modifies the backtrace to include the lua script itself in a reasonable
+    # way.
+    class LuaError < StandardError
+      PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+      WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+      CONTEXT_LINE_NUMBER = 2
 
-        private
+      attr_reader :error, :file, :content
 
-        def generate_error_context(content, line_number)
-          lines = content.lines.to_a
-          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
-          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
-          line_number_width = ending_line_number.to_s.length
+      # Is this error one that should be reformatted?
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @return [Boolean] is this an error that should be reformatted?
+      def self.intercepts? error
+        error.message =~ PATTERN
+      end
 
-          (beginning_line_number..ending_line_number).map do |number|
-            indicator = number == line_number ? '=>' : '  '
-            formatted_number = "%#{line_number_width}d" % number
-            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
-          end.join.chomp
-        end
+      # Initialize a new {LuaError} from an existing redis error, adjusting
+      # the message and backtrace in the process.
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @param file [Pathname] full path to the lua file the error ocurred in
+      # @param content [String] lua file content the error ocurred in
+      def initialize error, file, content
+        @error = error
+        @file = file
+        @content = content
+
+        @error.message =~ PATTERN
+        _stage, line_number, message = $1, $2, $3
+        error_context = generate_error_context(content, line_number.to_i)
+
+        super "#{message}\n\n#{error_context}\n\n"
+        set_backtrace generate_backtrace file, line_number
+      end
 
-        def generate_backtrace(file, line_number)
-          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
-          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
-          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
-          pre_wolverine.unshift("#{file}:#{line_number}")
-          pre_wolverine
-        end
+      private
 
-        def backtrace_before_entering_wolverine(backtrace)
-          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
-        end
+      def generate_error_context(content, line_number)
+        lines = content.lines.to_a
+        beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+        ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+        line_number_width = ending_line_number.to_s.length
+
+        (beginning_line_number..ending_line_number).map do |number|
+          indicator = number == line_number ? '=>' : '  '
+          formatted_number = "%#{line_number_width}d" % number
+          " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+        end.join.chomp
+      end
 
-        def line_from_wolverine(line)
-          line.split(':').first.include?(WOLVERINE_LIB_PATH)
-        end
+      def generate_backtrace(file, line_number)
+        pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+        index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+        pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+        pre_wolverine.unshift("#{file}:#{line_number}")
+        pre_wolverine
+      end
+
+      def backtrace_before_entering_wolverine(backtrace)
+        backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+      end
+
+      def line_from_wolverine(line)
+        line.split(':').first.include?(WOLVERINE_LIB_PATH)
       end
     end
   end

data/lib/canvas_sync/job_batches/status.rb

@@ -1,90 +1,88 @@
-module CanvasSync
-  module JobBatches
-    class Batch
-      class Status
-        attr_reader :bid
-
-        def initialize(bid)
-          bid = bid.bid if bid.is_a?(Batch)
-          @bid = bid
-        end
-
-        def join
-          raise "Not supported"
-        end
-
-        def pending
-          Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
-        end
-
-        def failures
-          Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
-        end
-
-        def dead
-          Batch.redis { |r| r.scard("BID-#{bid}-dead") }.to_i
-        end
-
-        def completed_count
-          job_count - pending
-        end
-
-        def job_count
-          Batch.redis { |r| r.hget("BID-#{bid}", "job_count") }.to_i
-        end
-
-        def created_at
-          Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
-        end
-
-        def parent_bid
-          Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
-        end
-
-        def failure_info
-          Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
-        end
-
-        def complete?
-          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
-        end
-
-        def success?
-          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
-        end
-
-        def child_count
-          Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
-        end
-
-        def completed_children_count
-          Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
-        end
-
-        def successful_children_count
-          Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
-        end
-
-        def failed_children_count
-          Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
-        end
-
-        def data
-          {
-            bid: bid,
-            failures: failures,
-            pending: pending,
-            created_at: created_at,
-            complete: complete?,
-            success: success?,
-            failure_info: failure_info,
-            parent_bid: parent_bid,
-            child_count: child_count,
-            completed_children_count: completed_children_count,
-            successful_children_count: successful_children_count,
-            failed_children_count: failed_children_count,
-          }
-        end
+module CanvasSync::JobBatches
+  class Batch
+    class Status
+      attr_reader :bid
+
+      def initialize(bid)
+        bid = bid.bid if bid.is_a?(Batch)
+        @bid = bid
+      end
+
+      def join
+        raise "Not supported"
+      end
+
+      def pending
+        Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+      end
+
+      def failures
+        Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+      end
+
+      def dead
+        Batch.redis { |r| r.scard("BID-#{bid}-dead") }.to_i
+      end
+
+      def completed_count
+        job_count - pending
+      end
+
+      def job_count
+        Batch.redis { |r| r.hget("BID-#{bid}", "job_count") }.to_i
+      end
+
+      def created_at
+        Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+      end
+
+      def parent_bid
+        Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+      end
+
+      def failure_info
+        Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
+      end
+
+      def complete?
+        'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
+      end
+
+      def success?
+        'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
+      end
+
+      def child_count
+        Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+      end
+
+      def completed_children_count
+        Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
+      end
+
+      def successful_children_count
+        Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
+      end
+
+      def failed_children_count
+        Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
+      end
+
+      def data
+        {
+          bid: bid,
+          failures: failures,
+          pending: pending,
+          created_at: created_at,
+          complete: complete?,
+          success: success?,
+          failure_info: failure_info,
+          parent_bid: parent_bid,
+          child_count: child_count,
+          completed_children_count: completed_children_count,
+          successful_children_count: successful_children_count,
+          failed_children_count: failed_children_count,
+        }
       end
     end
   end

data/lib/canvas_sync/job_uniqueness/compat/active_job.rb

@@ -0,0 +1,75 @@
+
+module CanvasSync::JobUniqueness
+  module Compat
+    module ActiveJob
+
+      class ActiveJobLockContext < LockContext
+        def job_scheduled_at
+          job_instance&.scheduled_at
+        end
+
+        def reenqueue(schedule_in:)
+          job_class.set(
+            queue: job_queue.to_sym,
+            set: schedule_in,
+            priortity: job_instance.priority,
+          ).perform_later(*job_instance.arguments)
+        end
+      end
+
+      module UniqueJobExtension
+        extend ActiveSupport::Concern
+
+        included do
+          set_callback(:enqueue, :around, prepend: true) do |job, block|
+            ctx = uniqueness_lock_context
+            @uniqueness_cache_data = ctx.cache_data
+            ctx.handle_lifecycle!(:enqueue, &block)
+          end
+
+          around_perform do |job, block|
+            ctx = uniqueness_lock_context
+            ctx.handle_lifecycle!(:perform, &block)
+          end
+        end
+
+        def serialize
+          super.tap do |data|
+            data['uniqueness_cache_data'] = @uniqueness_cache_data.stringify_keys
+          end
+        end
+
+        def deserialize(data)
+          super
+          @uniqueness_cache_data = data['uniqueness_cache_data'].symbolize_keys
+        end
+
+        def uniqueness_lock_context
+          ActiveJobLockContext.new({
+            job_clazz: self.class,
+            jid: self.job_id,
+            args: self.arguments,
+            queue: self.queue_name,
+            **(@uniqueness_cache_data || {})
+          }, job_instance: self)
+        end
+      end
+
+      module JobExtension
+        extend ActiveSupport::Concern
+        include UniqueJobCommon
+
+        class_methods do
+          def ensure_uniqueness(**kwargs)
+            super(**kwargs)
+            include UniqueJobExtension
+          end
+        end
+      end
+
+      def self.configure
+        ::ActiveJob::Base.include JobExtension
+      end
+    end
+  end
+end