canvas_sync 0.21.1 → 0.22.0.beta1

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -1,159 +1,157 @@
-module CanvasSync
-  module JobBatches
-    class ContextHash
-      delegate_missing_to :flatten
-
-      def initialize(bid, hash = nil)
-        @bid_stack = [bid]
-        @hash_map = {}
-        @dirty = false
-        @flattened = nil
-        @hash_map[bid] = hash.with_indifferent_access if hash
-      end
-
-      # Local is "the nearest batch with a context value"
-      # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
-      # that can be accessed transparently from one of it's internal, context-less Batches
-      def local_bid
-        bid = @bid_stack[-1]
-        while bid.present?
-          bhash = resolve_hash(bid)
-          return bid if bhash
-          bid = get_parent_bid(bid)
-        end
-        nil
-      end
+module CanvasSync::JobBatches
+  class ContextHash
+    delegate_missing_to :flatten
+
+    def initialize(bid, hash = nil)
+      @bid_stack = [bid]
+      @hash_map = {}
+      @dirty = false
+      @flattened = nil
+      @hash_map[bid] = hash.with_indifferent_access if hash
+    end
 
-      def local
-        @hash_map[local_bid]
-      end
+    # Local is "the nearest batch with a context value"
+    # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
+    # that can be accessed transparently from one of it's internal, context-less Batches
+    def local_bid
+      bid = @bid_stack[-1]
+      while bid.present?
+        bhash = resolve_hash(bid)
+        return bid if bhash
+        bid = get_parent_bid(bid)
+      end
+      nil
+    end
 
-      def own
-        resolve_hash(@bid_stack[-1]) || {}
-      end
+    def local
+      @hash_map[local_bid]
+    end
 
-      def set_local(new_hash)
-        @dirty = true
-        local.clear.merge!(new_hash)
-      end
+    def own
+      resolve_hash(@bid_stack[-1]) || {}
+    end
 
-      def clear
-        local.clear
-        @flattened = nil
-        @dirty = true
-        self
-      end
+    def set_local(new_hash)
+      @dirty = true
+      local.clear.merge!(new_hash)
+    end
 
-      def []=(key, value)
-        @flattened = nil
-        @dirty = true
-        local[key] = value
-      end
+    def clear
+      local.clear
+      @flattened = nil
+      @dirty = true
+      self
+    end
 
-      def [](key)
-        bid = @bid_stack[-1]
-        while bid.present?
-          bhash = resolve_hash(bid)
-          return bhash[key] if bhash&.key?(key)
-          bid = get_parent_bid(bid)
-        end
-        nil
-      end
+    def []=(key, value)
+      @flattened = nil
+      @dirty = true
+      local[key] = value
+    end
 
-      def reload!
-        @dirty = false
-        @hash_map = {}
-        self
+    def [](key)
+      bid = @bid_stack[-1]
+      while bid.present?
+        bhash = resolve_hash(bid)
+        return bhash[key] if bhash&.key?(key)
+        bid = get_parent_bid(bid)
       end
+      nil
+    end
 
-      def save!(force: false)
-        return unless dirty? || force
-        Batch.redis do |r|
-          r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
-        end
-      end
+    def reload!
+      @dirty = false
+      @hash_map = {}
+      self
+    end
 
-      def dirty?
-        @dirty
+    def save!(force: false)
+      return unless dirty? || force
+      Batch.redis do |r|
+        r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
       end
+    end
 
-      def is_a?(arg)
-        return true if Hash <= arg
-        super
-      end
+    def dirty?
+      @dirty
+    end
 
-      def flatten
-        return @flattened if @flattened
+    def is_a?(arg)
+      return true if Hash <= arg
+      super
+    end
 
-        load_all
-        flattened = {}
-        @bid_stack.compact.each do |bid|
-          flattened.merge!(@hash_map[bid]) if @hash_map[bid]
-        end
-        flattened.freeze
+    def flatten
+      return @flattened if @flattened
 
-        @flattened = flattened.with_indifferent_access
+      load_all
+      flattened = {}
+      @bid_stack.compact.each do |bid|
+        flattened.merge!(@hash_map[bid]) if @hash_map[bid]
       end
+      flattened.freeze
 
-      def to_h
-        flatten
-      end
+      @flattened = flattened.with_indifferent_access
+    end
 
-      private
+    def to_h
+      flatten
+    end
 
-      def get_parent_hash(bid)
-        resolve_hash(get_parent_bid(bid)).freeze
-      end
+    private
 
-      def get_parent_bid(bid)
-        index = @bid_stack.index(bid)
-        raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
-
-        index -= 1
-        if index >= 0
-          @bid_stack[index]
-        else
-          pbid = Batch.redis do |r|
-            callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
-            callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
-          end
-          @bid_stack.unshift(pbid)
-          pbid
+    def get_parent_hash(bid)
+      resolve_hash(get_parent_bid(bid)).freeze
+    end
+
+    def get_parent_bid(bid)
+      index = @bid_stack.index(bid)
+      raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
+
+      index -= 1
+      if index >= 0
+        @bid_stack[index]
+      else
+        pbid = Batch.redis do |r|
+          callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
+          callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
         end
+        @bid_stack.unshift(pbid)
+        pbid
       end
+    end
 
-      def resolve_hash(bid)
-        return nil unless bid.present?
-        return @hash_map[bid] if @hash_map.key?(bid)
+    def resolve_hash(bid)
+      return nil unless bid.present?
+      return @hash_map[bid] if @hash_map.key?(bid)
 
-        context_json, editable = Batch.redis do |r|
-          r.multi do |r|
-            r.hget("BID-#{bid}", "context")
-            r.hget("BID-#{bid}", "allow_context_changes")
-          end
+      context_json, editable = Batch.redis do |r|
+        r.multi do |r|
+          r.hget("BID-#{bid}", "context")
+          r.hget("BID-#{bid}", "allow_context_changes")
         end
+      end
 
-        if context_json.present?
-          context_hash = JSON.parse(context_json)
-          context_hash = context_hash.with_indifferent_access
-          context_hash.each do |k, v|
-            v.freeze
-          end
-          context_hash.freeze unless editable
-
-          @hash_map[bid] = context_hash
-        else
-          @hash_map[bid] = nil
+      if context_json.present?
+        context_hash = JSON.parse(context_json)
+        context_hash = context_hash.with_indifferent_access
+        context_hash.each do |k, v|
+          v.freeze
         end
+        context_hash.freeze unless editable
+
+        @hash_map[bid] = context_hash
+      else
+        @hash_map[bid] = nil
       end
+    end
 
-      def load_all
-        resolve_hash(@bid_stack[0]).freeze
-        while @bid_stack[0].present?
-          get_parent_hash(@bid_stack[0])
-        end
-        @hash_map
+    def load_all
+      resolve_hash(@bid_stack[0]).freeze
+      while @bid_stack[0].present?
+        get_parent_hash(@bid_stack[0])
       end
+      @hash_map
     end
   end
 end

data/lib/canvas_sync/job_batches/jobs/base_job.rb

@@ -1,7 +1,5 @@
-module CanvasSync
-  module JobBatches
-    class BaseJob < ::ActiveJob::Base
+module CanvasSync::JobBatches
+  class BaseJob < ::ActiveJob::Base
 
-    end
   end
 end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -1,22 +1,20 @@
 require_relative './base_job'
 
-module CanvasSync
-  module JobBatches
-    class ConcurrentBatchJob < BaseJob
-      def self.make_batch(sub_jobs, **kwargs, &blk)
-        ManagedBatchJob.make_batch(
-          sub_jobs,
-          **kwargs,
-          ordered: false,
-          concurrency: true,
-          desc_prefix: 'ConcurrentBatchJob: ',
-          &blk
-        )
-      end
+module CanvasSync::JobBatches
+  class ConcurrentBatchJob < BaseJob
+    def self.make_batch(sub_jobs, **kwargs, &blk)
+      ManagedBatchJob.make_batch(
+        sub_jobs,
+        **kwargs,
+        ordered: false,
+        concurrency: true,
+        desc_prefix: 'ConcurrentBatchJob: ',
+        &blk
+      )
+    end
 
-      def perform(sub_jobs, **kwargs)
-        self.class.make_batch(sub_jobs, **kwargs)
-      end
+    def perform(sub_jobs, **kwargs)
+      self.class.make_batch(sub_jobs, **kwargs)
     end
   end
 end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -1,170 +1,168 @@
 require_relative './base_job'
 
-module CanvasSync
-  module JobBatches
-    class ManagedBatchJob < BaseJob
-      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, preflight_check: nil, desc_prefix: nil, &blk)
-        desc_prefix ||= ''
-
-        if concurrency == 0 || concurrency == nil || concurrency == true
-          concurrency = sub_jobs.count
-        elsif concurrency == false
-          concurrency = 1
-        end
+module CanvasSync::JobBatches
+  class ManagedBatchJob < BaseJob
+    def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, preflight_check: nil, desc_prefix: nil, &blk)
+      desc_prefix ||= ''
+
+      if concurrency == 0 || concurrency == nil || concurrency == true
+        concurrency = sub_jobs.count
+      elsif concurrency == false
+        concurrency = 1
+      end
+
+      root_batch = Batch.new
+      man_batch_id = nil
+
+      if concurrency < sub_jobs.count
+        man_batch_id = SecureRandom.urlsafe_base64(10)
 
-        root_batch = Batch.new
-        man_batch_id = nil
-
-        if concurrency < sub_jobs.count
-          man_batch_id = SecureRandom.urlsafe_base64(10)
-
-          Batch.redis do |r|
-            r.multi do |r|
-              r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
-              r.hset("MNGBID-#{man_batch_id}", "ordered", ordered ? 1 : 0)
-              r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
-              r.hset("MNGBID-#{man_batch_id}", "preflight_check", preflight_check) if preflight_check.present?
-              r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
-
-              mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
-                j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
-                j = ::ActiveJob::Arguments.serialize([j])
-                JSON.unparse(j)
-              end
-              if ordered
-                r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-              else
-                r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-              end
-              r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
+        Batch.redis do |r|
+          r.multi do |r|
+            r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
+            r.hset("MNGBID-#{man_batch_id}", "ordered", ordered ? 1 : 0)
+            r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+            r.hset("MNGBID-#{man_batch_id}", "preflight_check", preflight_check) if preflight_check.present?
+            r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
+
+            mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
+              j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
+              j = ::ActiveJob::Arguments.serialize([j])
+              JSON.unparse(j)
             end
+            if ordered
+              r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+            else
+              r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+            end
+            r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
           end
+        end
 
-          root_batch.allow_context_changes = (concurrency == 1)
-          root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
+        root_batch.allow_context_changes = (concurrency == 1)
+        root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
 
-          desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
-        end
+        desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
+      end
 
-        root_batch.context = context
+      root_batch.context = context
 
-        blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
+      blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
 
-        root_batch.description = "#{desc_prefix}#{root_batch.description || 'Root'}"
+      root_batch.description = "#{desc_prefix}#{root_batch.description || 'Root'}"
 
-        root_batch.context["managed_batch_bid"] = man_batch_id if man_batch_id
+      root_batch.context["managed_batch_bid"] = man_batch_id if man_batch_id
 
-        if concurrency < sub_jobs.count
-          root_batch.jobs {}
-          concurrency.times do
-            perform_next_sequence_job(man_batch_id, skip_preflight: true)
-          end
-        else
-          root_batch.jobs do
-            sub_jobs.each do |j|
-              ChainBuilder.enqueue_job(j)
-            end
+      if concurrency < sub_jobs.count
+        root_batch.jobs {}
+        concurrency.times do
+          perform_next_sequence_job(man_batch_id, skip_preflight: true)
+        end
+      else
+        root_batch.jobs do
+          sub_jobs.each do |j|
+            ChainBuilder.enqueue_job(j)
           end
         end
-
-        root_batch
       end
 
-      def perform(sub_jobs, **kwargs)
-        self.class.make_batch(sub_jobs, **kwargs)
-      end
+      root_batch
+    end
 
-      def self.cleanup_redis(status, options)
-        man_batch_id = options['managed_batch_id']
-        Batch.redis do |r|
-          r.del(
-            "MNGBID-#{man_batch_id}",
-            "MNGBID-#{man_batch_id}-jobs",
-          )
-        end
-      end
+    def perform(sub_jobs, **kwargs)
+      self.class.make_batch(sub_jobs, **kwargs)
+    end
 
-      def self.job_succeeded_callback(status, options)
-        man_batch_id = options['managed_batch_id']
-        perform_next_sequence_job(man_batch_id)
+    def self.cleanup_redis(status, options)
+      man_batch_id = options['managed_batch_id']
+      Batch.redis do |r|
+        r.del(
+          "MNGBID-#{man_batch_id}",
+          "MNGBID-#{man_batch_id}-jobs",
+        )
       end
+    end
 
-      protected
+    def self.job_succeeded_callback(status, options)
+      man_batch_id = options['managed_batch_id']
+      perform_next_sequence_job(man_batch_id)
+    end
 
-      def self.perform_next_sequence_job(man_batch_id, skip_preflight: false)
-        root_bid, ordered, preflight_check = Batch.redis do |r|
-          r.multi do |r|
-            r.hget("MNGBID-#{man_batch_id}", "root_bid")
-            r.hget("MNGBID-#{man_batch_id}", "ordered")
-            r.hget("MNGBID-#{man_batch_id}", "preflight_check")
-          end
+    protected
+
+    def self.perform_next_sequence_job(man_batch_id, skip_preflight: false)
+      root_bid, ordered, preflight_check = Batch.redis do |r|
+        r.multi do |r|
+          r.hget("MNGBID-#{man_batch_id}", "root_bid")
+          r.hget("MNGBID-#{man_batch_id}", "ordered")
+          r.hget("MNGBID-#{man_batch_id}", "preflight_check")
         end
+      end
 
-        if !skip_preflight && preflight_check.present?
-          if preflight_check.include?(".")
-            clazz, method_name = preflight_check.split('.')
-            clazz = clazz.constantize
-          else
-            clazz = Object
-            method_name = preflight_check
-          end
-          preflight_check = ->(*args) { clazz.send(method_name, *args) }
+      if !skip_preflight && preflight_check.present?
+        if preflight_check.include?(".")
+          clazz, method_name = preflight_check.split('.')
+          clazz = clazz.constantize
         else
-          preflight_check = ->(*args) { true }
+          clazz = Object
+          method_name = preflight_check
         end
+        preflight_check = ->(*args) { clazz.send(method_name, *args) }
+      else
+        preflight_check = ->(*args) { true }
+      end
 
-        ordered = CanvasSync::MiscHelper.to_boolean(ordered)
+      ordered = CanvasSync::MiscHelper.to_boolean(ordered)
 
-        loop do
-          next_job_json = Batch.redis do |r|
-            if ordered
-              r.lpop("MNGBID-#{man_batch_id}-jobs")
-            else
-              r.spop("MNGBID-#{man_batch_id}-jobs")
-            end
+      loop do
+        next_job_json = Batch.redis do |r|
+          if ordered
+            r.lpop("MNGBID-#{man_batch_id}-jobs")
+          else
+            r.spop("MNGBID-#{man_batch_id}-jobs")
           end
+        end
 
-          break unless next_job_json.present?
+        break unless next_job_json.present?
 
-          next_job = JSON.parse(next_job_json)
-          next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
+        next_job = JSON.parse(next_job_json)
+        next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
 
-          preflight_result = preflight_check.call(next_job)
-          if preflight_result == :abort
-            cleanup_redis(nil, { "managed_batch_id" => man_batch_id })
-            break
-          elsif !preflight_check
-            next
-          end
+        preflight_result = preflight_check.call(next_job)
+        if preflight_result == :abort
+          cleanup_redis(nil, { "managed_batch_id" => man_batch_id })
+          break
+        elsif !preflight_check
+          next
+        end
 
-          Batch.new(root_bid).jobs do
-            Batch.new.tap do |batch|
-              batch.description = "Managed Batch Fiber (#{man_batch_id})"
-              batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
-              batch.jobs do
-                ChainBuilder.enqueue_job(next_job)
-              end
+        Batch.new(root_bid).jobs do
+          Batch.new.tap do |batch|
+            batch.description = "Managed Batch Fiber (#{man_batch_id})"
+            batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
+            batch.jobs do
+              ChainBuilder.enqueue_job(next_job)
             end
           end
-
-          break
         end
-      end
 
-      class ManagedBatchProxy
-        def initialize(real_batch)
-          @real_batch = real_batch
-        end
+        break
+      end
+    end
 
-        delegate_missing_to :real_batch
+    class ManagedBatchProxy
+      def initialize(real_batch)
+        @real_batch = real_batch
+      end
 
-        def jobs
-          raise "Managed Batches do not support calling .jobs directly!"
-        end
+      delegate_missing_to :real_batch
 
-        private
-        attr_reader :real_batch
+      def jobs
+        raise "Managed Batches do not support calling .jobs directly!"
       end
+
+      private
+      attr_reader :real_batch
     end
   end
 end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -1,22 +1,20 @@
 require_relative './base_job'
 
-module CanvasSync
-  module JobBatches
-    class SerialBatchJob < BaseJob
-      def self.make_batch(sub_jobs, **kwargs, &blk)
-        ManagedBatchJob.make_batch(
-          sub_jobs,
-          **kwargs,
-          ordered: true,
-          concurrency: false,
-          desc_prefix: 'SerialBatchJob: ',
-          &blk
-        )
-      end
+module CanvasSync::JobBatches
+  class SerialBatchJob < BaseJob
+    def self.make_batch(sub_jobs, **kwargs, &blk)
+      ManagedBatchJob.make_batch(
+        sub_jobs,
+        **kwargs,
+        ordered: true,
+        concurrency: false,
+        desc_prefix: 'SerialBatchJob: ',
+        &blk
+      )
+    end
 
-      def perform(sub_jobs, **kwargs)
-        self.class.make_batch(sub_jobs, **kwargs)
-      end
+    def perform(sub_jobs, **kwargs)
+      self.class.make_batch(sub_jobs, **kwargs)
     end
   end
 end