canvas_sync 0.23.4 → 0.24.0.beta1

data/lib/canvas_sync/job_batches/compat/sidekiq/web.rb

@@ -1,218 +0,0 @@
-
-begin
-  require "sidekiq/web"
-rescue LoadError
-  # client-only usage
-end
-
-require_relative "web/helpers"
-
-module CanvasSync::JobBatches::Compat::Sidekiq
-  module Web
-    DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
-    Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
-    Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
-
-    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      app.helpers do
-        include Web::Helpers
-
-        def dev_mode?
-          DEV_MODE
-        end
-      end
-
-      # =============== BATCHES =============== #
-
-      app.get "/batches" do
-        @count = (params['count'] || 25).to_i
-
-        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
-        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
-        @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
-
-        erb(get_template(:batches))
-      end
-
-      app.get "/batches/:bid" do
-        @bid = params[:bid]
-        @batch = CanvasSync::JobBatches::Batch.new(@bid)
-
-        @tree_data = tree_data(@bid)
-
-        @count = (params['count'] || 25).to_i
-        @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
-        @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
-
-        @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
-        @jobs = @jobs.map do |jid, score|
-          { jid: jid, }
-        end
-
-        erb(get_template(:batch))
-      end
-
-      app.get "/batches/:bid/tree" do
-        @bid = params[:bid]
-
-        json(tree_data(@bid, slice: params[:slice]))
-      end
-
-      app.helpers do
-        def tree_data(root_bid, slice: nil)
-          tree_bids = CanvasSync::JobBatches::Batch.bid_hierarchy(root_bid, slice: slice)
-
-          CanvasSync::JobBatches::Batch.redis do |r|
-            layer_data = ->(layer, parent = nil) {
-              bid = layer[0]
-              batch = CanvasSync::JobBatches::Batch.new(bid)
-
-              jobs_total = r.hget("BID-#{bid}", "job_count").to_i
-              jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
-              jobs_failed = r.scard("BID-#{bid}-failed").to_i
-              jobs_dead = r.scard("BID-#{bid}-dead").to_i
-              jobs_success = jobs_total - jobs_pending
-
-              batches_total = r.hget("BID-#{bid}", 'children').to_i
-              batches_success = r.scard("BID-#{bid}-batches-success").to_i
-              batches_pending = batches_total - batches_success
-              batches_failed = r.scard("BID-#{bid}-batches-failed").to_i
-
-              status = 'in_progress'
-              status = 'complete' if batches_pending == batches_failed && jobs_pending == jobs_failed
-              status = 'success' if batches_pending == 0 && jobs_pending == 0
-              status = 'deleted' if bid != root_bid && !batch.parent_bid
-
-              {
-                bid: bid,
-                created_at: r.hget("BID-#{bid}", 'created_at'),
-                status: status,
-                parent_bid: parent ? parent.bid : batch.parent_bid,
-                description: batch.description,
-                jobs: {
-                  pending_count: jobs_pending,
-                  successful_count: jobs_success,
-                  failed_count: jobs_failed,
-                  dead_count: jobs_dead,
-                  total_count: jobs_total,
-                  # items: batches.map{|b| layer_data[b] },
-                },
-                batches: {
-                  pending_count: batches_pending,
-                  successful_count: batches_success,
-                  failed_count: batches_failed,
-                  total_count: batches_total,
-                  items: layer[1].map{|b| layer_data[b, batch] },
-                },
-              }
-            }
-
-            data = layer_data[tree_bids]
-            data[:batches][:slice] = slice if slice
-            data
-          end
-        end
-
-        def format_context(batch)
-          bits = []
-          own_keys = batch.context.own.keys
-          batch.context.flatten.each do |k,v|
-            added = own_keys.include? k
-            bits << "  <span class=\"key #{added ? 'own' : 'inherited'}\">\"#{k}\": #{v.to_json},</span>"
-          end
-          bits = [
-            "{ // <span class=\"own\">Added</span> / <span class=\"inherited\">Inherited</span>",
-            *bits,
-            '}'
-          ]
-          bits.join("\n")
-        end
-      end
-
-      app.post "/batches/all" do
-        if params['delete']
-          index_key = CanvasSync::JobBatches::Batch::INDEX_ALL_BATCHES ? "batches" : "BID-ROOT-bids"
-          drain_zset(index_key) do |batches|
-            batches.each do |bid|
-              CanvasSync::JobBatches::Batch.cleanup_redis(bid)
-            end
-          end
-        end
-
-        redirect "#{root_path}batches"
-      end
-
-      app.post "/batches/:bid" do
-        @bid = params[:bid]
-        @batch = CanvasSync::JobBatches::Batch.new(@bid)
-
-        if params['delete']
-          CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
-        end
-
-        redirect_with_query("#{root_path}batches")
-      end
-
-      # =============== POOLS =============== #
-
-      app.get "/pools" do
-        @count = (params['count'] || 25).to_i
-        @current_page, @total_size, @pools = page('pools', params['page'], @count)
-        @pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
-
-        erb(get_template(:pools))
-      end
-
-      app.get "/pools/:pid" do
-        @pid = params[:pid]
-        @pool = CanvasSync::JobBatches::Pool.new(@pid)
-
-        @active_tasks = @pool.active_jobs
-
-        @count = (params['count'] || 25).to_i
-        @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
-        @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
-
-        erb(get_template(:pool))
-      end
-
-      app.post "/pools/all" do
-        if params['delete']
-          drain_zset('pools') do |pools|
-            pools.each do |pid|
-              CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
-            end
-          end
-        end
-
-        redirect "#{root_path}pools"
-      end
-
-      app.post "/pools/:pid" do
-        @pid = params[:pid]
-        @pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
-
-        if params['delete']
-          @pool.cleanup_redis
-        end
-
-        redirect_with_query("#{root_path}pools")
-      end
-    end
-  end
-end
-
-if defined?(::Sidekiq::Web)
-  rules = []
-  rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Compat::Sidekiq::Web::DEV_MODE
-
-  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
-                                   root: File.expand_path("#{File.dirname(__FILE__)}/web"),
-                                   cascade: true,
-                                   header_rules: rules
-
-  ::Sidekiq::Web.register CanvasSync::JobBatches::Compat::Sidekiq::Web
-  ::Sidekiq::Web.tabs["Batches"] = "batches"
-  ::Sidekiq::Web.tabs["Pools"] = "pools"
-  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
-end

data/lib/canvas_sync/job_batches/compat/sidekiq.rb

@@ -1,149 +0,0 @@
-begin
-  require 'sidekiq/batch'
-rescue LoadError
-end
-
-module CanvasSync::JobBatches
-  module Compat
-    module Sidekiq
-      module WorkerExtension
-        def bid
-          Thread.current[CURRENT_BATCH_THREAD_KEY].bid
-        end
-
-        def batch
-          Thread.current[CURRENT_BATCH_THREAD_KEY]
-        end
-
-        def batch_context
-          batch&.context || {}
-        end
-
-        def valid_within_batch?
-          batch.valid?
-        end
-      end
-
-      class SidekiqCallbackWorker
-        include ::Sidekiq::Worker
-        include WorkerExtension
-        include Batch::Callback::CallbackWorkerCommon
-
-        def self.enqueue_all(args, queue)
-          return if args.empty?
-
-          ::Sidekiq::Client.push_bulk(
-            'class' => self,
-            'args' => args,
-            'queue' => queue
-          )
-        end
-      end
-
-      class ClientMiddleware
-        include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
-
-        def call(_worker, msg, _queue, _redis_pool = nil)
-          if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
-            batch.append_jobs(msg['jid']) if (msg['bid'] = batch.bid)
-          end
-          yield
-        end
-
-        def should_handle_batch?(msg)
-          return false if CanvasSync::JobBatches::Compat::Sidekiq.is_activejob_job?(msg)
-          true
-        end
-      end
-
-      class ServerMiddleware
-        include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
-
-        def call(_worker, msg, _queue)
-          if (bid = msg['bid'])
-            prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
-            begin
-              Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
-              yield
-              Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
-              Batch.process_successful_job(bid, msg['jid'])
-            rescue
-              Batch.process_failed_job(bid, msg['jid'])
-              raise
-            ensure
-              Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
-            end
-          else
-            yield
-          end
-        end
-      end
-
-      def self.is_activejob_job?(msg)
-        return false unless defined?(::ActiveJob)
-
-        msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob
-      end
-
-      def self.switch_tenant(job)
-        if defined?(::Apartment)
-          ::Apartment::Tenant.switch(job['apartment'] || 'public') do
-            yield
-          end
-        else
-          yield
-        end
-      end
-
-      def self.sidekiq_middleware(placement, &blk)
-        install_middleware = ->(config) do
-          config.send("#{placement}_middleware") do |chain|
-            blk.call(chain)
-          end
-        end
-
-        ::Sidekiq.configure_client(&install_middleware) if placement == :client
-        ::Sidekiq.configure_server(&install_middleware)
-      end
-
-      def self.configure
-        return if @already_configured
-        @already_configured = true
-
-        if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != CanvasSync::JobBatches::Batch
-          print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
-        end
-
-        sidekiq_middleware(:client) do |chain|
-          chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
-          chain.add CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware
-        end
-
-        sidekiq_middleware(:server) do |chain|
-          chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
-          chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
-        end
-
-        ::Sidekiq.configure_server do |config|
-          config.death_handlers << ->(job, ex) do
-            switch_tenant(job) do
-              if is_activejob_job?(job)
-                CanvasSync::JobBatches::Compat::ActiveJob.handle_job_death(job["args"][0], ex)
-              elsif job['bid'].present?
-                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
-              end
-            end
-          end
-        end
-
-        ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
-        # This alias helps apartment-sidekiq set itself up correctly
-        ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
-        ::Sidekiq::Worker.send(:include, CanvasSync::JobBatches::Compat::Sidekiq::WorkerExtension)
-        Batch::Callback.worker_class = SidekiqCallbackWorker
-      end
-    end
-  end
-end
-
-require_relative 'sidekiq/web'

data/lib/canvas_sync/job_batches/compat.rb

@@ -1,20 +0,0 @@
-
-module CanvasSync::JobBatches
-  module Compat
-    def self.load_compat(name)
-      name = name.to_s
-      begin
-        require name
-      rescue LoadError
-      end
-
-      if name.classify.safe_constantize
-        require_relative "./compat/#{name}"
-        "CanvasSync::JobBatches::Compat::#{name.classify}".constantize.configure
-      end
-    end
-
-    load_compat(:active_job)
-    load_compat(:sidekiq)
-  end
-end

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -1,157 +0,0 @@
-module CanvasSync::JobBatches
-  class ContextHash
-    delegate_missing_to :flatten
-
-    def initialize(bid, hash = nil)
-      @bid_stack = [bid]
-      @hash_map = {}
-      @dirty = false
-      @flattened = nil
-      @hash_map[bid] = hash.with_indifferent_access if hash
-    end
-
-    # Local is "the nearest batch with a context value"
-    # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
-    # that can be accessed transparently from one of it's internal, context-less Batches
-    def local_bid
-      bid = @bid_stack[-1]
-      while bid.present?
-        bhash = resolve_hash(bid)
-        return bid if bhash
-        bid = get_parent_bid(bid)
-      end
-      nil
-    end
-
-    def local
-      @hash_map[local_bid]
-    end
-
-    def own
-      resolve_hash(@bid_stack[-1]) || {}
-    end
-
-    def set_local(new_hash)
-      @dirty = true
-      local.clear.merge!(new_hash)
-    end
-
-    def clear
-      local.clear
-      @flattened = nil
-      @dirty = true
-      self
-    end
-
-    def []=(key, value)
-      @flattened = nil
-      @dirty = true
-      local[key] = value
-    end
-
-    def [](key)
-      bid = @bid_stack[-1]
-      while bid.present?
-        bhash = resolve_hash(bid)
-        return bhash[key] if bhash&.key?(key)
-        bid = get_parent_bid(bid)
-      end
-      nil
-    end
-
-    def reload!
-      @dirty = false
-      @hash_map = {}
-      self
-    end
-
-    def save!(force: false)
-      return unless dirty? || force
-      Batch.redis do |r|
-        r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
-      end
-    end
-
-    def dirty?
-      @dirty
-    end
-
-    def is_a?(arg)
-      return true if Hash <= arg
-      super
-    end
-
-    def flatten
-      return @flattened if @flattened
-
-      load_all
-      flattened = {}
-      @bid_stack.compact.each do |bid|
-        flattened.merge!(@hash_map[bid]) if @hash_map[bid]
-      end
-      flattened.freeze
-
-      @flattened = flattened.with_indifferent_access
-    end
-
-    def to_h
-      flatten
-    end
-
-    private
-
-    def get_parent_hash(bid)
-      resolve_hash(get_parent_bid(bid)).freeze
-    end
-
-    def get_parent_bid(bid)
-      index = @bid_stack.index(bid)
-      raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
-
-      index -= 1
-      if index >= 0
-        @bid_stack[index]
-      else
-        pbid = Batch.redis do |r|
-          callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
-          callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
-        end
-        @bid_stack.unshift(pbid)
-        pbid
-      end
-    end
-
-    def resolve_hash(bid)
-      return nil unless bid.present?
-      return @hash_map[bid] if @hash_map.key?(bid)
-
-      context_json, editable = Batch.redis do |r|
-        r.multi do |r|
-          r.hget("BID-#{bid}", "context")
-          r.hget("BID-#{bid}", "allow_context_changes")
-        end
-      end
-
-      if context_json.present?
-        context_hash = JSON.parse(context_json)
-        context_hash = context_hash.with_indifferent_access
-        context_hash.each do |k, v|
-          v.freeze
-        end
-        context_hash.freeze unless editable
-
-        @hash_map[bid] = context_hash
-      else
-        @hash_map[bid] = nil
-      end
-    end
-
-    def load_all
-      resolve_hash(@bid_stack[0]).freeze
-      while @bid_stack[0].present?
-        get_parent_hash(@bid_stack[0])
-      end
-      @hash_map
-    end
-  end
-end

data/lib/canvas_sync/job_batches/hier_batch_ids.lua

@@ -1,25 +0,0 @@
-
-local function add_bids(root, depth)
-    local result_data = {}
-
-    if depth > 0 then
-        local sbids
-        if depth == tonumber(ARGV[2]) and ARGV[4] then
-            local min, max = ARGV[4]:match('(%d+):(%d+)')
-            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', min, max)
-        else
-            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', 0, tonumber(ARGV[3]) - 1)
-        end
-
-        local sub_data = {}
-        for _,v in ipairs(sbids) do
-            table.insert(sub_data, add_bids(v, depth - 1))
-        end
-
-        return { root, sub_data }
-    end
-
-    return { root, result_data}
-end
-
-return add_bids(ARGV[1], tonumber(ARGV[2]))

data/lib/canvas_sync/job_batches/jobs/base_job.rb

@@ -1,5 +0,0 @@
-module CanvasSync::JobBatches
-  class BaseJob < ::ActiveJob::Base
-
-  end
-end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -1,20 +0,0 @@
-require_relative './base_job'
-
-module CanvasSync::JobBatches
-  class ConcurrentBatchJob < BaseJob
-    def self.make_batch(sub_jobs, **kwargs, &blk)
-      ManagedBatchJob.make_batch(
-        sub_jobs,
-        concurrency: true,
-        **kwargs,
-        ordered: false,
-        desc_prefix: 'ConcurrentBatchJob: ',
-        &blk
-      )
-    end
-
-    def perform(sub_jobs, **kwargs)
-      self.class.make_batch(sub_jobs, **kwargs)
-    end
-  end
-end