canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb

@@ -0,0 +1,137 @@
+<h3><%= t('Pool') %></h3>
+
+<div class="table_container">
+    <table class="table table-striped table-bordered table-white table-hover">
+        <tbody>
+            <tr>
+                <th scope=row><%= t('PID') %></td>
+                <td><%= @pool.pid %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Created') %></td>
+                <td><%= safe_relative_time(@pool.created_at.to_f) %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Description') %></td>
+                <td><%= @pool.description %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Type') %></td>
+                <td><%= @pool.order.to_s.upcase %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Concurrency') %></td>
+                <td><%= @pool.concurrency %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Utilization') %></td>
+                <td><%= @pool.active_count %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Pending Tasks') %></td>
+                <td><%= @pool.pending_count %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Returned Tasks') %></td>
+                <td><%= @pool.complete_count %></td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+
+<%# Queued/Active Tasks %>
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Queued/Active Tasks') %>
+        </h3>
+    </div>
+</header>
+
+<% if @active_tasks.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Wrapper Batch BID') %></th>
+                <th><%= t('Job Class') %></th>
+                <th><%= t('Parameters') %></th>
+            </tr>
+        </thead>
+
+        <% @active_tasks.each do |job_desc| %>
+            <tr>
+                <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+                <td><%= job_desc['job'] %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended">
+                            <%= job_desc['args']&.to_json %>
+                            <%= job_desc['kwargs']&.to_json %>
+                        </div>
+                    </code>
+                </td>
+            </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<%# Pending Tasks %>
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Pending Tasks') %>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
+    %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}pools/#{@pool.pid}", page_key: :job_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @jobs.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Job Class') %></th>
+                <th><%= t('Parameters') %></th>
+                <th><%= t('Wrapper Batch BID') %></th>
+                <% if @pool.order == 'priority' %>
+                    <th><%= t('Priority') %></th>
+                <% end %>
+            </tr>
+        </thead>
+
+        <% @jobs.each do |job_desc| %>
+            <tr>
+                <td><%= job_desc['job'] %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended">
+                            <%= job_desc['args']&.to_json %>
+                            <%= job_desc['kwargs']&.to_json %>
+                        </div>
+                    </code>
+                </td>
+                <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+                <% if @pool.order == 'priority' %>
+                    <td><%= job_desc['priority'] %></td>
+                <% end %>
+            </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<form class="form-horizontal" action="<%= root_path %>pools/<%= @pool.pid %>" method="post">
+    <%= csrf_tag %>
+    <a class="btn btn-default" href="<%= root_path %>pools"><%= t('GoBack') %></a>
+    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb

@@ -0,0 +1,47 @@
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Pools') %>
+        </h3>
+    </div>
+    <% if @pools.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}pools" } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @pools.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Created') %></th>
+                <th><%= t('PID') %></th>
+                <th><%= t('Description') %></th>
+                <th><%= t('Type') %></th>
+                <th><%= t('Concurrency') %></th>
+                <th><%= t('Utilization') %></th>
+                <th><%= t('Pending Tasks') %></th>
+            </tr>
+        </thead>
+
+        <% @pools.each do |pool| %>
+        <tr>
+            <td><%= safe_relative_time(pool.created_at.to_f) %></th>
+            <td><a href="<%= root_path %>pools/<%= pool.pid %>"><%= pool.pid %></a></td>
+            <td><%= pool.description %></th>
+            <td><%= pool.order.to_s.upcase %></th>
+            <td><%= pool.concurrency %></th>
+            <td><%= pool.active_count %></th>
+            <td><%= pool.pending_count %></th>
+        </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<form action="<%= root_path %>pools/all" method="post">
+    <%= csrf_tag %>
+    <input class="btn btn-danger btn-xs pull-right flip" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -0,0 +1,218 @@
+
+begin
+  require "sidekiq/web"
+rescue LoadError
+  # client-only usage
+end
+
+require_relative "web/helpers"
+
+module CanvasSync::JobBatches::Sidekiq
+  module Web
+    DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
+
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      app.helpers do
+        include Web::Helpers
+
+        def dev_mode?
+          DEV_MODE
+        end
+      end
+
+      # =============== BATCHES =============== #
+
+      app.get "/batches" do
+        @count = (params['count'] || 25).to_i
+
+        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
+        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
+        @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        erb(get_template(:batches))
+      end
+
+      app.get "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        @tree_data = tree_data(@bid)
+
+        @count = (params['count'] || 25).to_i
+        @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
+        @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
+        @jobs = @jobs.map do |jid, score|
+          { jid: jid, }
+        end
+
+        erb(get_template(:batch))
+      end
+
+      app.get "/batches/:bid/tree" do
+        @bid = params[:bid]
+
+        json(tree_data(@bid, slice: params[:slice]))
+      end
+
+      app.helpers do
+        def tree_data(root_bid, slice: nil)
+          tree_bids = CanvasSync::JobBatches::Batch.bid_hierarchy(root_bid, slice: slice)
+
+          CanvasSync::JobBatches::Batch.redis do |r|
+            layer_data = ->(layer, parent = nil) {
+              bid = layer[0]
+              batch = CanvasSync::JobBatches::Batch.new(bid)
+
+              jobs_total = r.hget("BID-#{bid}", "job_count").to_i
+              jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
+              jobs_failed = r.scard("BID-#{bid}-failed").to_i
+              jobs_dead = r.scard("BID-#{bid}-dead").to_i
+              jobs_success = jobs_total - jobs_pending
+
+              batches_total = r.hget("BID-#{bid}", 'children').to_i
+              batches_success = r.scard("BID-#{bid}-batches-success").to_i
+              batches_pending = batches_total - batches_success
+              batches_failed = r.scard("BID-#{bid}-batches-failed").to_i
+
+              status = 'in_progress'
+              status = 'complete' if batches_pending == batches_failed && jobs_pending == jobs_failed
+              status = 'success' if batches_pending == 0 && jobs_pending == 0
+              status = 'deleted' if bid != root_bid && !batch.parent_bid
+
+              {
+                bid: bid,
+                created_at: r.hget("BID-#{bid}", 'created_at'),
+                status: status,
+                parent_bid: parent ? parent.bid : batch.parent_bid,
+                description: batch.description,
+                jobs: {
+                  pending_count: jobs_pending,
+                  successful_count: jobs_success,
+                  failed_count: jobs_failed,
+                  dead_count: jobs_dead,
+                  total_count: jobs_total,
+                  # items: batches.map{|b| layer_data[b] },
+                },
+                batches: {
+                  pending_count: batches_pending,
+                  successful_count: batches_success,
+                  failed_count: batches_failed,
+                  total_count: batches_total,
+                  items: layer[1].map{|b| layer_data[b, batch] },
+                },
+              }
+            }
+
+            data = layer_data[tree_bids]
+            data[:batches][:slice] = slice if slice
+            data
+          end
+        end
+
+        def format_context(batch)
+          bits = []
+          own_keys = batch.context.own.keys
+          batch.context.flatten.each do |k,v|
+            added = own_keys.include? k
+            bits << "  <span class=\"key #{added ? 'own' : 'inherited'}\">\"#{k}\": #{v.to_json},</span>"
+          end
+          bits = [
+            "{ // <span class=\"own\">Added</span> / <span class=\"inherited\">Inherited</span>",
+            *bits,
+            '}'
+          ]
+          bits.join("\n")
+        end
+      end
+
+      app.post "/batches/all" do
+        if params['delete']
+          index_key = CanvasSync::JobBatches::Batch::INDEX_ALL_BATCHES ? "batches" : "BID-ROOT-bids"
+          drain_zset(index_key) do |batches|
+            batches.each do |bid|
+              CanvasSync::JobBatches::Batch.cleanup_redis(bid)
+            end
+          end
+        end
+
+        redirect "#{root_path}batches"
+      end
+
+      app.post "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        if params['delete']
+          CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
+        end
+
+        redirect_with_query("#{root_path}batches")
+      end
+
+      # =============== POOLS =============== #
+
+      app.get "/pools" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @pools = page('pools', params['page'], @count)
+        @pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
+
+        erb(get_template(:pools))
+      end
+
+      app.get "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.new(@pid)
+
+        @active_tasks = @pool.active_jobs
+
+        @count = (params['count'] || 25).to_i
+        @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
+        @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
+
+        erb(get_template(:pool))
+      end
+
+      app.post "/pools/all" do
+        if params['delete']
+          drain_zset('pools') do |pools|
+            pools.each do |pid|
+              CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
+            end
+          end
+        end
+
+        redirect "#{root_path}pools"
+      end
+
+      app.post "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
+
+        if params['delete']
+          @pool.cleanup_redis
+        end
+
+        redirect_with_query("#{root_path}pools")
+      end
+    end
+  end
+end
+
+if defined?(::Sidekiq::Web)
+  rules = []
+  rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
+
+  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
+                                   root: File.expand_path("#{File.dirname(__FILE__)}/web"),
+                                   cascade: true,
+                                   header_rules: rules
+
+  ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
+  ::Sidekiq::Web.tabs["Batches"] = "batches"
+  ::Sidekiq::Web.tabs["Pools"] = "pools"
+  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
+end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -0,0 +1,136 @@
+begin
+  require 'sidekiq/batch'
+rescue LoadError
+end
+
+module CanvasSync
+  module JobBatches
+    module Sidekiq
+      module WorkerExtension
+        def bid
+          Thread.current[CURRENT_BATCH_THREAD_KEY].bid
+        end
+
+        def batch
+          Thread.current[CURRENT_BATCH_THREAD_KEY]
+        end
+
+        def batch_context
+          batch&.context || {}
+        end
+
+        def valid_within_batch?
+          batch.valid?
+        end
+      end
+
+      class SidekiqCallbackWorker
+        include ::Sidekiq::Worker
+        include WorkerExtension
+        include Batch::Callback::CallbackWorkerCommon
+
+        def self.enqueue_all(args, queue)
+          return if args.empty?
+
+          ::Sidekiq::Client.push_bulk(
+            'class' => self,
+            'args' => args,
+            'queue' => queue
+          )
+        end
+      end
+
+      class ClientMiddleware
+        def call(_worker, msg, _queue, _redis_pool = nil)
+          if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
+            batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
+          end
+          yield
+        end
+
+        def should_handle_batch?(msg)
+          return false if JobBatches::Sidekiq.is_activejob_job?(msg)
+          true
+        end
+      end
+
+      class ServerMiddleware
+        def call(_worker, msg, _queue)
+          if (bid = msg['bid'])
+            prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
+            begin
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
+              yield
+              Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
+              Batch.process_successful_job(bid, msg['jid'])
+            rescue
+              Batch.process_failed_job(bid, msg['jid'])
+              raise
+            ensure
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      def self.is_activejob_job?(msg)
+        return false unless defined?(::ActiveJob)
+
+        msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < JobBatches::ActiveJob::BatchAwareJob
+      end
+
+      def self.switch_tenant(job)
+        if defined?(::Apartment)
+          ::Apartment::Tenant.switch(job['apartment'] || 'public') do
+            yield
+          end
+        else
+          yield
+        end
+      end
+
+      def self.configure
+        if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != JobBatches::Batch
+          print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
+        end
+
+        ::Sidekiq.configure_client do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+        end
+        ::Sidekiq.configure_server do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+
+          config.server_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
+            chain.add JobBatches::Sidekiq::ServerMiddleware
+          end
+
+          config.death_handlers << ->(job, ex) do
+            switch_tenant(job) do
+              if is_activejob_job?(job)
+                JobBatches::ActiveJob.handle_job_death(job["args"][0], ex)
+              elsif job['bid'].present?
+                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              end
+            end
+          end
+        end
+        ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
+        # This alias helps apartment-sidekiq set itself up correctly
+        ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
+        ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
+        Batch::Callback.worker_class = SidekiqCallbackWorker
+      end
+    end
+  end
+end
+
+require_relative 'sidekiq/web'

data/lib/canvas_sync/job_batches/status.rb

@@ -0,0 +1,91 @@
+module CanvasSync
+  module JobBatches
+    class Batch
+      class Status
+        attr_reader :bid
+
+        def initialize(bid)
+          bid = bid.bid if bid.is_a?(Batch)
+          @bid = bid
+        end
+
+        def join
+          raise "Not supported"
+        end
+
+        def pending
+          Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+        end
+
+        def failures
+          Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+        end
+
+        def dead
+          Batch.redis { |r| r.scard("BID-#{bid}-dead") }.to_i
+        end
+
+        def completed_count
+          job_count - pending
+        end
+
+        def job_count
+          Batch.redis { |r| r.hget("BID-#{bid}", "job_count") }.to_i
+        end
+
+        def created_at
+          Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+        end
+
+        def parent_bid
+          Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+        end
+
+        def failure_info
+          Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
+        end
+
+        def complete?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
+        end
+
+        def success?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
+        end
+
+        def child_count
+          Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+        end
+
+        def completed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
+        end
+
+        def successful_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
+        end
+
+        def failed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
+        end
+
+        def data
+          {
+            bid: bid,
+            failures: failures,
+            pending: pending,
+            created_at: created_at,
+            complete: complete?,
+            success: success?,
+            failure_info: failure_info,
+            parent_bid: parent_bid,
+            child_count: child_count,
+            completed_children_count: completed_children_count,
+            successful_children_count: successful_children_count,
+            failed_children_count: failed_children_count,
+          }
+        end
+      end
+    end
+  end
+end