canvas_sync 0.17.3.beta1 → 0.17.5.beta2

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -27,6 +27,8 @@ module CanvasSync
                   value = value.to_f
                 elsif :#{type} == :json
                   value = JSON.parse(value)
+                elsif :#{type} == :symbol
+                  value = value&.to_sym
                 end
                 @#{key} = value
               end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -57,16 +57,21 @@ module CanvasSync
       class ServerMiddleware
         def call(_worker, msg, _queue)
           if (bid = msg['bid'])
+            prev_batch = Thread.current[:batch]
             begin
               Thread.current[:batch] = Batch.new(bid)
               yield
-              Thread.current[:batch] = nil
+              Thread.current[:batch].save_context_changes
               Batch.process_successful_job(bid, msg['jid'])
+            rescue SuccessfulFailure => err
+              Thread.current[:batch].save_context_changes
+              Batch.process_successful_job(@bid, msg['jid'])
+              raise ::Sidekiq::JobRetry::Handled # This ensure that Sidekiq logs the error, but doesn't attempt to Retry it
             rescue
               Batch.process_failed_job(bid, msg['jid'])
               raise
             ensure
-              Thread.current[:batch] = nil
+              Thread.current[:batch] = prev_batch
             end
           else
             yield
@@ -117,3 +122,5 @@ module CanvasSync
     end
   end
 end
+
+require_relative 'sidekiq/web'

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -0,0 +1,114 @@
+
+begin
+  require "sidekiq/web"
+rescue LoadError
+  # client-only usage
+end
+
+require_relative "web/helpers"
+
+module CanvasSync::JobBatches::Sidekiq
+  module Web
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      app.helpers do
+        include Web::Helpers
+      end
+
+      app.get "/batches" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @batches = page('batches', params['page'], @count)
+        @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        erb(get_template(:batches))
+      end
+
+      app.get "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        @count = (params['count'] || 25).to_i
+        @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
+        @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
+        @jobs = @jobs.map {|jid, score| jid }
+
+        erb(get_template(:batch))
+      end
+
+      app.post "/batches/all" do
+        if params['delete']
+          drain_zset('batches') do |batches|
+            batches.each do |bid|
+              CanvasSync::JobBatches::Batch.cleanup_redis(bid)
+            end
+          end
+        end
+
+        redirect "#{root_path}batches"
+      end
+
+      app.post "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        if params['delete']
+          CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
+        end
+
+        redirect_with_query("#{root_path}batches")
+      end
+
+      # =============== POOLS =============== #
+
+      app.get "/pools" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @pools = page('pools', params['page'], @count)
+        @pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
+
+        erb(get_template(:pools))
+      end
+
+      app.get "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.new(@pid)
+
+        @count = (params['count'] || 25).to_i
+        @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
+        @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
+
+        erb(get_template(:pool))
+      end
+
+      app.post "/pools/all" do
+        if params['delete']
+          drain_zset('pools') do |pools|
+            pools.each do |pid|
+              CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
+            end
+          end
+        end
+
+        redirect "#{root_path}pools"
+      end
+
+      app.post "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
+
+        if params['delete']
+          @pool.cleanup_redis
+        end
+
+        redirect_with_query("#{root_path}pools")
+      end
+    end
+  end
+end
+
+if defined?(::Sidekiq::Web)
+  ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
+  ::Sidekiq::Web.tabs["Batches"] = "batches"
+  ::Sidekiq::Web.tabs["Pools"] = "pools"
+  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
+end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module CanvasSync::JobBatches::Sidekiq
+  module Web
+    module Helpers
+      VIEW_PATH    = File.expand_path("../web/views", __dir__)
+
+      module_function
+
+      def get_template(name)
+        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+      end
+
+      def drain_zset(key)
+        items, _ = Sidekiq.redis do |r|
+          r.multi do
+            r.zrange(key, 0, -1)
+            r.zremrangebyrank(key, 0, -1)
+          end
+        end
+        yield items
+      end
+
+      def safe_relative_time(time)
+        time = parse_time(time)
+        relative_time(time)
+      end
+
+      def parse_time(time)
+        case time
+        when Time
+          time
+        when Integer, Float
+          Time.at(time)
+        else
+          Time.parse(time.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb

@@ -0,0 +1,42 @@
+<table class="table table-striped table-bordered table-hover">
+    <thead>
+        <tr>
+            <th rowspan="2"><%= t('Started') %></th>
+            <th rowspan="2"><%= t('BID') %></th>
+            <th rowspan="2"><%= t('Description') %></th>
+
+            <th colspan="4"><%= t('Jobs') %></th>
+            <th colspan="4"><%= t('Sub-Batches') %></th>
+        </tr>
+        <tr>
+            <th><%= t('Pending') %></th>
+            <th><%= t('Failed') %></th>
+            <th><%= t('Complete') %></th>
+            <th><%= t('Total') %></th>
+
+            <th><%= t('Pending') %></th>
+            <th><%= t('Failed') %></th>
+            <th><%= t('Success') %></th>
+            <th><%= t('Total') %></th>
+        </tr>
+    </thead>
+
+    <% batches.each do |batch| %>
+        <% status = CanvasSync::JobBatches::Batch::Status.new(batch) %>
+        <tr>
+            <td><%= safe_relative_time(batch.created_at.to_f) %></th>
+            <td><a href="<%= root_path %>batches/<%= batch.bid %>"><%= batch.bid %></a></td>
+            <td><%= batch.description %></th>
+
+            <td><%= status.pending %></th>
+            <td><%= status.failures %></th>
+            <td><%= status.completed_count %></th>
+            <td><%= status.job_count %></th>
+
+            <td><%= status.child_count - status.successful_children_count %></th>
+            <td><%= status.failed_children_count %></th>
+            <td><%= status.successful_children_count %></th>
+            <td><%= status.child_count %></th>
+        </tr>
+    <% end %>
+</table>

data/lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb

@@ -0,0 +1,26 @@
+
+<% if @total_size > @count %>
+  <ul class="pagination pull-right flip">
+    <% _page_key = defined?(page_key) ? page_key.to_s : 'page' %>
+
+    <li class="<%= 'disabled' if @current_page == 1 %>">
+      <a href="<%= url %>?<%= _page_key %>=1">&laquo;</a>
+    </li>
+    <% if @current_page > 1 %>
+      <li>
+        <a href="<%= url %>?<%= (qparams(page: @current_page - 1) || '').gsub('page', _page_key) %>"><%= @current_page - 1 %></a>
+      </li>
+    <% end %>
+    <li class="disabled">
+      <a href="<%= url %>?<%= (qparams(page: @current_page) || '').gsub('page', _page_key) %>"><%= @current_page %></a>
+    </li>
+    <% if @total_size > @current_page * @count %>
+      <li>
+        <a href="<%= url %>?<%= (qparams(page: @current_page + 1) || '').gsub('page', _page_key) %>"><%= @current_page + 1 %></a>
+      </li>
+    <% end %>
+    <li class="<%= 'disabled' if @total_size <= @current_page * @count %>">
+      <a href="<%= url %>?<%= (qparams(page: (@total_size.to_f / @count).ceil) || '').gsub('page', _page_key) %>">&raquo;</a>
+    </li>
+  </ul>
+<% end %>

data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb

@@ -0,0 +1,138 @@
+<h3><%= t('Batch') %></h3>
+<% status = CanvasSync::JobBatches::Batch::Status.new(@batch) %>
+
+<div class="table_container">
+    <table class="table table-striped table-bordered table-white table-hover">
+        <tbody>
+            <tr>
+                <th colspan="2" scope=row><%= t('Batch') %></td>
+                <td><%= @batch.bid %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Started') %></td>
+                <td><%= safe_relative_time(@batch.created_at.to_f) %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Description') %></td>
+                <td><%= @batch.description %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Added Context') %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended"><%= @batch.context.own.to_json %></div>
+                    </code>
+                </td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Full Context') %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended"><%= @batch.context.flatten.to_json %></div>
+                    </code>
+                </td>
+            </tr>
+
+            <tr>
+                <th colspan="3">Jobs</th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Pending') %></th>
+                <td><%= status.pending %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Failed') %></th>
+                <td><%= status.failures %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Complete') %></th>
+                <td><%= status.completed_count %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Total') %></th>
+                <td><%= status.job_count %></th>
+            </tr>
+
+            <tr>
+                <th colspan="3">Batches</th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Pending') %></th>
+                <td><%= status.child_count - status.successful_children_count %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Failed') %></th>
+                <td><%= status.failed_children_count %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Success') %></th>
+                <td><%= status.successful_children_count %></th>
+            </tr>
+            <tr>
+                <th></th>
+                <th><%= t('Total') %></th>
+                <td><%= status.child_count %></th>
+            </tr>
+
+        </tbody>
+    </table>
+</div>
+
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Jobs') %>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
+    %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @jobs.any? %>
+<div class="table_container">
+    <%= erb get_template(:_batches_table), locals: { batches: @jobs } %>
+</div>
+<% end %>
+
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Child Batches') %>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_batches_page
+        @total_size = @total_batches_size
+    %>
+    <% if @sub_batches.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :batch_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @sub_batches.any? %>
+<div class="table_container">
+    <%= erb get_template(:_batches_table), locals: { batches: @sub_batches } %>
+</div>
+<% end %>
+
+<form class="form-horizontal" action="<%= root_path %>batches/<%= @batch.bid %>" method="post">
+    <%= csrf_tag %>
+    <a class="btn btn-default" href="<%= root_path %>batches"><%= t('GoBack') %></a>
+    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb

@@ -0,0 +1,23 @@
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Batches') %>
+        </h3>
+    </div>
+    <% if @batches.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches" } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @batches.any? %>
+<div class="table_container">
+    <%= erb get_template(:_batches_table), locals: { batches: @batches } %>
+</div>
+<% end %>
+
+<form action="<%= root_path %>batches/all" method="post">
+    <%= csrf_tag %>
+    <input class="btn btn-danger btn-xs pull-right flip" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb

@@ -0,0 +1,85 @@
+<h3><%= t('Pool') %></h3>
+
+<div class="table_container">
+    <table class="table table-striped table-bordered table-white table-hover">
+        <tbody>
+            <tr>
+                <th scope=row><%= t('PID') %></td>
+                <td><%= @pool.pid %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Created') %></td>
+                <td><%= safe_relative_time(@pool.created_at.to_f) %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Description') %></td>
+                <td><%= @pool.description %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Type') %></td>
+                <td><%= @pool.order.to_s.upcase %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Concurrency') %></td>
+                <td><%= @pool.concurrency %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Utilization') %></td>
+                <td><%= @pool.active_count %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Pending Tasks') %></td>
+                <td><%= @pool.pending_count %></td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Tasks') %>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
+    %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}pools/#{@pool.pid}", page_key: :job_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @jobs.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Job Class') %></th>
+                <th><%= t('Parameters') %></th>
+                <th><%= t('Wrapper Batch BID') %></th>
+            </tr>
+        </thead>
+
+        <% @jobs.each do |job_desc| %>
+            <tr>
+                <td><%= job_desc['job'] %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended"><%= job_desc['parameters'].to_json  %></div>
+                    </code>
+                </td>
+                <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+            </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<form class="form-horizontal" action="<%= root_path %>pools/<%= @pool.pid %>" method="post">
+    <%= csrf_tag %>
+    <a class="btn btn-default" href="<%= root_path %>pools"><%= t('GoBack') %></a>
+    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>