canvas_sync 0.17.17.beta1 → 0.17.23.beta1

data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/batch_tree.js

@@ -0,0 +1,106 @@
+import { h, Component, render } from 'https://unpkg.com/preact?module';
+import htm from 'https://unpkg.com/htm?module';
+import { root_url } from './util.js';
+
+// Initialize htm with Preact
+const html = htm.bind(h);
+
+const StatusBlock = (props) => html`
+  <div class="status-block ${props.class || ''}">
+    ${props.title && props.title + ':'}
+    <span class="tree-stat pending">${props.pending_count}</span>
+    |
+    <span class="tree-stat failed">${props.failed_count}</span>
+    |
+    <span class="tree-stat success">${props.successful_count}</span>
+    /
+    <span class="tree-stat total">${props.total_count}</span>
+  </div>
+`
+
+class TreeLevel extends Component {
+  get bid() {
+    return this.props.data.bid;
+  }
+
+  get batch() {
+    return this.props.data;
+  }
+
+  load_more = async (event) => {
+    event.preventDefault();
+    const l = this.batch.batches.items.length;
+    const resp = await fetch(`${root_url}batches/${this.bid}/tree?slice=${l}:${l + 5 - 1}`)
+    const result = await resp.json()
+    const newEntries = result.batches.items;
+    for (let ent of newEntries) {
+      this.batch.batches.items.push(ent)
+    }
+    this.forceUpdate()
+  }
+
+  toggle = (event) => {
+    event.preventDefault();
+    this.setState({ collapsed: !this.state.collapsed })
+  }
+
+  render() {
+    const { data: bd } = this.props;
+
+    let sub_entries = [];
+    let sub_batches = bd.batches.items;
+    for (let b of sub_batches) {
+      sub_entries.push(html`<${TreeLevel} data=${b} />`)
+    }
+
+    let fully_loaded = !(sub_batches.length < bd.batches.total_count);
+
+    const load_more_elem = html`<div class="load-more">
+      ${sub_entries.length} / ${bd.batches.total_count} Items Loaded - <a href="#" onClick=${this.load_more}>Load More</a>
+    </div>`
+
+    return html`<div class="tree-entry tree-batch">
+      <div class="header">
+        <a class="row-toggle ${!bd.batches.total_count && 'not_applicable'}" onClick=${this.toggle} href="#">
+          ${this.state.collapsed ? '+' : '-'}
+        </a>
+
+        <div class="header-inner">
+          <div class="main">
+            <span class="bid">
+              ${bd.bid}
+              <a class="bid-goto" href="${root_url}batches/${bd.bid}">⇢</a>
+            </span>
+            ${bd.description || (bd.status == 'deleted' && html`<i class="text-inactive">Deleted</i>`) || html`<i class="text-inactive">No Description</i>`}
+          </div>
+
+          <span class="status-label ${bd.status}">${bd.status}</span>
+
+          ${bd.status != 'deleted' && html`
+            <${StatusBlock} class="job-status" title="Jobs" ...${bd.jobs} />
+            <${StatusBlock} class="batch-status" title="Batches" ...${bd.batches} />
+          `}
+        </div>
+      </div>
+
+      <div class="subitems ${this.state.collapsed ? 'hidden' : ''}">
+        ${sub_entries}
+        ${!fully_loaded && load_more_elem}
+      </div>
+    </div>`
+  }
+}
+
+class TreeRoot extends Component {
+  render() {
+    const tree_data = JSON.parse(document.querySelector('#batch-tree #initial-data').innerHTML);
+    return html`
+      <div class="tree-header">
+        <${StatusBlock} pending_count="pending" failed_count="failed" successful_count="successful" total_count="total" />
+      </div>
+      <${TreeLevel} data=${tree_data} />
+    `;
+  }
+}
+
+render(html`<${TreeRoot} />`, document.querySelector('#batch-tree'));

data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/util.js

@@ -0,0 +1,2 @@
+
+export const root_url = (document.querySelector('meta[name="sidekiq-baseurl"]')).content;

data/lib/canvas_sync/job_batches/sidekiq/web/views/_batch_tree.erb

@@ -0,0 +1,6 @@
+
+<div id="batch-tree" class="batch-tree">
+    <script id="initial-data" type="application/json"><%= @tree_data.to_json %></script>
+</div>
+
+<script type="module" src="<%= root_path %>batches_assets/js/batch_tree.js"></script>

data/lib/canvas_sync/job_batches/sidekiq/web/views/_common.erb

@@ -0,0 +1,13 @@
+
+<% add_to_head do %>
+    <meta name="sidekiq-baseurl" content="<%= root_path %>" />
+    <% if dev_mode? %>
+        <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
+        <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
+    <% else %>
+        <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
+        <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
+        <%# TODO: Pre-compile LESS %>
+        <!-- <link href="<%= root_path %>batches_assets/css/styles.css" media="screen" rel="stylesheet" type="text/css" /> -->
+    <% end %>
+<% end %>

data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb

@@ -1,3 +1,5 @@
+<%= erb get_template(:_common) %>
+
 <h3><%= t('Batch') %></h3>
 <% status = CanvasSync::JobBatches::Batch::Status.new(@batch) %>
 
@@ -17,70 +19,13 @@
                 <td><%= @batch.description %></td>
             </tr>
             <tr>
-                <th colspan="2" scope=row><%= t('Added Context') %></td>
+                <th colspan="2" scope=row><%= t('Context') %></td>
                 <td>
-                    <code class="code-wrap">
-                        <div class="args-extended"><%= @batch.context.own.to_json %></div>
+                    <code class="code-wrap batch-context">
+                        <div class="args-extended"><%= format_context(@batch) %></div>
                     </code>
                 </td>
             </tr>
-            <tr>
-                <th colspan="2" scope=row><%= t('Full Context') %></td>
-                <td>
-                    <code class="code-wrap">
-                        <div class="args-extended"><%= @batch.context.flatten.to_json %></div>
-                    </code>
-                </td>
-            </tr>
-
-            <tr>
-                <th colspan="3">Jobs</th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Pending') %></th>
-                <td><%= status.pending %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Failed') %></th>
-                <td><%= status.failures %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Complete') %></th>
-                <td><%= status.completed_count %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Total') %></th>
-                <td><%= status.job_count %></th>
-            </tr>
-
-            <tr>
-                <th colspan="3">Batches</th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Pending') %></th>
-                <td><%= status.child_count - status.successful_children_count %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Failed') %></th>
-                <td><%= status.failed_children_count %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Success') %></th>
-                <td><%= status.successful_children_count %></th>
-            </tr>
-            <tr>
-                <th></th>
-                <th><%= t('Total') %></th>
-                <td><%= status.child_count %></th>
-            </tr>
-
         </tbody>
     </table>
 </div>
@@ -88,51 +33,33 @@
 <header class="row">
     <div class="col-sm-5">
         <h3>
-            <%= t('Jobs') %>
+            <%= t('Child Batches') %>
         </h3>
     </div>
-    <%
-        @current_page = @current_jobs_page
-        @total_size = @total_jobs_size
-    %>
-    <% if @jobs.any? && @total_size > @count.to_i %>
-        <div class="col-sm-4">
-            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
-        </div>
-    <% end %>
 </header>
 
-<% if @jobs.any? %>
-<div class="table_container">
-    <%= erb get_template(:_jobs_table), locals: { jobs: @jobs } %>
-</div>
-<% end %>
+<%= erb get_template(:_batch_tree), locals: { } %>
 
 <header class="row">
     <div class="col-sm-5">
         <h3>
-            <%= t('Child Batches') %>
+            <%= t('Jobs') %>
+            <i>(Queued and Current)</i>
         </h3>
     </div>
     <%
-        @current_page = @current_batches_page
-        @total_size = @total_batches_size
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
     %>
-    <% if @sub_batches.any? && @total_size > @count.to_i %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
         <div class="col-sm-4">
-            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :batch_page } %>
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
         </div>
     <% end %>
 </header>
 
-<% if @sub_batches.any? %>
+<% if @jobs.any? %>
 <div class="table_container">
-    <%= erb get_template(:_batches_table), locals: { batches: @sub_batches } %>
+    <%= erb get_template(:_jobs_table), locals: { jobs: @jobs } %>
 </div>
 <% end %>
-
-<form class="form-horizontal" action="<%= root_path %>batches/<%= @batch.bid %>" method="post">
-    <%= csrf_tag %>
-    <a class="btn btn-default" href="<%= root_path %>batches"><%= t('GoBack') %></a>
-    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
-</form>

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -9,11 +9,19 @@ require_relative "web/helpers"
 
 module CanvasSync::JobBatches::Sidekiq
   module Web
+    DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+
     def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       app.helpers do
         include Web::Helpers
+
+        def dev_mode?
+          DEV_MODE
+        end
       end
 
+      # =============== BATCHES =============== #
+
       app.get "/batches" do
         @count = (params['count'] || 25).to_i
         @current_page, @total_size, @batches = page('batches', params['page'], @count)
@@ -26,6 +34,8 @@ module CanvasSync::JobBatches::Sidekiq
         @bid = params[:bid]
         @batch = CanvasSync::JobBatches::Batch.new(@bid)
 
+        @tree_data = tree_data(@bid)
+
         @count = (params['count'] || 25).to_i
         @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
         @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
@@ -36,6 +46,81 @@ module CanvasSync::JobBatches::Sidekiq
         erb(get_template(:batch))
       end
 
+      app.get "/batches/:bid/tree" do
+        @bid = params[:bid]
+
+        json(tree_data(@bid, slice: params[:slice]))
+      end
+
+      app.helpers do
+        def tree_data(root_bid, slice: nil)
+          tree_bids = CanvasSync::JobBatches::Batch.bid_hierarchy(root_bid, slice: slice)
+
+          CanvasSync::JobBatches::Batch.redis do |r|
+            layer_data = ->(layer, parent = nil) {
+              bid = layer[0]
+              batch = CanvasSync::JobBatches::Batch.new(bid)
+
+              jobs_total = r.hget("BID-#{bid}", "job_count").to_i
+              jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
+              jobs_failed = r.scard("BID-#{bid}-failed").to_i
+              jobs_success = jobs_total - jobs_pending
+
+              batches_total = r.hget("BID-#{bid}", 'children').to_i
+              batches_success = r.scard("BID-#{bid}-batches-success").to_i
+              batches_pending = batches_total - batches_success
+              batches_failed = r.scard("BID-#{bid}-batches-failed").to_i
+
+              status = 'in_progress'
+              status = 'complete' if batches_pending == batches_failed && jobs_pending == jobs_failed
+              status = 'success' if batches_pending == 0 && jobs_pending == 0
+              status = 'deleted' if bid != root_bid && !batch.parent_bid
+
+              {
+                bid: bid,
+                created_at: r.hget("BID-#{bid}", 'created_at'),
+                status: status,
+                parent_bid: parent ? parent.bid : batch.parent_bid,
+                description: batch.description,
+                jobs: {
+                  pending_count: jobs_pending,
+                  successful_count: jobs_success,
+                  failed_count: jobs_failed,
+                  total_count: jobs_total,
+                  # items: batches.map{|b| layer_data[b] },
+                },
+                batches: {
+                  pending_count: batches_pending,
+                  successful_count: batches_success,
+                  failed_count: batches_failed,
+                  total_count: batches_total,
+                  items: layer[1].map{|b| layer_data[b, batch] },
+                },
+              }
+            }
+
+            data = layer_data[tree_bids]
+            data[:batches][:slice] = slice if slice
+            data
+          end
+        end
+
+        def format_context(batch)
+          bits = []
+          own_keys = batch.context.own.keys
+          batch.context.flatten.each do |k,v|
+            added = own_keys.include? k
+            bits << "  <span class=\"key #{added ? 'own' : 'inherited'}\">\"#{k}\": #{v.to_json},</span>"
+          end
+          bits = [
+            "{ // <span class=\"own\">Added</span> / <span class=\"inherited\">Inherited</span>",
+            *bits,
+            '}'
+          ]
+          bits.join("\n")
+        end
+      end
+
       app.post "/batches/all" do
         if params['delete']
           drain_zset('batches') do |batches|
@@ -107,6 +192,14 @@ module CanvasSync::JobBatches::Sidekiq
 end
 
 if defined?(::Sidekiq::Web)
+  rules = []
+  rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
+
+  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
+                                   root: File.expand_path("#{File.dirname(__FILE__)}/web"),
+                                   cascade: true,
+                                   header_rules: rules
+
   ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
   ::Sidekiq::Web.tabs["Batches"] = "batches"
   ::Sidekiq::Web.tabs["Pools"] = "pools"

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -46,7 +46,7 @@ module CanvasSync
           m = Regexp.last_match
           day = m[1]
           skip = m[2] || "1"
-          Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
+          DateTime.now.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
         when opt.match?(%r{^(\d+)\%$})
           m = Regexp.last_match
           rand(100) < m[1].to_i

data/lib/canvas_sync/jobs/report_checker.rb

@@ -4,8 +4,9 @@ module CanvasSync
     # Re-enqueues itself if the report is still processing on Canvas.
     # Enqueues the ReportProcessor when the report has completed.
     class ReportChecker < CanvasSync::Job
-      REPORT_TIMEOUT = 12.hours
+      REPORT_TIMEOUT = 24.hours
       COMPILATION_TIMEOUT = 1.hour
+      MAX_TRIES = 3
 
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_id [Integer]
@@ -13,6 +14,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(report_name, report_id, processor, options, checker_context = {}) # rubocop:disable Metrics/AbcSize
+        max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || MAX_TRIES
         account_id = options[:account_id] || batch_context[:account_id] || "self"
         report_status = CanvasSync.get_canvas_sync_client(batch_context)
                                   .report_status(account_id, report_name, report_id)
@@ -27,9 +29,17 @@ module CanvasSync
             report_id,
           )
         when "error", "deleted"
-          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}" # rubocop:disable Metrics/LineLength
+          checker_context[:failed_attempts] ||= 0
+          checker_context[:failed_attempts] += 1
+          failed_attempts = checker_context[:failed_attempts]
+          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}, #{current_organization.name}.  This report has now failed #{checker_context[:failed_attempts]} time." # rubocop:disable Metrics/LineLength
           Rails.logger.error(message)
-          raise message
+          if failed_attempts >= max_tries
+            Rails.logger.error("This report has failed #{failed_attempts} times.  Giving up.")
+            raise message
+          else
+            restart_report(options, report_name, processor, checker_context)
+          end
         else
           report_timeout = parse_timeout(options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT)
           if timeout_met?(options[:sync_start_time], report_timeout)
@@ -51,7 +61,7 @@ module CanvasSync
               report_id,
               processor,
               options,
-              checker_context,
+              checker_context
             )
         end
       end
@@ -66,6 +76,29 @@ module CanvasSync
       def parse_timeout(val)
         val
       end
+
+      def restart_report(options, report_name, processor, checker_context)
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
+        new_context = {}
+        new_context[:failed_attempts] = checker_context[:failed_attempts]
+        report_id = start_report(account_id, report_name, options[:report_params])
+        CanvasSync::Jobs::ReportChecker
+            .set(wait: report_checker_wait_time)
+            .perform_later(
+              report_name,
+              report_id,
+              processor,
+              options,
+              new_context
+            )
+      end
+
+      def start_report(account_id, report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(batch_context)
+                           .start_report(account_id, report_name, report_params)
+        report["id"]
+      end
     end
   end
 end