canvas_sync 0.17.16 → 0.17.20

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8a8e74e8ab5225dfe9c48d0c298213a7565a4faeee2e8652c3b1faa66f8df196
4
- data.tar.gz: '081468ec404d3042a0c447f992ded395440dc443d2c12a89c928e246113bb530'
3
+ metadata.gz: 761f46113fd5ade5684101672ca0fda352e0fbec602bb8fe0fdbcaf584e728ba
4
+ data.tar.gz: '0827e3a31d1e2bf7bf0afdece5a6a123f2221f2db9a4d239a4c543024641da57'
5
5
  SHA512:
6
- metadata.gz: f3744b1a9a3647f7f9e4d6a35f517b526e108a21cb4c953e52242861068d1862d4fc72f8f11dbd2464267b8dfebf06b46b686eb68cd2a7a2fd40f9cc1d7da7f6
7
- data.tar.gz: fa71d50cbafe3e690fec57d5f3ee0c470e7d3399a5f785d8feb050a2f7cefd479ceb01357911ce34d6a25dc34a1ba312b5015fcb9285bb63b07924dfdbb056b6
6
+ metadata.gz: 0ae25ea9b6305902f220604a4d040737cca48aa6a7d89960357179ad852951bcb99a93693c3fd684ef3a12b3db89874023f48f5ac6d54dac35edb3fd7eefcc57
7
+ data.tar.gz: ed161a60791e7d326faabea6431d6352eee48ea2c23dd0ff224e902ac9e545c881f18a3d0d31bffa186e95d5d840f605e492f58ef7b68d9583da7afd011dde89
data/README.md CHANGED
@@ -383,6 +383,27 @@ Available config options (if you add more, please update this!):
383
383
 
384
384
  * `config.classes_to_only_log_errors_on` - use this if you are utilizing the `CanvasSync::JobLog` table, but want certain classes to only persist in the `job_logs` table if an error is encountered. This is useful if you've got a very frequently used job that's filling up your database, and only really care about tracking failures.
385
385
 
386
+ ## Global Options
387
+ You can pass in global_options to a job chain. Global options are added to the batch_context and referenced by
388
+ various internal processes.
389
+
390
+ Pass global options into a job chain, using the options param nested in a :global key.
391
+ options: { global: {...} }
392
+
393
+ report_timeout (integer): Number of days until a Canvas report should timeout. Default is 1.
394
+ report_compilation_timeout (integer): Number of days until a Canvas report should timeout. Default is 1 hour.
395
+ You can likely pass a float to achieve sub-day timeouts, but not tested.
396
+ report_max_tries (integer): The number of times to attempt a report before giving up. A report is considered failed
397
+ if it has an 'error' status in Canvas or is deleted.
398
+
399
+ This is an example job chain with global options:
400
+ job_chain = CanvasSync.default_provisioning_report_chain(
401
+ MODELS_TO_SYNC,
402
+ term_scope: :active,
403
+ full_sync_every: 'sunday',
404
+ options: { global: { report_timeout: 2 } }
405
+ )
406
+
386
407
  ## Handling Job errors
387
408
 
388
409
  If you need custom handling for when a CanvasSync Job fails, you can add an `:on_failure` option to you Job Chain's `:global_options`.
@@ -2,7 +2,7 @@ module CanvasSync
2
2
  # An array that "processes" after so many items are added.
3
3
  #
4
4
  # Example Usage:
5
- # batches = BatchProcessor.new(of: 1000) do |batch|
5
+ # batches = CanvasSync::BatchProcessor.new(of: 1000) do |batch|
6
6
  # # Process the batch somehow
7
7
  # end
8
8
  # enumerator_of_some_kind.each { |item| batches << item }
@@ -30,27 +30,27 @@ module CanvasSync::Concerns
30
30
  end
31
31
 
32
32
  def bulk_sync_from_api_result(api_array, conflict_target: :canvas_id, import_args: {}, all_pages: true, batch_size: 1000)
33
- columns = api_sync_options.keys
33
+ columns = api_sync_options[:field_map].keys
34
34
 
35
35
  update_conditions = {
36
- condition: Importers::BulkImporter.condition_sql(self, columns),
36
+ condition: CanvasSync::Importers::BulkImporter.condition_sql(self, columns),
37
37
  columns: columns,
38
38
  }
39
39
  update_conditions[:conflict_target] = conflict_target if conflict_target.present?
40
40
  options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
41
41
 
42
42
  if all_pages
43
- batcher = BatchProcessor.new(of: batch_size) do |batch|
43
+ batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
44
44
  import(columns, batch, options)
45
45
  end
46
46
  api_array.all_pages_each do |api_item|
47
- item = new.assign_from_api_params(api_items)
47
+ item = new.assign_from_api_params(api_item)
48
48
  batcher << item
49
49
  end
50
50
  batcher.flush
51
51
  else
52
52
  items = api_array.map do |api_item|
53
- new.assign_from_api_params(api_items)
53
+ new.assign_from_api_params(api_item)
54
54
  end
55
55
  import(columns, batch, options)
56
56
  end
@@ -29,7 +29,7 @@ module CanvasSync
29
29
  database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
30
30
 
31
31
  row_ids = {}
32
- batcher = BatchProcessor.new(of: batch_size) do |batch|
32
+ batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
33
33
  row_ids = {}
34
34
  perform_import(klass, database_column_names, batch, database_conflict_column_name, import_args)
35
35
  end
@@ -96,7 +96,7 @@ module CanvasSync
96
96
  # started_at = Time.now
97
97
  # run_the_users_sync!
98
98
  # changed = User.where("updated_at >= ?", started_at)
99
- def self.condition_sql(klass, columns, report_start)
99
+ def self.condition_sql(klass, columns, report_start = nil)
100
100
  columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
101
101
  excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
102
102
  condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
@@ -28,6 +28,7 @@ module CanvasSync
28
28
 
29
29
  BID_EXPIRE_TTL = 2_592_000
30
30
  SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
31
+ BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
31
32
 
32
33
  attr_reader :bid
33
34
 
@@ -423,6 +424,14 @@ module CanvasSync
423
424
  def push_callbacks(args, queue)
424
425
  Batch::Callback::worker_class.enqueue_all(args, queue)
425
426
  end
427
+
428
+ def bid_hierarchy(bid, depth: 4, per_depth: 5, slice: nil)
429
+ args = [bid, depth, per_depth]
430
+ args << slice if slice
431
+ redis do |r|
432
+ BID_HIERARCHY.call(r, [], args)
433
+ end
434
+ end
426
435
  end
427
436
  end
428
437
 
@@ -172,6 +172,14 @@ module CanvasSync
172
172
  mapper[key] ||= []
173
173
  end
174
174
 
175
+ # TODO: Add a Chain progress web View
176
+ # Augment Batch tree-view with Chain data
177
+ # > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
178
+ # > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
179
+ # > Chain-jobs will be supplied chain_id and chain_step_id metadata
180
+ # > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
181
+ # > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
182
+ # > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
175
183
  def enqueue_job(job_def)
176
184
  job_class = job_def[:job].constantize
177
185
  job_options = job_def[:parameters] || []
@@ -0,0 +1,25 @@
1
+
2
+ local function add_bids(root, depth)
3
+ local result_data = {}
4
+
5
+ if depth > 0 then
6
+ local sbids
7
+ if depth == tonumber(ARGV[2]) and ARGV[4] then
8
+ local min, max = ARGV[4]:match('(%d+):(%d+)')
9
+ sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', min, max)
10
+ else
11
+ sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', 0, tonumber(ARGV[3]) - 1)
12
+ end
13
+
14
+ local sub_data = {}
15
+ for _,v in ipairs(sbids) do
16
+ table.insert(sub_data, add_bids(v, depth - 1))
17
+ end
18
+
19
+ return { root, sub_data }
20
+ end
21
+
22
+ return { root, result_data}
23
+ end
24
+
25
+ return add_bids(ARGV[1], tonumber(ARGV[2]))
@@ -0,0 +1,178 @@
1
+
2
+ @color-green: #25c766;
3
+ @color-red: #c7254e;
4
+ @color-yellow: #c4c725;
5
+
6
+ .code-wrap.batch-context .args-extended {
7
+ white-space: pre;
8
+
9
+ .key {
10
+ white-space: pre-wrap;
11
+ margin-left: 2em;
12
+ text-indent: -2em;
13
+ display: inline-block;
14
+ }
15
+
16
+ .own {
17
+ color: @color-green;
18
+ }
19
+ }
20
+
21
+
22
+ .batch-tree {
23
+ .status-block {
24
+ .tree-stat {
25
+ margin: 0 4px;
26
+
27
+ &.pending {
28
+ color: @color-yellow;
29
+ }
30
+ &.failed {
31
+ color: @color-red;
32
+ }
33
+ &.success {
34
+ color: @color-green;
35
+ }
36
+ &.total {
37
+
38
+ }
39
+ }
40
+ }
41
+
42
+ .text-inactive {
43
+ color: darken(#fff, 35%);
44
+ font-size: 80%;
45
+ }
46
+
47
+ .tree-header {
48
+ position: relative;
49
+
50
+ .status-block {
51
+ position: absolute;
52
+ bottom: 0;
53
+ width: 100%;
54
+
55
+ margin-right: 8px;
56
+ font-size: 90%;
57
+ text-align: right;
58
+
59
+ .tree-stat {
60
+ font-style: italic;
61
+ }
62
+ }
63
+ }
64
+
65
+ .tree-entry {
66
+ > .header {
67
+ display: flex;
68
+ align-items: center;
69
+
70
+ .header-inner {
71
+ padding: 4px 0;
72
+ border-bottom: 1px dashed white;
73
+ display: flex;
74
+ align-items: center;
75
+ flex: 1;
76
+ }
77
+
78
+ &:hover {
79
+ background-color: rgba(0,0,0,0.20);
80
+ border-radius: 3px;
81
+ }
82
+
83
+ .row-toggle {
84
+ width: 16px;
85
+ height: 16px;
86
+ text-align: center;
87
+ align-self: center;
88
+ border-radius: 50%;
89
+ border: 1px solid #999;
90
+ text-decoration: none;
91
+ margin: 0 4px;
92
+ font-size: 16px;
93
+ line-height: 15px;
94
+
95
+ &.not_applicable {
96
+ opacity: 0;
97
+ pointer-events: none;
98
+ }
99
+ }
100
+
101
+ .main {
102
+ flex: 1;
103
+ display: flex;
104
+ align-items: baseline;
105
+
106
+ .bid {
107
+ font-family: monospace;
108
+ padding: 3px 6px;
109
+ background: rgba(0,0,0,0.2);
110
+ border-radius: 3px;
111
+ font-size: 12px;
112
+ margin: 0 12px 0 0;
113
+
114
+ &:hover {
115
+ .bid-goto {
116
+ display: inline-block;
117
+ padding: 0 0 0 4px;
118
+ font-size: 200%;
119
+ line-height: 10px;
120
+ vertical-align: sub;
121
+ text-decoration: dotted;
122
+ }
123
+ }
124
+
125
+ .bid-goto {
126
+ display: none;
127
+ }
128
+ }
129
+ }
130
+
131
+ .goto-link {
132
+ margin: 0 8px;
133
+ display: inline-block;
134
+ height: 16px;
135
+ font-size: 90%;
136
+ border-bottom: 1px dotted white;
137
+ }
138
+
139
+ .status-label {
140
+ font-family: monospace;
141
+ padding: 3px 6px;
142
+ background: rgba(0,0,0,0.2);
143
+ border-radius: 3px;
144
+ font-size: 12px;
145
+ margin: 0 12px 0 0;
146
+
147
+ &.deleted {
148
+ background: #99999933;
149
+ }
150
+ &.failed, &.complete {
151
+ background: #99000033;
152
+ }
153
+ &.success {
154
+ background: #00990033;
155
+ }
156
+ }
157
+
158
+ .status-block {
159
+ width: 10em;
160
+ text-align: center;
161
+ }
162
+ }
163
+
164
+ > .subitems {
165
+ padding-left: 16px;
166
+
167
+ >.load-more {
168
+ padding: 4px 0;
169
+ text-align: center;
170
+ border-bottom: 1px dashed white;
171
+ a {
172
+ border-bottom: 1px dotted white;
173
+ text-decoration: none;
174
+ }
175
+ }
176
+ }
177
+ }
178
+ }
@@ -0,0 +1,106 @@
1
+ import { h, Component, render } from 'https://unpkg.com/preact?module';
2
+ import htm from 'https://unpkg.com/htm?module';
3
+ import { root_url } from './util.js';
4
+
5
+ // Initialize htm with Preact
6
+ const html = htm.bind(h);
7
+
8
+ const StatusBlock = (props) => html`
9
+ <div class="status-block ${props.class || ''}">
10
+ ${props.title && props.title + ':'}
11
+ <span class="tree-stat pending">${props.pending_count}</span>
12
+ |
13
+ <span class="tree-stat failed">${props.failed_count}</span>
14
+ |
15
+ <span class="tree-stat success">${props.successful_count}</span>
16
+ /
17
+ <span class="tree-stat total">${props.total_count}</span>
18
+ </div>
19
+ `
20
+
21
+ class TreeLevel extends Component {
22
+ get bid() {
23
+ return this.props.data.bid;
24
+ }
25
+
26
+ get batch() {
27
+ return this.props.data;
28
+ }
29
+
30
+ load_more = async (event) => {
31
+ event.preventDefault();
32
+ const l = this.batch.batches.items.length;
33
+ const resp = await fetch(`${root_url}batches/${this.bid}/tree?slice=${l}:${l + 5 - 1}`)
34
+ const result = await resp.json()
35
+ const newEntries = result.batches.items;
36
+ for (let ent of newEntries) {
37
+ this.batch.batches.items.push(ent)
38
+ }
39
+ this.forceUpdate()
40
+ }
41
+
42
+ toggle = (event) => {
43
+ event.preventDefault();
44
+ this.setState({ collapsed: !this.state.collapsed })
45
+ }
46
+
47
+ render() {
48
+ const { data: bd } = this.props;
49
+
50
+ let sub_entries = [];
51
+ let sub_batches = bd.batches.items;
52
+ for (let b of sub_batches) {
53
+ sub_entries.push(html`<${TreeLevel} data=${b} />`)
54
+ }
55
+
56
+ let fully_loaded = !(sub_batches.length < bd.batches.total_count);
57
+
58
+ const load_more_elem = html`<div class="load-more">
59
+ ${sub_entries.length} / ${bd.batches.total_count} Items Loaded - <a href="#" onClick=${this.load_more}>Load More</a>
60
+ </div>`
61
+
62
+ return html`<div class="tree-entry tree-batch">
63
+ <div class="header">
64
+ <a class="row-toggle ${!bd.batches.total_count && 'not_applicable'}" onClick=${this.toggle} href="#">
65
+ ${this.state.collapsed ? '+' : '-'}
66
+ </a>
67
+
68
+ <div class="header-inner">
69
+ <div class="main">
70
+ <span class="bid">
71
+ ${bd.bid}
72
+ <a class="bid-goto" href="${root_url}batches/${bd.bid}">⇢</a>
73
+ </span>
74
+ ${bd.description || (bd.status == 'deleted' && html`<i class="text-inactive">Deleted</i>`) || html`<i class="text-inactive">No Description</i>`}
75
+ </div>
76
+
77
+ <span class="status-label ${bd.status}">${bd.status}</span>
78
+
79
+ ${bd.status != 'deleted' && html`
80
+ <${StatusBlock} class="job-status" title="Jobs" ...${bd.jobs} />
81
+ <${StatusBlock} class="batch-status" title="Batches" ...${bd.batches} />
82
+ `}
83
+ </div>
84
+ </div>
85
+
86
+ <div class="subitems ${this.state.collapsed ? 'hidden' : ''}">
87
+ ${sub_entries}
88
+ ${!fully_loaded && load_more_elem}
89
+ </div>
90
+ </div>`
91
+ }
92
+ }
93
+
94
+ class TreeRoot extends Component {
95
+ render() {
96
+ const tree_data = JSON.parse(document.querySelector('#batch-tree #initial-data').innerHTML);
97
+ return html`
98
+ <div class="tree-header">
99
+ <${StatusBlock} pending_count="pending" failed_count="failed" successful_count="successful" total_count="total" />
100
+ </div>
101
+ <${TreeLevel} data=${tree_data} />
102
+ `;
103
+ }
104
+ }
105
+
106
+ render(html`<${TreeRoot} />`, document.querySelector('#batch-tree'));
@@ -0,0 +1,2 @@
1
+
2
+ export const root_url = (document.querySelector('meta[name="sidekiq-baseurl"]')).content;
@@ -0,0 +1,6 @@
1
+
2
+ <div id="batch-tree" class="batch-tree">
3
+ <script id="initial-data" type="application/json"><%= @tree_data.to_json %></script>
4
+ </div>
5
+
6
+ <script type="module" src="<%= root_path %>batches_assets/js/batch_tree.js"></script>
@@ -0,0 +1,13 @@
1
+
2
+ <% add_to_head do %>
3
+ <meta name="sidekiq-baseurl" content="<%= root_path %>" />
4
+ <% if dev_mode? %>
5
+ <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
6
+ <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
7
+ <% else %>
8
+ <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
9
+ <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
10
+ <%# TODO: Pre-compile LESS %>
11
+ <!-- <link href="<%= root_path %>batches_assets/css/styles.css" media="screen" rel="stylesheet" type="text/css" /> -->
12
+ <% end %>
13
+ <% end %>
@@ -1,3 +1,5 @@
1
+ <%= erb get_template(:_common) %>
2
+
1
3
  <h3><%= t('Batch') %></h3>
2
4
  <% status = CanvasSync::JobBatches::Batch::Status.new(@batch) %>
3
5
 
@@ -17,70 +19,13 @@
17
19
  <td><%= @batch.description %></td>
18
20
  </tr>
19
21
  <tr>
20
- <th colspan="2" scope=row><%= t('Added Context') %></td>
22
+ <th colspan="2" scope=row><%= t('Context') %></td>
21
23
  <td>
22
- <code class="code-wrap">
23
- <div class="args-extended"><%= @batch.context.own.to_json %></div>
24
+ <code class="code-wrap batch-context">
25
+ <div class="args-extended"><%= format_context(@batch) %></div>
24
26
  </code>
25
27
  </td>
26
28
  </tr>
27
- <tr>
28
- <th colspan="2" scope=row><%= t('Full Context') %></td>
29
- <td>
30
- <code class="code-wrap">
31
- <div class="args-extended"><%= @batch.context.flatten.to_json %></div>
32
- </code>
33
- </td>
34
- </tr>
35
-
36
- <tr>
37
- <th colspan="3">Jobs</th>
38
- </tr>
39
- <tr>
40
- <th></th>
41
- <th><%= t('Pending') %></th>
42
- <td><%= status.pending %></th>
43
- </tr>
44
- <tr>
45
- <th></th>
46
- <th><%= t('Failed') %></th>
47
- <td><%= status.failures %></th>
48
- </tr>
49
- <tr>
50
- <th></th>
51
- <th><%= t('Complete') %></th>
52
- <td><%= status.completed_count %></th>
53
- </tr>
54
- <tr>
55
- <th></th>
56
- <th><%= t('Total') %></th>
57
- <td><%= status.job_count %></th>
58
- </tr>
59
-
60
- <tr>
61
- <th colspan="3">Batches</th>
62
- </tr>
63
- <tr>
64
- <th></th>
65
- <th><%= t('Pending') %></th>
66
- <td><%= status.child_count - status.successful_children_count %></th>
67
- </tr>
68
- <tr>
69
- <th></th>
70
- <th><%= t('Failed') %></th>
71
- <td><%= status.failed_children_count %></th>
72
- </tr>
73
- <tr>
74
- <th></th>
75
- <th><%= t('Success') %></th>
76
- <td><%= status.successful_children_count %></th>
77
- </tr>
78
- <tr>
79
- <th></th>
80
- <th><%= t('Total') %></th>
81
- <td><%= status.child_count %></th>
82
- </tr>
83
-
84
29
  </tbody>
85
30
  </table>
86
31
  </div>
@@ -88,51 +33,33 @@
88
33
  <header class="row">
89
34
  <div class="col-sm-5">
90
35
  <h3>
91
- <%= t('Jobs') %>
36
+ <%= t('Child Batches') %>
92
37
  </h3>
93
38
  </div>
94
- <%
95
- @current_page = @current_jobs_page
96
- @total_size = @total_jobs_size
97
- %>
98
- <% if @jobs.any? && @total_size > @count.to_i %>
99
- <div class="col-sm-4">
100
- <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
101
- </div>
102
- <% end %>
103
39
  </header>
104
40
 
105
- <% if @jobs.any? %>
106
- <div class="table_container">
107
- <%= erb get_template(:_jobs_table), locals: { jobs: @jobs } %>
108
- </div>
109
- <% end %>
41
+ <%= erb get_template(:_batch_tree), locals: { } %>
110
42
 
111
43
  <header class="row">
112
44
  <div class="col-sm-5">
113
45
  <h3>
114
- <%= t('Child Batches') %>
46
+ <%= t('Jobs') %>
47
+ <i>(Queued and Current)</i>
115
48
  </h3>
116
49
  </div>
117
50
  <%
118
- @current_page = @current_batches_page
119
- @total_size = @total_batches_size
51
+ @current_page = @current_jobs_page
52
+ @total_size = @total_jobs_size
120
53
  %>
121
- <% if @sub_batches.any? && @total_size > @count.to_i %>
54
+ <% if @jobs.any? && @total_size > @count.to_i %>
122
55
  <div class="col-sm-4">
123
- <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :batch_page } %>
56
+ <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
124
57
  </div>
125
58
  <% end %>
126
59
  </header>
127
60
 
128
- <% if @sub_batches.any? %>
61
+ <% if @jobs.any? %>
129
62
  <div class="table_container">
130
- <%= erb get_template(:_batches_table), locals: { batches: @sub_batches } %>
63
+ <%= erb get_template(:_jobs_table), locals: { jobs: @jobs } %>
131
64
  </div>
132
65
  <% end %>
133
-
134
- <form class="form-horizontal" action="<%= root_path %>batches/<%= @batch.bid %>" method="post">
135
- <%= csrf_tag %>
136
- <a class="btn btn-default" href="<%= root_path %>batches"><%= t('GoBack') %></a>
137
- <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
138
- </form>
@@ -9,11 +9,19 @@ require_relative "web/helpers"
9
9
 
10
10
  module CanvasSync::JobBatches::Sidekiq
11
11
  module Web
12
+ DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
13
+
12
14
  def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
13
15
  app.helpers do
14
16
  include Web::Helpers
17
+
18
+ def dev_mode?
19
+ DEV_MODE
20
+ end
15
21
  end
16
22
 
23
+ # =============== BATCHES =============== #
24
+
17
25
  app.get "/batches" do
18
26
  @count = (params['count'] || 25).to_i
19
27
  @current_page, @total_size, @batches = page('batches', params['page'], @count)
@@ -26,6 +34,8 @@ module CanvasSync::JobBatches::Sidekiq
26
34
  @bid = params[:bid]
27
35
  @batch = CanvasSync::JobBatches::Batch.new(@bid)
28
36
 
37
+ @tree_data = tree_data(@bid)
38
+
29
39
  @count = (params['count'] || 25).to_i
30
40
  @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
31
41
  @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
@@ -36,6 +46,81 @@ module CanvasSync::JobBatches::Sidekiq
36
46
  erb(get_template(:batch))
37
47
  end
38
48
 
49
+ app.get "/batches/:bid/tree" do
50
+ @bid = params[:bid]
51
+
52
+ json(tree_data(@bid, slice: params[:slice]))
53
+ end
54
+
55
+ app.helpers do
56
+ def tree_data(root_bid, slice: nil)
57
+ tree_bids = CanvasSync::JobBatches::Batch.bid_hierarchy(root_bid, slice: slice)
58
+
59
+ CanvasSync::JobBatches::Batch.redis do |r|
60
+ layer_data = ->(layer, parent = nil) {
61
+ bid = layer[0]
62
+ batch = CanvasSync::JobBatches::Batch.new(bid)
63
+
64
+ jobs_total = r.hget("BID-#{bid}", "job_count").to_i
65
+ jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
66
+ jobs_failed = r.scard("BID-#{bid}-failed").to_i
67
+ jobs_success = jobs_total - jobs_pending
68
+
69
+ batches_total = r.hget("BID-#{bid}", 'children').to_i
70
+ batches_success = r.scard("BID-#{bid}-batches-success").to_i
71
+ batches_pending = batches_total - batches_success
72
+ batches_failed = r.scard("BID-#{bid}-batches-failed").to_i
73
+
74
+ status = 'in_progress'
75
+ status = 'complete' if batches_pending == batches_failed && jobs_pending == jobs_failed
76
+ status = 'success' if batches_pending == 0 && jobs_pending == 0
77
+ status = 'deleted' if bid != root_bid && !batch.parent_bid
78
+
79
+ {
80
+ bid: bid,
81
+ created_at: r.hget("BID-#{bid}", 'created_at'),
82
+ status: status,
83
+ parent_bid: parent ? parent.bid : batch.parent_bid,
84
+ description: batch.description,
85
+ jobs: {
86
+ pending_count: jobs_pending,
87
+ successful_count: jobs_success,
88
+ failed_count: jobs_failed,
89
+ total_count: jobs_total,
90
+ # items: batches.map{|b| layer_data[b] },
91
+ },
92
+ batches: {
93
+ pending_count: batches_pending,
94
+ successful_count: batches_success,
95
+ failed_count: batches_failed,
96
+ total_count: batches_total,
97
+ items: layer[1].map{|b| layer_data[b, batch] },
98
+ },
99
+ }
100
+ }
101
+
102
+ data = layer_data[tree_bids]
103
+ data[:batches][:slice] = slice if slice
104
+ data
105
+ end
106
+ end
107
+
108
+ def format_context(batch)
109
+ bits = []
110
+ own_keys = batch.context.own.keys
111
+ batch.context.flatten.each do |k,v|
112
+ added = own_keys.include? k
113
+ bits << " <span class=\"key #{added ? 'own' : 'inherited'}\">\"#{k}\": #{v.to_json},</span>"
114
+ end
115
+ bits = [
116
+ "{ // <span class=\"own\">Added</span> / <span class=\"inherited\">Inherited</span>",
117
+ *bits,
118
+ '}'
119
+ ]
120
+ bits.join("\n")
121
+ end
122
+ end
123
+
39
124
  app.post "/batches/all" do
40
125
  if params['delete']
41
126
  drain_zset('batches') do |batches|
@@ -107,6 +192,14 @@ module CanvasSync::JobBatches::Sidekiq
107
192
  end
108
193
 
109
194
  if defined?(::Sidekiq::Web)
195
+ rules = []
196
+ rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
197
+
198
+ ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
199
+ root: File.expand_path("#{File.dirname(__FILE__)}/web"),
200
+ cascade: true,
201
+ header_rules: rules
202
+
110
203
  ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
111
204
  ::Sidekiq::Web.tabs["Batches"] = "batches"
112
205
  ::Sidekiq::Web.tabs["Pools"] = "pools"
@@ -46,7 +46,7 @@ module CanvasSync
46
46
  m = Regexp.last_match
47
47
  day = m[1]
48
48
  skip = m[2] || "1"
49
- Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
49
+ DateTime.now.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
50
50
  when opt.match?(%r{^(\d+)\%$})
51
51
  m = Regexp.last_match
52
52
  rand(100) < m[1].to_i
@@ -4,8 +4,9 @@ module CanvasSync
4
4
  # Re-enqueues itself if the report is still processing on Canvas.
5
5
  # Enqueues the ReportProcessor when the report has completed.
6
6
  class ReportChecker < CanvasSync::Job
7
- REPORT_TIMEOUT = 12.hours
7
+ REPORT_TIMEOUT = 24.hours
8
8
  COMPILATION_TIMEOUT = 1.hour
9
+ MAX_TRIES = 3
9
10
 
10
11
  # @param report_name [Hash] e.g., 'provisioning_csv'
11
12
  # @param report_id [Integer]
@@ -13,6 +14,7 @@ module CanvasSync
13
14
  # @param options [Hash] hash of options that will be passed to the job processor
14
15
  # @return [nil]
15
16
  def perform(report_name, report_id, processor, options, checker_context = {}) # rubocop:disable Metrics/AbcSize
17
+ max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || MAX_TRIES
16
18
  account_id = options[:account_id] || batch_context[:account_id] || "self"
17
19
  report_status = CanvasSync.get_canvas_sync_client(batch_context)
18
20
  .report_status(account_id, report_name, report_id)
@@ -27,9 +29,17 @@ module CanvasSync
27
29
  report_id,
28
30
  )
29
31
  when "error", "deleted"
30
- message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}" # rubocop:disable Metrics/LineLength
32
+ checker_context[:failed_attempts] ||= 0
33
+ checker_context[:failed_attempts] += 1
34
+ failed_attempts = checker_context[:failed_attempts]
35
+ message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}, #{current_organization.name}. This report has now failed #{checker_context[:failed_attempts]} time." # rubocop:disable Metrics/LineLength
31
36
  Rails.logger.error(message)
32
- raise message
37
+ if failed_attempts >= max_tries
38
+ Rails.logger.error("This report has failed #{failed_attempts} times. Giving up.")
39
+ raise message
40
+ else
41
+ restart_report(options, report_name, processor, checker_context)
42
+ end
33
43
  else
34
44
  report_timeout = parse_timeout(options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT)
35
45
  if timeout_met?(options[:sync_start_time], report_timeout)
@@ -51,7 +61,7 @@ module CanvasSync
51
61
  report_id,
52
62
  processor,
53
63
  options,
54
- checker_context,
64
+ checker_context
55
65
  )
56
66
  end
57
67
  end
@@ -66,6 +76,29 @@ module CanvasSync
66
76
  def parse_timeout(val)
67
77
  val
68
78
  end
79
+
80
+ def restart_report(options, report_name, processor, checker_context)
81
+ account_id = options[:account_id] || batch_context[:account_id] || "self"
82
+ options[:sync_start_time] = DateTime.now.utc.iso8601
83
+ new_context = {}
84
+ new_context[:failed_attempts] = checker_context[:failed_attempts]
85
+ report_id = start_report(account_id, report_name, options[:report_params])
86
+ CanvasSync::Jobs::ReportChecker
87
+ .set(wait: report_checker_wait_time)
88
+ .perform_later(
89
+ report_name,
90
+ report_id,
91
+ processor,
92
+ options,
93
+ new_context
94
+ )
95
+ end
96
+
97
+ def start_report(account_id, report_name, report_params)
98
+ report = CanvasSync.get_canvas_sync_client(batch_context)
99
+ .start_report(account_id, report_name, report_params)
100
+ report["id"]
101
+ end
69
102
  end
70
103
  end
71
104
  end
@@ -12,7 +12,7 @@ module CanvasSync
12
12
  def perform(report_name, report_params, processor, options, allow_redownloads: false)
13
13
  account_id = options[:account_id] || batch_context[:account_id] || "self"
14
14
  options[:sync_start_time] = DateTime.now.utc.iso8601
15
-
15
+ options[:report_params] = report_params
16
16
  report_id = start_report(account_id, report_name, report_params)
17
17
  # TODO: Restore report caching support (does nayone actually use it?)
18
18
  # report_id = if allow_redownloads
@@ -28,7 +28,7 @@ module CanvasSync
28
28
  report_name,
29
29
  report_id,
30
30
  processor,
31
- options,
31
+ options
32
32
  )
33
33
  end
34
34
  end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.16".freeze
2
+ VERSION = "0.17.20".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.16
4
+ version: 0.17.20
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nate Collings
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-09 00:00:00.000000000 Z
11
+ date: 2021-08-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -442,6 +442,7 @@ files:
442
442
  - lib/canvas_sync/job_batches/callback.rb
443
443
  - lib/canvas_sync/job_batches/chain_builder.rb
444
444
  - lib/canvas_sync/job_batches/context_hash.rb
445
+ - lib/canvas_sync/job_batches/hier_batch_ids.lua
445
446
  - lib/canvas_sync/job_batches/hincr_max.lua
446
447
  - lib/canvas_sync/job_batches/jobs/base_job.rb
447
448
  - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
@@ -453,8 +454,13 @@ files:
453
454
  - lib/canvas_sync/job_batches/schedule_callback.lua
454
455
  - lib/canvas_sync/job_batches/sidekiq.rb
455
456
  - lib/canvas_sync/job_batches/sidekiq/web.rb
457
+ - lib/canvas_sync/job_batches/sidekiq/web/batches_assets/css/styles.less
458
+ - lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/batch_tree.js
459
+ - lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/util.js
456
460
  - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb
461
+ - lib/canvas_sync/job_batches/sidekiq/web/views/_batch_tree.erb
457
462
  - lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb
463
+ - lib/canvas_sync/job_batches/sidekiq/web/views/_common.erb
458
464
  - lib/canvas_sync/job_batches/sidekiq/web/views/_jobs_table.erb
459
465
  - lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb
460
466
  - lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb