canvas_sync 0.16.4 → 0.21.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (260) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +235 -151
  3. data/app/controllers/{api → canvas_sync/api}/v1/health_check_controller.rb +1 -1
  4. data/app/controllers/canvas_sync/api/v1/live_events_controller.rb +122 -0
  5. data/app/models/canvas_sync/sync_batch.rb +5 -0
  6. data/config/initializers/apartment.rb +10 -1
  7. data/config/routes.rb +7 -0
  8. data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
  9. data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
  10. data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
  11. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  12. data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb +7 -0
  13. data/lib/canvas_sync/batch_processor.rb +41 -0
  14. data/lib/canvas_sync/concerns/ability_helper.rb +72 -0
  15. data/lib/canvas_sync/concerns/account/ancestry.rb +2 -0
  16. data/lib/canvas_sync/concerns/account/base.rb +15 -0
  17. data/lib/canvas_sync/concerns/api_syncable.rb +17 -10
  18. data/lib/canvas_sync/concerns/live_event_sync.rb +46 -0
  19. data/lib/canvas_sync/concerns/role/base.rb +57 -0
  20. data/lib/canvas_sync/concerns/sync_mapping.rb +120 -0
  21. data/lib/canvas_sync/engine.rb +80 -0
  22. data/lib/canvas_sync/generators/install_generator.rb +1 -0
  23. data/lib/canvas_sync/generators/install_live_events_generator.rb +0 -1
  24. data/lib/canvas_sync/generators/templates/migrations/create_content_migrations.rb +24 -0
  25. data/lib/canvas_sync/generators/templates/migrations/create_course_nicknames.rb +17 -0
  26. data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb +18 -0
  27. data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb +22 -0
  28. data/lib/canvas_sync/generators/templates/migrations/create_learning_outcome_results.rb +46 -0
  29. data/lib/canvas_sync/generators/templates/migrations/create_learning_outcomes.rb +30 -0
  30. data/lib/canvas_sync/generators/templates/migrations/create_rubric_assessments.rb +31 -0
  31. data/lib/canvas_sync/generators/templates/migrations/create_rubric_associations.rb +36 -0
  32. data/lib/canvas_sync/generators/templates/migrations/create_rubrics.rb +38 -0
  33. data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb +17 -0
  34. data/lib/canvas_sync/generators/templates/migrations/create_users.rb +0 -1
  35. data/lib/canvas_sync/generators/templates/models/account.rb +3 -0
  36. data/lib/canvas_sync/generators/templates/models/admin.rb +2 -0
  37. data/lib/canvas_sync/generators/templates/models/assignment.rb +3 -0
  38. data/lib/canvas_sync/generators/templates/models/assignment_group.rb +3 -0
  39. data/lib/canvas_sync/generators/templates/models/content_migration.rb +12 -0
  40. data/lib/canvas_sync/generators/templates/models/context_module.rb +3 -0
  41. data/lib/canvas_sync/generators/templates/models/context_module_item.rb +3 -0
  42. data/lib/canvas_sync/generators/templates/models/course.rb +11 -0
  43. data/lib/canvas_sync/generators/templates/models/course_nickname.rb +13 -0
  44. data/lib/canvas_sync/generators/templates/models/enrollment.rb +14 -0
  45. data/lib/canvas_sync/generators/templates/models/grading_period.rb +10 -0
  46. data/lib/canvas_sync/generators/templates/models/grading_period_group.rb +9 -0
  47. data/lib/canvas_sync/generators/templates/models/group.rb +2 -0
  48. data/lib/canvas_sync/generators/templates/models/group_membership.rb +2 -0
  49. data/lib/canvas_sync/generators/templates/models/learning_outcome.rb +24 -0
  50. data/lib/canvas_sync/generators/templates/models/learning_outcome_result.rb +48 -0
  51. data/lib/canvas_sync/generators/templates/models/pseudonym.rb +2 -0
  52. data/lib/canvas_sync/generators/templates/models/role.rb +2 -0
  53. data/lib/canvas_sync/generators/templates/models/rubric.rb +29 -0
  54. data/lib/canvas_sync/generators/templates/models/rubric_assessment.rb +17 -0
  55. data/lib/canvas_sync/generators/templates/models/rubric_association.rb +14 -0
  56. data/lib/canvas_sync/generators/templates/models/section.rb +9 -0
  57. data/lib/canvas_sync/generators/templates/models/submission.rb +3 -0
  58. data/lib/canvas_sync/generators/templates/models/term.rb +3 -0
  59. data/lib/canvas_sync/generators/templates/models/user.rb +11 -0
  60. data/lib/canvas_sync/generators/templates/models/user_observer.rb +13 -0
  61. data/lib/canvas_sync/generators/templates/services/live_events/assignment_event.rb +1 -1
  62. data/lib/canvas_sync/generators/templates/services/live_events/assignment_group_event.rb +1 -1
  63. data/lib/canvas_sync/generators/templates/services/live_events/course_event.rb +1 -3
  64. data/lib/canvas_sync/generators/templates/services/live_events/course_section_event.rb +1 -1
  65. data/lib/canvas_sync/generators/templates/services/live_events/enrollment_event.rb +1 -1
  66. data/lib/canvas_sync/generators/templates/services/live_events/grade_event.rb +1 -1
  67. data/lib/canvas_sync/generators/templates/services/live_events/module_event.rb +1 -1
  68. data/lib/canvas_sync/generators/templates/services/live_events/module_item_event.rb +1 -1
  69. data/lib/canvas_sync/generators/templates/services/live_events/submission_event.rb +1 -1
  70. data/lib/canvas_sync/generators/templates/services/live_events/syllabus_event.rb +1 -1
  71. data/lib/canvas_sync/generators/templates/services/live_events/user_event.rb +1 -3
  72. data/lib/canvas_sync/importers/bulk_importer.rb +138 -31
  73. data/lib/canvas_sync/job.rb +7 -5
  74. data/lib/canvas_sync/job_batches/active_job.rb +108 -0
  75. data/lib/canvas_sync/job_batches/batch.rb +543 -0
  76. data/lib/canvas_sync/job_batches/callback.rb +149 -0
  77. data/lib/canvas_sync/job_batches/chain_builder.rb +249 -0
  78. data/lib/canvas_sync/job_batches/context_hash.rb +159 -0
  79. data/lib/canvas_sync/job_batches/hier_batch_ids.lua +25 -0
  80. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  81. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +22 -0
  82. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +170 -0
  83. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +22 -0
  84. data/lib/canvas_sync/job_batches/pool.rb +245 -0
  85. data/lib/canvas_sync/job_batches/pool_refill.lua +47 -0
  86. data/lib/canvas_sync/job_batches/redis_model.rb +69 -0
  87. data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
  88. data/lib/canvas_sync/job_batches/schedule_callback.lua +14 -0
  89. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/css/styles.less +182 -0
  90. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/batch_tree.js +108 -0
  91. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/util.js +2 -0
  92. data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +41 -0
  93. data/lib/canvas_sync/job_batches/sidekiq/web/views/_batch_tree.erb +6 -0
  94. data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb +44 -0
  95. data/lib/canvas_sync/job_batches/sidekiq/web/views/_common.erb +13 -0
  96. data/lib/canvas_sync/job_batches/sidekiq/web/views/_jobs_table.erb +21 -0
  97. data/lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb +26 -0
  98. data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb +81 -0
  99. data/lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb +23 -0
  100. data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb +137 -0
  101. data/lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb +47 -0
  102. data/lib/canvas_sync/job_batches/sidekiq/web.rb +218 -0
  103. data/lib/canvas_sync/job_batches/sidekiq.rb +136 -0
  104. data/lib/canvas_sync/job_batches/status.rb +91 -0
  105. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +99 -0
  106. data/lib/canvas_sync/jobs/canvas_process_waiter.rb +41 -0
  107. data/lib/canvas_sync/jobs/report_checker.rb +70 -8
  108. data/lib/canvas_sync/jobs/report_processor_job.rb +4 -7
  109. data/lib/canvas_sync/jobs/report_starter.rb +34 -20
  110. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  111. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  112. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  113. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  114. data/lib/canvas_sync/jobs/sync_content_migrations_job.rb +20 -0
  115. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  116. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  117. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +16 -50
  118. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  119. data/lib/canvas_sync/jobs/sync_rubric_assessments_job.rb +15 -0
  120. data/lib/canvas_sync/jobs/sync_rubric_associations_job.rb +15 -0
  121. data/lib/canvas_sync/jobs/sync_rubrics_job.rb +15 -0
  122. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  123. data/lib/canvas_sync/jobs/sync_submissions_job.rb +6 -4
  124. data/lib/canvas_sync/jobs/sync_terms_job.rb +9 -8
  125. data/lib/canvas_sync/jobs/term_batches_job.rb +50 -0
  126. data/lib/canvas_sync/{generators/templates/services/live_events/base_event.rb → live_events/base_handler.rb} +6 -10
  127. data/lib/canvas_sync/live_events/process_event_job.rb +26 -0
  128. data/lib/canvas_sync/live_events.rb +38 -0
  129. data/lib/canvas_sync/misc_helper.rb +63 -0
  130. data/lib/canvas_sync/processors/assignment_groups_processor.rb +3 -8
  131. data/lib/canvas_sync/processors/assignments_processor.rb +3 -8
  132. data/lib/canvas_sync/processors/content_migrations_processor.rb +19 -0
  133. data/lib/canvas_sync/processors/context_module_items_processor.rb +3 -8
  134. data/lib/canvas_sync/processors/context_modules_processor.rb +3 -8
  135. data/lib/canvas_sync/processors/model_mappings.yml +420 -0
  136. data/lib/canvas_sync/processors/normal_processor.rb +3 -3
  137. data/lib/canvas_sync/processors/provisioning_report_processor.rb +42 -55
  138. data/lib/canvas_sync/processors/report_processor.rb +15 -9
  139. data/lib/canvas_sync/processors/rubric_assessments_processor.rb +19 -0
  140. data/lib/canvas_sync/processors/rubric_associations_processor.rb +19 -0
  141. data/lib/canvas_sync/processors/rubrics_processor.rb +19 -0
  142. data/lib/canvas_sync/processors/submissions_processor.rb +3 -8
  143. data/lib/canvas_sync/record.rb +103 -0
  144. data/lib/canvas_sync/version.rb +1 -1
  145. data/lib/canvas_sync.rb +124 -125
  146. data/spec/canvas_sync/canvas_sync_spec.rb +224 -155
  147. data/spec/canvas_sync/jobs/canvas_process_waiter_spec.rb +34 -0
  148. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  149. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  150. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  151. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  152. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  153. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  154. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  155. data/spec/canvas_sync/jobs/sync_content_migrations_job_spec.rb +30 -0
  156. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  157. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  158. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +7 -41
  159. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  160. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  161. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +8 -2
  162. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  163. data/spec/canvas_sync/live_events/live_event_sync_spec.rb +27 -0
  164. data/spec/canvas_sync/live_events/live_events_controller_spec.rb +54 -0
  165. data/spec/canvas_sync/live_events/process_event_job_spec.rb +38 -0
  166. data/spec/canvas_sync/misc_helper_spec.rb +58 -0
  167. data/spec/canvas_sync/models/assignment_spec.rb +1 -1
  168. data/spec/canvas_sync/processors/content_migrations_processor_spec.rb +13 -0
  169. data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb +101 -1
  170. data/spec/canvas_sync/processors/rubric_assessments_spec.rb +16 -0
  171. data/spec/canvas_sync/processors/rubric_associations_spec.rb +16 -0
  172. data/spec/canvas_sync/processors/rubrics_processor_spec.rb +17 -0
  173. data/spec/dummy/app/models/account.rb +6 -0
  174. data/spec/dummy/app/models/admin.rb +2 -0
  175. data/spec/dummy/app/models/assignment.rb +3 -0
  176. data/spec/dummy/app/models/assignment_group.rb +3 -0
  177. data/spec/dummy/app/models/content_migration.rb +18 -0
  178. data/spec/dummy/app/models/context_module.rb +3 -0
  179. data/spec/dummy/app/models/context_module_item.rb +3 -0
  180. data/spec/dummy/app/models/course.rb +11 -0
  181. data/spec/dummy/app/models/course_nickname.rb +19 -0
  182. data/spec/dummy/app/models/enrollment.rb +14 -0
  183. data/spec/dummy/app/models/grading_period.rb +16 -0
  184. data/spec/dummy/app/models/grading_period_group.rb +15 -0
  185. data/spec/dummy/app/models/group.rb +2 -0
  186. data/spec/dummy/app/models/group_membership.rb +2 -0
  187. data/spec/dummy/app/models/learning_outcome.rb +30 -0
  188. data/spec/dummy/app/models/learning_outcome_result.rb +54 -0
  189. data/spec/dummy/app/models/pseudonym.rb +16 -0
  190. data/spec/dummy/app/models/role.rb +2 -0
  191. data/spec/dummy/app/models/rubric.rb +35 -0
  192. data/spec/dummy/app/models/rubric_assessment.rb +22 -0
  193. data/spec/dummy/app/models/rubric_association.rb +20 -0
  194. data/spec/dummy/app/models/section.rb +9 -0
  195. data/spec/dummy/app/models/submission.rb +4 -0
  196. data/spec/dummy/app/models/term.rb +3 -0
  197. data/spec/dummy/app/models/user.rb +11 -0
  198. data/spec/dummy/app/models/user_observer.rb +19 -0
  199. data/spec/dummy/app/services/live_events/assignment_event.rb +1 -1
  200. data/spec/dummy/app/services/live_events/course_event.rb +1 -3
  201. data/spec/dummy/app/services/live_events/course_section_event.rb +1 -1
  202. data/spec/dummy/app/services/live_events/enrollment_event.rb +1 -1
  203. data/spec/dummy/app/services/live_events/grade_event.rb +1 -1
  204. data/spec/dummy/app/services/live_events/module_event.rb +1 -1
  205. data/spec/dummy/app/services/live_events/module_item_event.rb +1 -1
  206. data/spec/dummy/app/services/live_events/submission_event.rb +1 -1
  207. data/spec/dummy/app/services/live_events/syllabus_event.rb +1 -1
  208. data/spec/dummy/app/services/live_events/user_event.rb +1 -3
  209. data/spec/dummy/config/environments/test.rb +2 -0
  210. data/spec/dummy/config/routes.rb +1 -0
  211. data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
  212. data/spec/dummy/db/migrate/20210907233329_create_user_observers.rb +23 -0
  213. data/spec/dummy/db/migrate/20210907233330_create_grading_periods.rb +28 -0
  214. data/spec/dummy/db/migrate/20211001184920_create_grading_period_groups.rb +24 -0
  215. data/spec/dummy/db/migrate/20220308072643_create_content_migrations.rb +30 -0
  216. data/spec/dummy/db/migrate/20220712210559_create_learning_outcomes.rb +36 -0
  217. data/spec/dummy/db/migrate/{20190702203620_create_users.rb → 20220926221926_create_users.rb} +0 -1
  218. data/spec/dummy/db/migrate/20240408223326_create_course_nicknames.rb +23 -0
  219. data/spec/dummy/db/migrate/20240509105100_create_rubrics.rb +44 -0
  220. data/spec/dummy/db/migrate/20240510094100_create_rubric_associations.rb +42 -0
  221. data/spec/dummy/db/migrate/20240510101100_create_rubric_assessments.rb +37 -0
  222. data/spec/dummy/db/migrate/20240523101010_create_learning_outcome_results.rb +52 -0
  223. data/spec/dummy/db/schema.rb +244 -5
  224. data/spec/factories/user_factory.rb +0 -1
  225. data/spec/job_batching/active_job_spec.rb +107 -0
  226. data/spec/job_batching/batch_spec.rb +489 -0
  227. data/spec/job_batching/callback_spec.rb +38 -0
  228. data/spec/job_batching/context_hash_spec.rb +54 -0
  229. data/spec/job_batching/flow_spec.rb +82 -0
  230. data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
  231. data/spec/job_batching/integration/integration.rb +57 -0
  232. data/spec/job_batching/integration/nested.rb +88 -0
  233. data/spec/job_batching/integration/simple.rb +47 -0
  234. data/spec/job_batching/integration/workflow.rb +134 -0
  235. data/spec/job_batching/integration_helper.rb +50 -0
  236. data/spec/job_batching/pool_spec.rb +161 -0
  237. data/spec/job_batching/sidekiq_spec.rb +125 -0
  238. data/spec/job_batching/status_spec.rb +76 -0
  239. data/spec/job_batching/support/base_job.rb +14 -0
  240. data/spec/job_batching/support/sample_callback.rb +2 -0
  241. data/spec/spec_helper.rb +17 -0
  242. data/spec/support/fixtures/reports/content_migrations.csv +3 -0
  243. data/spec/support/fixtures/reports/course_nicknames.csv +3 -0
  244. data/spec/support/fixtures/reports/grading_period_groups.csv +2 -0
  245. data/spec/support/fixtures/reports/grading_periods.csv +3 -0
  246. data/spec/support/fixtures/reports/learning_outcome_results.csv +3 -0
  247. data/spec/support/fixtures/reports/learning_outcomes.csv +3 -0
  248. data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +3 -0
  249. data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +4 -0
  250. data/spec/support/fixtures/reports/rubric_assessments.csv +3 -0
  251. data/spec/support/fixtures/reports/rubric_associations.csv +3 -0
  252. data/spec/support/fixtures/reports/rubrics.csv +3 -0
  253. data/spec/support/fixtures/reports/user_observers.csv +3 -0
  254. data/spec/support/fixtures/reports/users.csv +3 -2
  255. data/spec/support/fixtures/reports/xlist.csv +1 -1
  256. metadata +329 -27
  257. data/app/controllers/api/v1/live_events_controller.rb +0 -18
  258. data/lib/canvas_sync/job_chain.rb +0 -57
  259. data/lib/canvas_sync/jobs/fork_gather.rb +0 -59
  260. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
@@ -0,0 +1,99 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class BeginSyncChainJob < CanvasSync::Job
4
+ attr_reader :globals
5
+
6
+ def perform(chain_definition, globals = {})
7
+ @globals = globals
8
+
9
+ if globals[:updated_after] == nil && Rails.env.development?
10
+ globals[:updated_after] = false
11
+ end
12
+
13
+ if globals[:updated_after] == false
14
+ globals[:updated_after] = nil
15
+ elsif !globals[:updated_after].present? || globals[:updated_after] == true
16
+ last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
17
+ globals[:full_sync_every] ||= "sunday/2"
18
+ globals[:updated_after] = ((last_batch.started_at - 1.day).iso8601 rescue nil)
19
+ end
20
+
21
+ # Refuse to run syncs of the same genre if there is a running full sync
22
+ if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
23
+ Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
24
+ return
25
+ end
26
+
27
+ if should_full_sync?(globals[:full_sync_every])
28
+ globals[:updated_after] = nil
29
+ end
30
+
31
+ sync_batch = SyncBatch.create!(
32
+ started_at: DateTime.now,
33
+ full_sync: globals[:updated_after] == nil,
34
+ batch_genre: genre,
35
+ status: 'processing',
36
+ )
37
+
38
+ globals[:batch_genre] = genre
39
+ globals[:batch_start_time] = sync_batch.started_at.iso8601
40
+ globals[:sync_batch_id] = sync_batch.id
41
+
42
+ JobBatches::Batch.new.tap do |b|
43
+ b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
44
+ b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
45
+ b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
46
+ b.context = globals
47
+ b.jobs do
48
+ JobBatches::SerialBatchJob.perform_now(chain_definition)
49
+ end
50
+ sync_batch.update(batch_bid: b.bid)
51
+ end
52
+ end
53
+
54
+ def should_full_sync?(opt)
55
+ return true unless last_full_sync.present?
56
+ return false unless opt.is_a?(String)
57
+
58
+ case opt.strip
59
+ when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
60
+ m = Regexp.last_match
61
+ day = m[1]
62
+ skip = m[2] || "1"
63
+ DateTime.now.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
64
+ when %r{^(\d+)\%$}
65
+ m = Regexp.last_match
66
+ rand(100) < m[1].to_i
67
+ when %r{^(\d+) ?days$}
68
+ m = Regexp.last_match
69
+ last_full_sync.end_of_day <= m[1].to_i.days.ago.end_of_day
70
+ when %r{^(\d+)$} # N.days is converted to a string of seconds
71
+ m = Regexp.last_match
72
+ last_full_sync.end_of_day <= m[1].to_i.seconds.ago.end_of_day
73
+ else
74
+ false
75
+ end
76
+ end
77
+
78
+ def last_full_sync_record
79
+ @last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
80
+ end
81
+
82
+ def last_full_sync
83
+ last_full_sync_record&.started_at
84
+ end
85
+
86
+ def genre
87
+ globals[:batch_genre] || "default"
88
+ end
89
+
90
+ def self.batch_completed(status, options)
91
+ sbatch = SyncBatch.find(options['sync_batch_id'])
92
+ sbatch.update!(
93
+ status: status.success? ? 'completed' : 'failed',
94
+ completed_at: DateTime.now,
95
+ )
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,41 @@
1
+ module CanvasSync::Jobs
2
+ class CanvasProcessWaiter < ActiveJob::Base
3
+ # rubocop:disable Metrics/PerceivedComplexity
4
+ def perform(progress_url, next_job, kwargs = {})
5
+ kwargs = kwargs.symbolize_keys
6
+
7
+ response = canvas_sync_client.get(progress_url)
8
+ status = kwargs[:status_key].present? ? response[kwargs[:status_key]] : response['workflow_state'] || response['status']
9
+
10
+ if %w[completed complete imported imported_with_messages].include? status
11
+ InvokeCallbackWorker.perform_later(build_next_job(next_job, kwargs, response)) if next_job
12
+ elsif %w[failed error failed_with_messages].include? status
13
+ if kwargs[:on_failure].is_a?(Hash)
14
+ InvokeCallbackWorker.perform_later(build_next_job(kwargs[:on_failure], kwargs, response))
15
+ else
16
+ Rails.logger.error("Progress #{progress_url} failed")
17
+ end
18
+ else # if status == 'queued' || status == 'running'
19
+ interval = kwargs[:interval] || (Rails.env.development? ? 3 : 60)
20
+ CanvasProcessWaiter.set(wait: interval).perform_later(progress_url, next_job, kwargs)
21
+ end
22
+ end
23
+ # rubocop:enable Metrics/PerceivedComplexity
24
+
25
+ def build_next_job(job, kwargs, response)
26
+ job = job.symbolize_keys
27
+ if kwargs[:progress_as].present?
28
+ job[:kwargs] ||= {}
29
+ job[:kwargs][kwargs[:progress_as].to_sym] = response
30
+ end
31
+ job
32
+ end
33
+
34
+ # This is a separate job so that, if it fails and a retry is triggered, it doesn't query the API needlessly
35
+ class InvokeCallbackWorker < ActiveJob::Base
36
+ def perform(job)
37
+ CanvasSync::MiscHelper.invoke_task(job)
38
+ end
39
+ end
40
+ end
41
+ end
@@ -4,21 +4,28 @@ module CanvasSync
4
4
  # Re-enqueues itself if the report is still processing on Canvas.
5
5
  # Enqueues the ReportProcessor when the report has completed.
6
6
  class ReportChecker < CanvasSync::Job
7
- # @param job_chain [Hash]
7
+ REPORT_TIMEOUT = 24.hours
8
+ COMPILATION_TIMEOUT = 3.hours
9
+ MAX_TRIES = 3
10
+
11
+ class FatalReportError < ::RuntimeError; end
12
+
13
+ discard_on FatalReportError
14
+
8
15
  # @param report_name [Hash] e.g., 'provisioning_csv'
9
16
  # @param report_id [Integer]
10
17
  # @param processor [String] a stringified report processor class name
11
18
  # @param options [Hash] hash of options that will be passed to the job processor
12
19
  # @return [nil]
13
- def perform(job_chain, report_name, report_id, processor, options) # rubocop:disable Metrics/AbcSize
14
- account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
15
- report_status = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
20
+ def perform(report_name, report_id, processor, options, checker_context = {}) # rubocop:disable Metrics/AbcSize
21
+ max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || MAX_TRIES
22
+ account_id = options[:account_id] || batch_context[:account_id] || "self"
23
+ report_status = CanvasSync.get_canvas_sync_client(batch_context)
16
24
  .report_status(account_id, report_name, report_id)
17
25
 
18
26
  case report_status["status"].downcase
19
27
  when "complete"
20
28
  CanvasSync::Jobs::ReportProcessorJob.perform_later(
21
- job_chain,
22
29
  report_name,
23
30
  report_status["attachment"]["url"],
24
31
  processor,
@@ -26,21 +33,76 @@ module CanvasSync
26
33
  report_id,
27
34
  )
28
35
  when "error", "deleted"
29
- message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}" # rubocop:disable Metrics/LineLength
36
+ checker_context[:failed_attempts] ||= 0
37
+ checker_context[:failed_attempts] += 1
38
+ failed_attempts = checker_context[:failed_attempts]
39
+ message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}, #{current_organization.name}. This report has now failed #{checker_context[:failed_attempts]} time." # rubocop:disable Metrics/LineLength
30
40
  Rails.logger.error(message)
31
- raise message
41
+ if failed_attempts >= max_tries
42
+ Rails.logger.error("This report has failed #{failed_attempts} times. Giving up.")
43
+ raise FatalReportError, message
44
+ else
45
+ restart_report(options, report_name, processor, checker_context)
46
+ end
32
47
  else
48
+ report_timeout = parse_timeout(options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT)
49
+ if timeout_met?(options[:sync_start_time], report_timeout)
50
+ raise FatalReportError, "Report appears to be stuck #{report_name}##{report_id}"
51
+ end
52
+
53
+ if report_status["status"].downcase == 'compiling'
54
+ checker_context['compiling_since'] ||= DateTime.now.iso8601
55
+ compilation_timeout = parse_timeout(options[:report_compilation_timeout] || batch_context[:report_compilation_timeout] || COMPILATION_TIMEOUT)
56
+ if timeout_met?(checker_context['compiling_since'], compilation_timeout)
57
+ raise FatalReportError, "Report appears to be stuck #{report_name}##{report_id}"
58
+ end
59
+ end
60
+
33
61
  CanvasSync::Jobs::ReportChecker
34
62
  .set(wait: report_checker_wait_time)
35
63
  .perform_later(
36
- job_chain,
37
64
  report_name,
38
65
  report_id,
39
66
  processor,
40
67
  options,
68
+ checker_context
41
69
  )
42
70
  end
43
71
  end
72
+
73
+ protected
74
+
75
+ def timeout_met?(base_time, timeout_length)
76
+ return false unless base_time.present? && timeout_length.present?
77
+ DateTime.now > (DateTime.parse(base_time) + timeout_length)
78
+ end
79
+
80
+ def parse_timeout(val)
81
+ val
82
+ end
83
+
84
+ def restart_report(options, report_name, processor, checker_context)
85
+ account_id = options[:account_id] || batch_context[:account_id] || "self"
86
+ options[:sync_start_time] = DateTime.now.utc.iso8601
87
+ new_context = {}
88
+ new_context[:failed_attempts] = checker_context[:failed_attempts]
89
+ report_id = start_report(account_id, report_name, options[:report_params])
90
+ CanvasSync::Jobs::ReportChecker
91
+ .set(wait: report_checker_wait_time)
92
+ .perform_later(
93
+ report_name,
94
+ report_id,
95
+ processor,
96
+ options,
97
+ new_context
98
+ )
99
+ end
100
+
101
+ def start_report(account_id, report_name, report_params)
102
+ report = CanvasSync.get_canvas_sync_client(batch_context)
103
+ .start_report(account_id, report_name, report_params)
104
+ report["id"]
105
+ end
44
106
  end
45
107
  end
46
108
  end
@@ -6,22 +6,19 @@ module CanvasSync
6
6
  # download the report, and then pass the file path and options into the
7
7
  # process method on the processor.
8
8
  class ReportProcessorJob < CanvasSync::Job
9
- # @param job_chain [Hash]
10
9
  # @param report_name [Hash] e.g., 'provisioning_csv'
11
10
  # @param report_url [String]
12
11
  # @param processor [String] a stringified report processor class name
13
12
  # @param options [Hash] hash of options that will be passed to the job processor
14
13
  # @return [nil]
15
- def perform(job_chain, report_name, report_url, processor, options, report_id)
16
- @job_log.update_attributes(job_class: processor)
14
+ def perform(report_name, report_url, processor, options, report_id)
15
+ @job_log.update(job_class: processor)
17
16
  download(report_name, report_url) do |file_path|
18
- options = job_chain[:global_options].merge(options).merge({
17
+ options = batch_context.merge(options).merge({
19
18
  report_processor_job_id: @job_log.job_id
20
19
  })
21
20
  processor.constantize.process(file_path, options, report_id)
22
21
  end
23
-
24
- CanvasSync.invoke_next(job_chain)
25
22
  end
26
23
 
27
24
  private
@@ -29,7 +26,7 @@ module CanvasSync
29
26
  def download(report_name, report_url)
30
27
  Dir.mktmpdir do |dir|
31
28
  file_path = "#{dir}/#{report_name}"
32
- IO.copy_stream(open(report_url), file_path)
29
+ IO.copy_stream(URI.open(report_url), file_path)
33
30
  yield file_path
34
31
  end
35
32
  end
@@ -2,7 +2,6 @@ module CanvasSync
2
2
  module Jobs
3
3
  # Starts a Canvas report and enqueues a ReportChecker
4
4
  class ReportStarter < CanvasSync::Job
5
- # @param job_chain [Hash]
6
5
  # @param report_name [Hash] e.g., 'provisioning_csv'
7
6
  # @param report_params [Hash] The Canvas report parameters
8
7
  # @param processor [String] a stringified report processor class name
@@ -10,38 +9,53 @@ module CanvasSync
10
9
  # @param allow_redownloads [Boolean] whether you want the job_chain to cache this report,
11
10
  # so that any later jobs in the chain will use the same generated report
12
11
  # @return [nil]
13
- def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
14
- account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
12
+ def perform(report_name, report_params, processor, options, allow_redownloads: false)
13
+ account_id = options[:account_id] || batch_context[:account_id] || "self"
14
+ options[:sync_start_time] = DateTime.now.utc.iso8601
15
+ options[:report_params] = report_params
16
+ report_id = start_report(account_id, report_name, report_params)
17
+ # TODO: Restore report caching support (does nayone actually use it?)
18
+ # report_id = if allow_redownloads
19
+ # get_cached_report(account_id, report_name, report_params)
20
+ # else
21
+ # start_report(account_id, report_name, report_params)
22
+ # end
15
23
 
16
- report_id = if allow_redownloads
17
- get_cached_report(job_chain, account_id, report_name, report_params)
18
- else
19
- start_report(job_chain, account_id, report_name, report_params)
20
- end
21
-
22
- CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
23
- job_chain,
24
- report_name,
25
- report_id,
26
- processor,
27
- options,
28
- )
24
+ batch = JobBatches::Batch.new
25
+ batch.description = "CanvasSync #{report_name} Fiber"
26
+ batch.jobs do
27
+ CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
28
+ report_name,
29
+ report_id,
30
+ processor.to_s,
31
+ options
32
+ )
33
+ end
29
34
  end
30
35
 
31
36
  protected
32
37
 
33
- def merge_report_params(job_chain, options={}, params={}, term_scope: true)
34
- term_scope = job_chain[:global_options][:canvas_term_id] if term_scope == true
38
+ # Ruby 3 changed how kwargs are handled. _kw_placeholder allows for backwards compatibility
39
+ # In Ruby 2, merge_report_params(options, params) would parse as merge_report_params(options, params={}, **params) (wtf?), so
40
+ # merge_report_params(options, params, {}) is used. That doesn't work in Ruby 3.
41
+ # In order to maintain compatibility with 2 and with any apps, this oddness is needed
42
+ def merge_report_params(options, params={}, _kw_placeholder=nil, term_scope: true)
43
+ term_scope = options[:canvas_term_id] || batch_context[:canvas_term_id] if term_scope == true
35
44
  if term_scope.present?
36
45
  params[:enrollment_term_id] = term_scope
37
46
  end
47
+ if (updated_after = batch_context[:updated_after]).present?
48
+ params[:updated_after] = updated_after
49
+ end
38
50
  params.merge!(options[:report_params]) if options[:report_params].present?
51
+ params.merge!(options[:report_parameters]) if options[:report_parameters].present?
39
52
  { parameters: params }
40
53
  end
41
54
 
42
55
  private
43
56
 
44
57
  def get_cached_report(job_chain, account_id, report_name, report_params)
58
+ # TODO: job_chain[:global_options] is no longer available and batch_context won't work for this
45
59
  if job_chain[:global_options][report_name].present?
46
60
  job_chain[:global_options][report_name]
47
61
  else
@@ -51,8 +65,8 @@ module CanvasSync
51
65
  end
52
66
  end
53
67
 
54
- def start_report(job_chain, account_id, report_name, report_params)
55
- report = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
68
+ def start_report(account_id, report_name, report_params)
69
+ report = CanvasSync.get_canvas_sync_client(batch_context)
56
70
  .start_report(account_id, report_name, report_params)
57
71
  report["id"]
58
72
  end
@@ -7,18 +7,16 @@ module CanvasSync
7
7
  # running provisioning by term we sync users first so we don't duplicate
8
8
  # the work of syncing all accounts for each term.
9
9
  #
10
- # @param job_chain [Hash]
11
10
  # @param options [Hash]
12
- def perform(job_chain, options)
11
+ def perform(options)
13
12
  unless options[:root_account] == false
14
- acc_params = CanvasSync.get_canvas_sync_client(job_chain[:global_options]).account("self")
13
+ acc_params = CanvasSync.get_canvas_sync_client(batch_context).account("self")
15
14
  update_or_create_model(Account, acc_params)
16
15
  end
17
16
 
18
17
  super(
19
- job_chain,
20
18
  "proservices_provisioning_csv",
21
- merge_report_params(job_chain, options, {
19
+ merge_report_params(options, {
22
20
  accounts: true,
23
21
  include_deleted: true,
24
22
  }, term_scope: false),
@@ -4,11 +4,10 @@ module CanvasSync
4
4
  # Syncs Admins using the Canvas API
5
5
  #
6
6
  #
7
- # @param job_chain [Hash]
8
7
  # @param options [Hash]
9
- def perform(job_chain, _options)
8
+ def perform(options)
10
9
  updated_admin_ids = []
11
- api_client = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
10
+ api_client = CanvasSync.get_canvas_sync_client(batch_context)
12
11
  CanvasSync.sync_scope(Account).find_each do |acc|
13
12
  api_client.account_admins(acc.canvas_id).all_pages_each do |admin_params|
14
13
  admin_params[:account_id] = acc.canvas_id
@@ -17,7 +16,6 @@ module CanvasSync
17
16
  end
18
17
  end
19
18
  Admin.where.not(id: updated_admin_ids).update_all(workflow_state: 'inactive')
20
- CanvasSync.invoke_next(job_chain)
21
19
  end
22
20
  end
23
21
  end
@@ -6,13 +6,11 @@ module CanvasSync
6
6
  # Starts a report processor for the assignment_groups report
7
7
  # (the proserv_assignment_group_export_csv report must be enabled)
8
8
  #
9
- # @param job_chain [Hash]
10
9
  # @param options [Hash]
11
- def perform(job_chain, options)
10
+ def perform(options)
12
11
  super(
13
- job_chain,
14
12
  "proserv_assignment_group_export_csv",
15
- merge_report_params(job_chain, options),
13
+ merge_report_params(options),
16
14
  CanvasSync::Processors::AssignmentGroupsProcessor.to_s,
17
15
  {},
18
16
  )
@@ -6,13 +6,11 @@ module CanvasSync
6
6
  # Starts a report processor for the assignment report
7
7
  # (the proserv_assignment_export_csv report must be enabled)
8
8
  #
9
- # @param job_chain [Hash]
10
9
  # @param options [Hash]
11
- def perform(job_chain, options)
10
+ def perform(options)
12
11
  super(
13
- job_chain,
14
12
  "proserv_assignment_export_csv",
15
- merge_report_params(job_chain, options),
13
+ merge_report_params(options),
16
14
  CanvasSync::Processors::AssignmentsProcessor.to_s,
17
15
  {},
18
16
  )
@@ -0,0 +1,20 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class SyncContentMigrationsJob < ReportStarter
4
+ # Syncs ContentMigrations
5
+ #
6
+ # Starts a report processor for the content migrations report
7
+ # (the proserv_content_migrations_csv report must be enabled)
8
+ #
9
+ # @param options [Hash]
10
+ def perform(options)
11
+ super(
12
+ "proserv_content_migrations_csv",
13
+ merge_report_params(options),
14
+ CanvasSync::Processors::ContentMigrationsProcessor.to_s,
15
+ {},
16
+ )
17
+ end
18
+ end
19
+ end
20
+ end
@@ -6,13 +6,11 @@ module CanvasSync
6
6
  # Starts a report processor for the context modules report
7
7
  # (the proserv_context_module_items_csv report must be enabled)
8
8
  #
9
- # @param job_chain [Hash]
10
9
  # @param options [Hash]
11
- def perform(job_chain, options)
10
+ def perform(options)
12
11
  super(
13
- job_chain,
14
12
  "proserv_context_module_items_csv",
15
- merge_report_params(job_chain, options),
13
+ merge_report_params(options),
16
14
  CanvasSync::Processors::ContextModuleItemsProcessor.to_s,
17
15
  {},
18
16
  )
@@ -6,13 +6,11 @@ module CanvasSync
6
6
  # Starts a report processor for the context modules report
7
7
  # (the proserv_context_modules_csv report must be enabled)
8
8
  #
9
- # @param job_chain [Hash]
10
9
  # @param options [Hash]
11
- def perform(job_chain, options)
10
+ def perform(options)
12
11
  super(
13
- job_chain,
14
12
  "proserv_context_modules_csv",
15
- merge_report_params(job_chain, options),
13
+ merge_report_params(options),
16
14
  CanvasSync::Processors::ContextModulesProcessor.to_s,
17
15
  {},
18
16
  )
@@ -1,49 +1,8 @@
1
1
  module CanvasSync
2
2
  module Jobs
3
3
  # ActiveJob class that starts a Canvas provisioning report
4
- class SyncProvisioningReportJob < CanvasSync::Job
5
- # @param job_chain [Hash]
6
- # @param options [Hash] If options contains a :term_scope a seperate provisioning report
7
- # will be started for each term in that scope. :models should be an array of
8
- # models to sync.
9
- def perform(job_chain, options)
10
- if options[:term_scope]
11
- sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
12
- Term.send(options[:term_scope]).find_each.map do |term|
13
- # Deep copy the job_chain so each report gets the correct term id passed into
14
- # its options with no side effects
15
- term_id = get_term_id(term)
16
- duped_job_chain = Marshal.load(Marshal.dump(job_chain))
17
- duped_job_chain[:global_options][:canvas_term_id] = term_id
18
- {
19
- job_chain: duped_job_chain,
20
- params: report_params(options, term_id),
21
- options: options,
22
- }
23
- end
24
- end
25
-
26
- sub_reports.each do |r|
27
- start_report(r[:params], r[:job_chain], r[:options])
28
- end
29
- else
30
- start_report(report_params(options), job_chain, options)
31
- end
32
- end
33
-
34
- protected
35
-
36
- def start_report(report_params, job_chain, options)
37
- CanvasSync::Jobs::ReportStarter.perform_later(
38
- job_chain,
39
- "proservices_provisioning_csv",
40
- report_params,
41
- CanvasSync::Processors::ProvisioningReportProcessor.to_s,
42
- options,
43
- )
44
- end
45
-
46
- def report_params(options, canvas_term_id=nil)
4
+ class SyncProvisioningReportJob < ReportStarter
5
+ def perform(options)
47
6
  params = {
48
7
  include_deleted: true,
49
8
  }
@@ -51,18 +10,25 @@ module CanvasSync
51
10
  options[:models].each do |model|
52
11
  # group_membership is the only model param that is singular :(
53
12
  model = 'group_membership' if model == 'group_memberships'
13
+
54
14
  params[model] = true
55
15
  end
56
16
 
57
- params[:enrollment_term_id] = canvas_term_id if canvas_term_id
17
+ merged_params = merge_report_params(options, params, {}).with_indifferent_access
58
18
 
59
- params.merge!(options[:report_parameters]) if options[:report_parameters].present?
60
-
61
- { parameters: params }
62
- end
19
+ # Make sure the report also checks last_activity_at when checking updated_at
20
+ if options[:models].include?("enrollments")
21
+ if (%w[last_activity_at total_activity_time] & CanvasSync::Concerns::SyncMapping.mapping_for(Enrollment)[:columns].keys).present? && merged_params.dig(:parameters, :include_last_activity) == nil
22
+ merged_params[:parameters][:include_last_activity] = true
23
+ end
24
+ end
63
25
 
64
- def get_term_id(term)
65
- term.try(:canvas_id) || term.canvas_term_id
26
+ super(
27
+ "proservices_provisioning_csv",
28
+ merged_params,
29
+ CanvasSync::Processors::ProvisioningReportProcessor.to_s,
30
+ options,
31
+ )
66
32
  end
67
33
  end
68
34
  end
@@ -3,12 +3,10 @@ module CanvasSync
3
3
  class SyncRolesJob < CanvasSync::Job
4
4
  # Syncs Roles using the Canvas API
5
5
  #
6
- #
7
- # @param job_chain [Hash]
8
6
  # @param options [Hash]
9
- def perform(job_chain, _options)
7
+ def perform(options)
10
8
  updated_role_ids = []
11
- api_client = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
9
+ api_client = CanvasSync.get_canvas_sync_client(batch_context)
12
10
  CanvasSync.sync_scope(Account).find_each do |acc|
13
11
  api_client.list_roles(acc.canvas_id, state: %w[active inactive]).all_pages_each do |role_params|
14
12
  role = update_or_create_model(Role, role_params)
@@ -16,7 +14,6 @@ module CanvasSync
16
14
  end
17
15
  end
18
16
  Role.where.not(id: updated_role_ids).update_all(workflow_state: 'inactive')
19
- CanvasSync.invoke_next(job_chain)
20
17
  end
21
18
  end
22
19
  end
@@ -0,0 +1,15 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class SyncRubricAssessmentsJob < ReportStarter
4
+ # @param options [Hash]
5
+ def perform(options)
6
+ super(
7
+ "rubric_assessments_csv",
8
+ merge_report_params(options),
9
+ CanvasSync::Processors::RubricAssessmentsProcessor.to_s,
10
+ {},
11
+ )
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,15 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class SyncRubricAssociationsJob < ReportStarter
4
+ # @param options [Hash]
5
+ def perform(options)
6
+ super(
7
+ "rubric_associations_csv",
8
+ merge_report_params(options),
9
+ CanvasSync::Processors::RubricAssociationsProcessor.to_s,
10
+ {},
11
+ )
12
+ end
13
+ end
14
+ end
15
+ end