canvas_sync 0.21.1 → 0.22.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
  3. data/lib/canvas_sync/config.rb +3 -5
  4. data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
  5. data/lib/canvas_sync/job_batches/batch.rb +432 -402
  6. data/lib/canvas_sync/job_batches/callback.rb +100 -114
  7. data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
  8. data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
  9. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
  10. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
  11. data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
  12. data/lib/canvas_sync/job_batches/compat.rb +20 -0
  13. data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
  14. data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
  15. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
  16. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
  17. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
  18. data/lib/canvas_sync/job_batches/pool.rb +193 -195
  19. data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
  20. data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
  21. data/lib/canvas_sync/job_batches/status.rb +85 -87
  22. data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
  23. data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
  24. data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
  25. data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
  26. data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
  27. data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
  28. data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
  29. data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
  30. data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
  31. data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
  32. data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
  33. data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
  34. data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
  35. data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
  36. data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
  37. data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
  38. data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
  39. data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
  40. data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
  41. data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
  42. data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
  43. data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
  44. data/lib/canvas_sync/misc_helper.rb +1 -1
  45. data/lib/canvas_sync/version.rb +1 -1
  46. data/lib/canvas_sync.rb +4 -3
  47. data/spec/dummy/app/models/rubric.rb +2 -1
  48. data/spec/dummy/config/environments/test.rb +1 -1
  49. data/spec/job_batching/batch_spec.rb +49 -7
  50. data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
  51. data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
  52. data/spec/job_batching/flow_spec.rb +1 -1
  53. data/spec/job_batching/integration_helper.rb +1 -1
  54. data/spec/job_batching/status_spec.rb +2 -2
  55. data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
  56. data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
  57. data/spec/job_uniqueness/lock_context_spec.rb +95 -0
  58. data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
  59. data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
  60. data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
  61. data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
  62. data/spec/job_uniqueness/spec_helper.rb +14 -0
  63. data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
  64. data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
  65. data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
  66. data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
  67. data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
  68. data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
  69. data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
  70. data/spec/job_uniqueness/support/on_conflict.rb +24 -0
  71. data/spec/job_uniqueness/support/test_worker.rb +19 -0
  72. data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
  73. data/spec/spec_helper.rb +1 -1
  74. metadata +278 -204
  75. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
  76. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
  77. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
  78. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
  79. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
  80. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
  81. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
  82. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
  83. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
  84. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
  85. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
  86. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,146 +1,132 @@
1
- module CanvasSync
2
- module JobBatches
3
- class Batch
4
- module Callback
5
- mattr_accessor :worker_class
6
-
7
- VALID_CALLBACKS = %w[success complete death].freeze
8
-
9
- module CallbackWorkerCommon
10
- def perform(definition, event, opts, bid, parent_bid)
11
- return unless VALID_CALLBACKS.include?(event)
12
-
13
- method = nil
14
- target = :instance
15
- clazz = definition
16
- if clazz.is_a?(String)
17
- if clazz.include?('#')
18
- clazz, method = clazz.split("#")
19
- elsif clazz.include?('.')
20
- clazz, method = clazz.split(".")
21
- target = :class
22
- end
1
+ module CanvasSync::JobBatches
2
+ class Batch
3
+ module Callback
4
+ mattr_accessor :worker_class
5
+
6
+ VALID_CALLBACKS = %i[success complete death stagnated].freeze
7
+
8
+ module CallbackWorkerCommon
9
+ def perform(definition, event, opts, bid, parent_bid)
10
+ return unless VALID_CALLBACKS.include?(event.to_sym)
11
+
12
+ method = nil
13
+ target = :instance
14
+ clazz = definition
15
+ if clazz.is_a?(String)
16
+ if clazz.include?('#')
17
+ clazz, method = clazz.split("#")
18
+ elsif clazz.include?('.')
19
+ clazz, method = clazz.split(".")
20
+ target = :class
23
21
  end
22
+ end
24
23
 
25
- method ||= "on_#{event}"
26
- status = Batch::Status.new(bid)
24
+ method ||= "on_#{event}"
25
+ status = Batch::Status.new(bid)
27
26
 
28
- if clazz && object = Object.const_get(clazz)
29
- target = target == :instance ? object.new : object
30
- if target.respond_to?(method, true)
31
- target.send(method, status, opts)
32
- else
33
- Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
34
- end
27
+ if clazz && object = Object.const_get(clazz)
28
+ target = target == :instance ? object.new : object
29
+ if target.respond_to?(method, true)
30
+ target.send(method, status, opts.with_indifferent_access)
35
31
  else
36
- Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
32
+ Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
37
33
  end
34
+ else
35
+ Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
38
36
  end
39
37
  end
38
+ end
40
39
 
41
- class Finalize
42
- def dispatch(status, opts)
43
- bid = opts["bid"]
44
- event = opts["event"].to_sym
40
+ class Finalize
41
+ # The methods in this class are called after all same-named callbacks have been
42
+ # completed for the passed Batch.
43
+ # These methods mainly handle bubbling events up to the parent Batch
44
+ # You could say that they are the callbacks for callbacks.
45
45
 
46
- Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}"}
46
+ def dispatch(status, opts)
47
+ bid = opts["bid"]
48
+ event = opts["event"].to_sym
47
49
 
48
- batch_status = Status.new bid
49
- send(event, bid, batch_status, batch_status.parent_bid)
50
+ Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}"}
50
51
 
51
- Batch.redis do |r|
52
- r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
53
- end
52
+ batch_status = Status.new bid
53
+ send(event, bid, batch_status, batch_status.parent_bid)
54
+
55
+ Batch.redis do |r|
56
+ r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
57
+ end
54
58
 
55
- if event == :success
56
- if opts['origin'].present?
57
- # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
58
- origin_bid = opts['origin']['for_bid']
59
- _, pending, success_ran = Batch.redis do |r|
60
- r.multi do |r|
61
- r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
62
- r.scard("BID-#{origin_bid}-pending_callbacks")
63
- r.hget("BID-#{origin_bid}", "success")
64
- end
59
+ if event == :success
60
+ if opts['origin'].present?
61
+ # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
62
+ origin_bid = opts['origin']['for_bid']
63
+ _, pending, success_ran = Batch.redis do |r|
64
+ r.multi do |r|
65
+ r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
66
+ r.scard("BID-#{origin_bid}-pending_callbacks")
67
+ r.hget("BID-#{origin_bid}", "success")
65
68
  end
66
- Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
67
69
  end
70
+ Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
71
+ end
68
72
 
69
- if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
70
- Batch.cleanup_redis(bid)
71
- end
73
+ if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
74
+ Batch.cleanup_redis(bid)
72
75
  end
73
76
  end
77
+ end
74
78
 
75
- def success(bid, status, parent_bid)
76
- return unless parent_bid
79
+ def success(bid, status, parent_bid)
80
+ return unless parent_bid
77
81
 
78
- _, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
79
- r.multi do |r|
80
- r.sadd("BID-#{parent_bid}-batches-success", bid)
81
- r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
82
- r.scard("BID-#{parent_bid}-batches-success")
82
+ Batch.with_callback_check(parent_bid) do |r|
83
+ r.sadd("BID-#{parent_bid}-batches-success", bid)
84
+ r.srem("BID-#{parent_bid}-batches-failed", bid)
85
+ r.sadd("BID-#{parent_bid}-batches-complete", bid)
86
+ end
87
+ end
83
88
 
84
- r.srem("BID-#{parent_bid}-batches-failed", bid)
85
- r.sadd("BID-#{parent_bid}-batches-complete", bid)
86
- r.scard("BID-#{parent_bid}-batches-complete")
89
+ def complete(bid, status, parent_bid)
90
+ return unless parent_bid
87
91
 
88
- r.hincrby("BID-#{parent_bid}", "pending", 0)
89
- r.hincrby("BID-#{parent_bid}", "children", 0)
90
- r.scard("BID-#{parent_bid}-batches-success")
91
- r.scard("BID-#{parent_bid}-failed")
92
- end
92
+ pending, children, success = Batch.redis do |r|
93
+ r.multi do |r|
94
+ r.hincrby("BID-#{bid}", "pending", 0)
95
+ r.hincrby("BID-#{bid}", "children", 0)
96
+ r.scard("BID-#{bid}-batches-success")
93
97
  end
98
+ end
94
99
 
95
- if complete == children && pending == failure
96
- Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
97
- Batch.enqueue_callbacks(:complete, parent_bid)
98
- end
99
- if pending.to_i.zero? && children == success
100
- Batch.logger.debug {"Finalize parent success bid: #{parent_bid}"}
101
- Batch.enqueue_callbacks(:success, parent_bid)
100
+ if !(pending.to_i.zero? && children == success)
101
+ # If batch was not successfull check and see if its parent is complete
102
+ # if the parent is complete we can trigger its complete callback.
103
+ #
104
+ # Otherwise, we don't want to to trigger the parent's :complete here (and
105
+ # we instead opt to have success tigger parent :complete) - this
106
+ # allows the success callback to add additional jobs to the parent batch
107
+ # before triggering :complete.
108
+
109
+ Batch.with_callback_check(parent_bid, except: [:success]) do |r|
110
+ r.sadd("BID-#{parent_bid}-batches-complete", bid)
111
+ r.sadd("BID-#{parent_bid}-batches-failed", bid)
102
112
  end
103
113
  end
114
+ end
104
115
 
105
- def complete(bid, status, parent_bid)
106
- pending, children, success = Batch.redis do |r|
107
- r.multi do |r|
108
- r.hincrby("BID-#{bid}", "pending", 0)
109
- r.hincrby("BID-#{bid}", "children", 0)
110
- r.scard("BID-#{bid}-batches-success")
111
- end
112
- end
116
+ def death(bid, status, parent_bid)
117
+ return unless parent_bid
113
118
 
114
- if parent_bid && !(pending.to_i.zero? && children == success)
115
- # If batch was not successfull check and see if its parent is complete
116
- # if the parent is complete we trigger its complete callback.
117
- #
118
- # Otherwise, we don't want to to trigger the parent's :complete here (and
119
- # instead opt to have success tigger parent :complete) - this
120
- # allows the success callback to add additional jobs to the parent batch
121
- # before triggering :complete.
122
-
123
- Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
124
- _, _, complete, pending, children, failure = Batch.redis do |r|
125
- r.multi do |r|
126
- r.sadd("BID-#{parent_bid}-batches-complete", bid)
127
- r.sadd("BID-#{parent_bid}-batches-failed", bid)
128
- r.scard("BID-#{parent_bid}-batches-complete")
129
- r.hincrby("BID-#{parent_bid}", "pending", 0)
130
- r.hincrby("BID-#{parent_bid}", "children", 0)
131
- r.scard("BID-#{parent_bid}-failed")
132
- end
133
- end
134
- if complete == children && pending == failure
135
- Batch.enqueue_callbacks(:complete, parent_bid)
136
- end
137
- end
138
- end
119
+ # We only need to bubble the event here - other events (eg stagnation) will be checked and bubbled elsewhere.
120
+
121
+ Batch.enqueue_callbacks(:death, parent_bid)
122
+ end
139
123
 
140
- def death(bid, status, parent_bid)
141
- return unless parent_bid
124
+ def stagnated(bid, status, parent_bid)
125
+ return unless parent_bid
142
126
 
143
- Batch.enqueue_callbacks(:death, parent_bid)
127
+ Batch.with_callback_check(parent_bid) do |r|
128
+ r.sadd("BID-#{parent_bid}-batches-stagnated", bid)
129
+ r.expire("BID-#{parent_bid}-batches-stagnated", BID_EXPIRE_TTL)
144
130
  end
145
131
  end
146
132
  end