resque-stages 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yml +103 -0
  5. data/.rubocop_todo.yml +34 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +6 -0
  8. data/CODE_OF_CONDUCT.md +74 -0
  9. data/Gemfile +9 -0
  10. data/Gemfile.lock +172 -0
  11. data/LICENSE.txt +21 -0
  12. data/README.md +250 -0
  13. data/Rakefile +8 -0
  14. data/bin/console +16 -0
  15. data/bin/setup +8 -0
  16. data/lib/resque-stages.rb +11 -0
  17. data/lib/resque/plugins/stages.rb +110 -0
  18. data/lib/resque/plugins/stages/cleaner.rb +36 -0
  19. data/lib/resque/plugins/stages/redis_access.rb +16 -0
  20. data/lib/resque/plugins/stages/staged_group.rb +181 -0
  21. data/lib/resque/plugins/stages/staged_group_list.rb +79 -0
  22. data/lib/resque/plugins/stages/staged_group_stage.rb +275 -0
  23. data/lib/resque/plugins/stages/staged_job.rb +271 -0
  24. data/lib/resque/plugins/stages/version.rb +9 -0
  25. data/lib/resque/server/public/stages.css +56 -0
  26. data/lib/resque/server/views/_group_stages_list_pagination.erb +67 -0
  27. data/lib/resque/server/views/_group_stages_list_table.erb +25 -0
  28. data/lib/resque/server/views/_stage_job_list_pagination.erb +72 -0
  29. data/lib/resque/server/views/_stage_job_list_table.erb +46 -0
  30. data/lib/resque/server/views/_staged_group_list_pagination.erb +67 -0
  31. data/lib/resque/server/views/_staged_group_list_table.erb +44 -0
  32. data/lib/resque/server/views/group_stages_list.erb +58 -0
  33. data/lib/resque/server/views/groups.erb +40 -0
  34. data/lib/resque/server/views/job_details.erb +91 -0
  35. data/lib/resque/server/views/stage.erb +64 -0
  36. data/lib/resque/stages_server.rb +240 -0
  37. data/read_me/groups_list.png +0 -0
  38. data/read_me/job.png +0 -0
  39. data/read_me/stage.png +0 -0
  40. data/read_me/stages.png +0 -0
  41. data/resque-stages.gemspec +49 -0
  42. data/spec/rails_helper.rb +40 -0
  43. data/spec/resque/plugins/stages/cleaner_spec.rb +82 -0
  44. data/spec/resque/plugins/stages/staged_group_list_spec.rb +96 -0
  45. data/spec/resque/plugins/stages/staged_group_spec.rb +226 -0
  46. data/spec/resque/plugins/stages/staged_group_stage_spec.rb +293 -0
  47. data/spec/resque/plugins/stages/staged_job_spec.rb +324 -0
  48. data/spec/resque/plugins/stages_spec.rb +369 -0
  49. data/spec/resque/server/public/stages.css_spec.rb +18 -0
  50. data/spec/resque/server/views/group_stages_list.erb_spec.rb +67 -0
  51. data/spec/resque/server/views/groups.erb_spec.rb +81 -0
  52. data/spec/resque/server/views/job_details.erb_spec.rb +100 -0
  53. data/spec/resque/server/views/stage.erb_spec.rb +68 -0
  54. data/spec/spec_helper.rb +104 -0
  55. data/spec/support/01_utils/fake_logger.rb +7 -0
  56. data/spec/support/config/redis-auth.yml +12 -0
  57. data/spec/support/fake_logger.rb +7 -0
  58. data/spec/support/jobs/basic_job.rb +17 -0
  59. data/spec/support/jobs/compressed_job.rb +18 -0
  60. data/spec/support/jobs/retry_job.rb +21 -0
  61. data/spec/support/purge_all.rb +15 -0
  62. metadata +297 -0
@@ -0,0 +1,181 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Resque
4
+ module Plugins
5
+ module Stages
6
+ # This class represents the toplevel grouping of a set of staged jobs.
7
+ #
8
+ # The group defines individual numbered stages (starting with 0) and
9
+ # intiates subsequent stages as the current stage completes.
10
+ #
11
+ # There are methods on the group to initiate groups and add jobs to
12
+ # individual stages or get/create new stages
13
+ class StagedGroup
14
+ include Resque::Plugins::Stages::RedisAccess
15
+ include Comparable
16
+
17
+ attr_reader :group_id
18
+
19
+ def initialize(group_id, description: "")
20
+ @group_id = group_id
21
+
22
+ Resque::Plugins::Stages::StagedGroupList.new.add_group(self)
23
+
24
+ redis.hsetnx(group_info_key, "created_at", Time.now)
25
+ self.description = description if description.present?
26
+ end
27
+
28
+ def <=>(other)
29
+ return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroup)
30
+
31
+ group_id <=> other.group_id
32
+ end
33
+
34
+ class << self
35
+ def within_a_grouping(description = nil)
36
+ group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: description)
37
+
38
+ yield group
39
+
40
+ group.initiate
41
+ end
42
+ end
43
+
44
+ def initiate
45
+ stage = next_stage
46
+
47
+ if stage
48
+ stage.initiate
49
+ else
50
+ delete
51
+ end
52
+ end
53
+
54
+ def description
55
+ @description ||= redis.hget(group_info_key, "description").presence || group_id
56
+ end
57
+
58
+ def created_at
59
+ @created_at ||= redis.hget(group_info_key, "created_at").presence.to_time || Time.now
60
+ end
61
+
62
+ def description=(value)
63
+ @description = value.presence
64
+ redis.hset(group_info_key, "description", description)
65
+ end
66
+
67
+ def current_stage
68
+ next_stage || new_stage
69
+ end
70
+
71
+ def last_stage
72
+ group_stages = stages
73
+ last_key = group_stages.keys.max
74
+
75
+ group_stages[last_key]
76
+ end
77
+
78
+ def add_stage(staged_group_stage)
79
+ redis.rpush group_key, staged_group_stage.group_stage_id
80
+ end
81
+
82
+ def remove_stage(staged_group_stage)
83
+ redis.lrem(group_key, 0, staged_group_stage.group_stage_id)
84
+ end
85
+
86
+ def num_stages
87
+ redis.llen group_key
88
+ end
89
+
90
+ def stages
91
+ all_stages = redis.lrange(group_key, 0, -1).map { |stage_id| Resque::Plugins::Stages::StagedGroupStage.new(stage_id) }
92
+
93
+ all_stages.each_with_object({}) do |stage, hash|
94
+ num = stage.number
95
+ num += 1 while hash.key?(num)
96
+
97
+ hash[num] = stage
98
+ stage.number = num if stage.number != num
99
+ end
100
+ end
101
+
102
+ def paged_stages(page_num = 1, page_size = nil)
103
+ page_size ||= 20
104
+ page_size = page_size.to_i
105
+ page_size = 20 if page_size < 1
106
+ start = (page_num - 1) * page_size
107
+ start = 0 if start >= num_stages || start.negative?
108
+
109
+ stages.values[start..start + page_size - 1]
110
+ end
111
+
112
+ def stage(stage_number)
113
+ found_stage = stages[stage_number]
114
+
115
+ found_stage || create_stage(stage_number)
116
+ end
117
+
118
+ def delete
119
+ stages.each_value(&:delete)
120
+
121
+ Resque::Plugins::Stages::StagedGroupList.new.remove_group(self)
122
+
123
+ redis.del group_key
124
+ redis.del group_info_key
125
+ end
126
+
127
+ def stage_completed
128
+ initiate
129
+ end
130
+
131
+ def blank?
132
+ !redis.exists(group_key) && !redis.exists(group_info_key)
133
+ end
134
+
135
+ def verify_stage(stage)
136
+ ids = redis.lrange(group_key, 0, -1)
137
+
138
+ return if ids.include?(stage.group_stage_id)
139
+
140
+ redis.lpush(group_key, stage.group_stage_id)
141
+ end
142
+
143
+ private
144
+
145
+ def next_stage
146
+ group_stages = stages
147
+ keys = group_stages.keys.sort
148
+
149
+ current_number = keys.detect do |key|
150
+ group_stages[key].status != :complete
151
+ end
152
+
153
+ group_stages[current_number]
154
+ end
155
+
156
+ def group_key
157
+ "StagedGroup::#{group_id}"
158
+ end
159
+
160
+ def group_info_key
161
+ "#{group_key}::Info"
162
+ end
163
+
164
+ def new_stage
165
+ next_stage_number = (last_stage&.number || 0) + 1
166
+
167
+ create_stage(next_stage_number)
168
+ end
169
+
170
+ def create_stage(stage_number)
171
+ stage = Resque::Plugins::Stages::StagedGroupStage.new(SecureRandom.uuid)
172
+
173
+ stage.staged_group = self
174
+ stage.number = stage_number
175
+
176
+ stage
177
+ end
178
+ end
179
+ end
180
+ end
181
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Resque
4
+ module Plugins
5
+ module Stages
6
+ # A utility class that keeps track of all Groups and lists them.
7
+ class StagedGroupList
8
+ include Resque::Plugins::Stages::RedisAccess
9
+
10
+ def groups
11
+ redis.smembers(list_key).map { |group_id| Resque::Plugins::Stages::StagedGroup.new(group_id) }
12
+ end
13
+
14
+ def order_param(sort_option, current_sort, current_order)
15
+ current_order ||= "asc"
16
+
17
+ if sort_option == current_sort
18
+ current_order == "asc" ? "desc" : "asc"
19
+ else
20
+ "asc"
21
+ end
22
+ end
23
+
24
+ def paginated_groups(sort_key = :description,
25
+ sort_order = "asc",
26
+ page_num = 1,
27
+ queue_page_size = 20)
28
+ queue_page_size = queue_page_size.to_i
29
+ queue_page_size = 20 if queue_page_size < 1
30
+
31
+ group_list = sorted_groups(sort_key)
32
+
33
+ page_start = (page_num - 1) * queue_page_size
34
+ page_start = 0 if page_start > group_list.length || page_start.negative?
35
+
36
+ (sort_order == "desc" ? group_list.reverse : group_list)[page_start..(page_start + queue_page_size - 1)]
37
+ end
38
+
39
+ def num_groups
40
+ groups.length
41
+ end
42
+
43
+ def add_group(group)
44
+ redis.sadd list_key, group.group_id
45
+ end
46
+
47
+ def remove_group(group)
48
+ redis.srem list_key, group.group_id
49
+ end
50
+
51
+ def delete_all
52
+ groups.each(&:delete)
53
+ end
54
+
55
+ private
56
+
57
+ def sorted_groups(sort_key)
58
+ groups.sort_by do |group|
59
+ group_sort_value(group, sort_key)
60
+ end
61
+ end
62
+
63
+ def group_sort_value(group, sort_key)
64
+ case sort_key.to_sym
65
+ when :description,
66
+ :num_stages
67
+ group.public_send(sort_key)
68
+ when :created_at
69
+ group.public_send(sort_key).to_s
70
+ end
71
+ end
72
+
73
+ def list_key
74
+ "StagedGroupList"
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,275 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Resque
4
+ module Plugins
5
+ module Stages
6
+ # A class which represents a single stage and all of the jobs for that stage.
7
+ #
8
+ # Each stage has a status that it progresses through
9
+ # :pending - not started yet
10
+ # :running - currently running
11
+ # :complete - all jobs in the stage are completed
12
+
13
+ # rubocop:disable Metrics/ClassLength
14
+ class StagedGroupStage
15
+ include Resque::Plugins::Stages::RedisAccess
16
+ include Comparable
17
+
18
+ attr_reader :group_stage_id
19
+
20
+ def initialize(group_stage_id)
21
+ @group_stage_id = group_stage_id
22
+ end
23
+
24
+ def <=>(other)
25
+ return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroupStage)
26
+
27
+ group_stage_id <=> other.group_stage_id
28
+ end
29
+
30
+ def enqueue(klass, *args)
31
+ job = create_enqueue_job(klass, args)
32
+
33
+ return job if status == :pending
34
+
35
+ self.status = :running if status != :running
36
+
37
+ job.status = :queued
38
+ Resque.enqueue(*job.enqueue_args)
39
+
40
+ job
41
+ end
42
+
43
+ def enqueue_to(queue, klass, *args)
44
+ job = create_enqueue_job(klass, args)
45
+
46
+ return job if status == :pending
47
+
48
+ self.status = :running if status != :running
49
+
50
+ job.status = :queued
51
+ Resque.enqueue_to(queue, *job.enqueue_args)
52
+
53
+ job
54
+ end
55
+
56
+ def enqueue_at(timestamp, klass, *args)
57
+ job = create_enqueue_job(klass, args)
58
+
59
+ return job if status == :pending
60
+
61
+ self.status = :running if status != :running
62
+
63
+ job.status = :queued
64
+ Resque.enqueue_at(timestamp, *job.enqueue_args)
65
+
66
+ job
67
+ end
68
+
69
+ def enqueue_at_with_queue(queue, timestamp, klass, *args)
70
+ job = create_enqueue_job(klass, args)
71
+
72
+ return job if status == :pending
73
+
74
+ self.status = :running if status != :running
75
+
76
+ job.status = :queued
77
+ Resque.enqueue_at_with_queue(queue, timestamp, *job.enqueue_args)
78
+
79
+ job
80
+ end
81
+
82
+ def enqueue_in(number_of_seconds_from_now, klass, *args)
83
+ job = create_enqueue_job(klass, args)
84
+
85
+ return job if status == :pending
86
+
87
+ self.status = :running if status != :running
88
+
89
+ job.status = :queued
90
+ Resque.enqueue_in(number_of_seconds_from_now, *job.enqueue_args)
91
+
92
+ job
93
+ end
94
+
95
+ def enqueue_in_with_queue(queue, number_of_seconds_from_now, klass, *args)
96
+ job = create_enqueue_job(klass, args)
97
+
98
+ return job if status == :pending
99
+
100
+ self.status = :running if status != :running
101
+
102
+ job.status = :queued
103
+ Resque.enqueue_in_with_queue(queue, number_of_seconds_from_now, *job.enqueue_args)
104
+
105
+ job
106
+ end
107
+
108
+ def status
109
+ redis.hget(staged_group_key, "status")&.to_sym || :pending
110
+ end
111
+
112
+ def status=(value)
113
+ redis.hset(staged_group_key, "status", value.to_s)
114
+
115
+ staged_group&.stage_completed if status == :complete
116
+ end
117
+
118
+ def number
119
+ redis.hget(staged_group_key, "number")&.to_i || 1
120
+ end
121
+
122
+ def number=(value)
123
+ redis.hset(staged_group_key, "number", value)
124
+ end
125
+
126
+ def staged_group
127
+ return nil if staged_group_id.blank?
128
+
129
+ @staged_group ||= Resque::Plugins::Stages::StagedGroup.new(staged_group_id)
130
+ end
131
+
132
+ def staged_group=(value)
133
+ @staged_group = value
134
+ @staged_group_id = value.group_id
135
+
136
+ value.add_stage(self)
137
+ redis.hset(staged_group_key, "staged_group_id", value.group_id)
138
+ end
139
+
140
+ def jobs(start = 0, stop = -1)
141
+ redis.lrange(stage_key, start, stop).map { |id| Resque::Plugins::Stages::StagedJob.new(id) }
142
+ end
143
+
144
+ def jobs_by_status(status)
145
+ jobs.select { |job| job.status == status }
146
+ end
147
+
148
+ def paginated_jobs(sort_key = :class_name,
149
+ sort_order = "asc",
150
+ page_num = 1,
151
+ queue_page_size = 20)
152
+ queue_page_size = queue_page_size.to_i
153
+ queue_page_size = 20 if queue_page_size < 1
154
+
155
+ job_list = sorted_jobs(sort_key)
156
+
157
+ page_start = (page_num - 1) * queue_page_size
158
+ page_start = 0 if page_start > job_list.length || page_start.negative?
159
+
160
+ (sort_order == "desc" ? job_list.reverse : job_list)[page_start..(page_start + queue_page_size - 1)]
161
+ end
162
+
163
+ def order_param(sort_option, current_sort, current_order)
164
+ current_order ||= "asc"
165
+
166
+ if sort_option == current_sort
167
+ current_order == "asc" ? "desc" : "asc"
168
+ else
169
+ "asc"
170
+ end
171
+ end
172
+
173
+ def num_jobs
174
+ redis.llen(stage_key)
175
+ end
176
+
177
+ def add_job(staged_group_job)
178
+ redis.rpush stage_key, staged_group_job.job_id
179
+ end
180
+
181
+ def remove_job(staged_group_job)
182
+ redis.lrem(stage_key, 0, staged_group_job.job_id)
183
+ end
184
+
185
+ def delete
186
+ jobs.each(&:delete)
187
+
188
+ staged_group&.remove_stage self
189
+
190
+ redis.del stage_key
191
+ redis.del staged_group_key
192
+ end
193
+
194
+ def initiate
195
+ self.status = :running
196
+
197
+ jobs.each do |job|
198
+ next if job.completed?
199
+ next if job.queued?
200
+
201
+ job.enqueue_job
202
+ end
203
+
204
+ job_completed
205
+ end
206
+
207
+ def job_completed
208
+ self.status = :complete if jobs.all?(&:completed?)
209
+ end
210
+
211
+ def blank?
212
+ !redis.exists(stage_key) && !redis.exists(staged_group_key)
213
+ end
214
+
215
+ def verify
216
+ return build_new_structure if staged_group.blank?
217
+
218
+ staged_group.verify_stage(self)
219
+ end
220
+
221
+ def verify_job(job)
222
+ ids = redis.lrange(stage_key, 0, -1)
223
+
224
+ return if ids.include?(job.job_id)
225
+
226
+ redis.lpush(stage_key, job.job_id)
227
+ end
228
+
229
+ private
230
+
231
+ def build_new_structure
232
+ group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid)
233
+
234
+ self.staged_group = group
235
+ end
236
+
237
+ def create_enqueue_job(klass, args)
238
+ Resque::Plugins::Stages::StagedJob.create_job self, klass, *args
239
+ end
240
+
241
+ def stage_key
242
+ "StagedGroupStage::#{group_stage_id}"
243
+ end
244
+
245
+ def staged_group_key
246
+ "#{stage_key}::staged_group"
247
+ end
248
+
249
+ def staged_group_id
250
+ @staged_group_id ||= redis.hget(staged_group_key, "staged_group_id")
251
+ end
252
+
253
+ def sorted_jobs(sort_key)
254
+ jobs.sort_by do |job|
255
+ job_sort_value(job, sort_key)
256
+ end
257
+ end
258
+
259
+ def job_sort_value(job, sort_key)
260
+ case sort_key.to_sym
261
+ when :class_name,
262
+ :status,
263
+ :status_message
264
+ job.public_send(sort_key)
265
+
266
+ when :queue_time
267
+ job.public_send(sort_key).to_s
268
+ end
269
+ end
270
+ end
271
+
272
+ # rubocop:enable Metrics/ClassLength
273
+ end
274
+ end
275
+ end