resque-stages 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.rspec +3 -0
- data/.rubocop.yml +103 -0
- data/.rubocop_todo.yml +34 -0
- data/.ruby-version +1 -0
- data/.travis.yml +6 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +172 -0
- data/LICENSE.txt +21 -0
- data/README.md +250 -0
- data/Rakefile +8 -0
- data/bin/console +16 -0
- data/bin/setup +8 -0
- data/lib/resque-stages.rb +11 -0
- data/lib/resque/plugins/stages.rb +110 -0
- data/lib/resque/plugins/stages/cleaner.rb +36 -0
- data/lib/resque/plugins/stages/redis_access.rb +16 -0
- data/lib/resque/plugins/stages/staged_group.rb +181 -0
- data/lib/resque/plugins/stages/staged_group_list.rb +79 -0
- data/lib/resque/plugins/stages/staged_group_stage.rb +275 -0
- data/lib/resque/plugins/stages/staged_job.rb +271 -0
- data/lib/resque/plugins/stages/version.rb +9 -0
- data/lib/resque/server/public/stages.css +56 -0
- data/lib/resque/server/views/_group_stages_list_pagination.erb +67 -0
- data/lib/resque/server/views/_group_stages_list_table.erb +25 -0
- data/lib/resque/server/views/_stage_job_list_pagination.erb +72 -0
- data/lib/resque/server/views/_stage_job_list_table.erb +46 -0
- data/lib/resque/server/views/_staged_group_list_pagination.erb +67 -0
- data/lib/resque/server/views/_staged_group_list_table.erb +44 -0
- data/lib/resque/server/views/group_stages_list.erb +58 -0
- data/lib/resque/server/views/groups.erb +40 -0
- data/lib/resque/server/views/job_details.erb +91 -0
- data/lib/resque/server/views/stage.erb +64 -0
- data/lib/resque/stages_server.rb +240 -0
- data/read_me/groups_list.png +0 -0
- data/read_me/job.png +0 -0
- data/read_me/stage.png +0 -0
- data/read_me/stages.png +0 -0
- data/resque-stages.gemspec +49 -0
- data/spec/rails_helper.rb +40 -0
- data/spec/resque/plugins/stages/cleaner_spec.rb +82 -0
- data/spec/resque/plugins/stages/staged_group_list_spec.rb +96 -0
- data/spec/resque/plugins/stages/staged_group_spec.rb +226 -0
- data/spec/resque/plugins/stages/staged_group_stage_spec.rb +293 -0
- data/spec/resque/plugins/stages/staged_job_spec.rb +324 -0
- data/spec/resque/plugins/stages_spec.rb +369 -0
- data/spec/resque/server/public/stages.css_spec.rb +18 -0
- data/spec/resque/server/views/group_stages_list.erb_spec.rb +67 -0
- data/spec/resque/server/views/groups.erb_spec.rb +81 -0
- data/spec/resque/server/views/job_details.erb_spec.rb +100 -0
- data/spec/resque/server/views/stage.erb_spec.rb +68 -0
- data/spec/spec_helper.rb +104 -0
- data/spec/support/01_utils/fake_logger.rb +7 -0
- data/spec/support/config/redis-auth.yml +12 -0
- data/spec/support/fake_logger.rb +7 -0
- data/spec/support/jobs/basic_job.rb +17 -0
- data/spec/support/jobs/compressed_job.rb +18 -0
- data/spec/support/jobs/retry_job.rb +21 -0
- data/spec/support/purge_all.rb +15 -0
- metadata +297 -0
@@ -0,0 +1,181 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Plugins
|
5
|
+
module Stages
|
6
|
+
# This class represents the toplevel grouping of a set of staged jobs.
|
7
|
+
#
|
8
|
+
# The group defines individual numbered stages (starting with 0) and
|
9
|
+
# intiates subsequent stages as the current stage completes.
|
10
|
+
#
|
11
|
+
# There are methods on the group to initiate groups and add jobs to
|
12
|
+
# individual stages or get/create new stages
|
13
|
+
class StagedGroup
|
14
|
+
include Resque::Plugins::Stages::RedisAccess
|
15
|
+
include Comparable
|
16
|
+
|
17
|
+
attr_reader :group_id
|
18
|
+
|
19
|
+
def initialize(group_id, description: "")
|
20
|
+
@group_id = group_id
|
21
|
+
|
22
|
+
Resque::Plugins::Stages::StagedGroupList.new.add_group(self)
|
23
|
+
|
24
|
+
redis.hsetnx(group_info_key, "created_at", Time.now)
|
25
|
+
self.description = description if description.present?
|
26
|
+
end
|
27
|
+
|
28
|
+
def <=>(other)
|
29
|
+
return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroup)
|
30
|
+
|
31
|
+
group_id <=> other.group_id
|
32
|
+
end
|
33
|
+
|
34
|
+
class << self
|
35
|
+
def within_a_grouping(description = nil)
|
36
|
+
group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: description)
|
37
|
+
|
38
|
+
yield group
|
39
|
+
|
40
|
+
group.initiate
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def initiate
|
45
|
+
stage = next_stage
|
46
|
+
|
47
|
+
if stage
|
48
|
+
stage.initiate
|
49
|
+
else
|
50
|
+
delete
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def description
|
55
|
+
@description ||= redis.hget(group_info_key, "description").presence || group_id
|
56
|
+
end
|
57
|
+
|
58
|
+
def created_at
|
59
|
+
@created_at ||= redis.hget(group_info_key, "created_at").presence.to_time || Time.now
|
60
|
+
end
|
61
|
+
|
62
|
+
def description=(value)
|
63
|
+
@description = value.presence
|
64
|
+
redis.hset(group_info_key, "description", description)
|
65
|
+
end
|
66
|
+
|
67
|
+
def current_stage
|
68
|
+
next_stage || new_stage
|
69
|
+
end
|
70
|
+
|
71
|
+
def last_stage
|
72
|
+
group_stages = stages
|
73
|
+
last_key = group_stages.keys.max
|
74
|
+
|
75
|
+
group_stages[last_key]
|
76
|
+
end
|
77
|
+
|
78
|
+
def add_stage(staged_group_stage)
|
79
|
+
redis.rpush group_key, staged_group_stage.group_stage_id
|
80
|
+
end
|
81
|
+
|
82
|
+
def remove_stage(staged_group_stage)
|
83
|
+
redis.lrem(group_key, 0, staged_group_stage.group_stage_id)
|
84
|
+
end
|
85
|
+
|
86
|
+
def num_stages
|
87
|
+
redis.llen group_key
|
88
|
+
end
|
89
|
+
|
90
|
+
def stages
|
91
|
+
all_stages = redis.lrange(group_key, 0, -1).map { |stage_id| Resque::Plugins::Stages::StagedGroupStage.new(stage_id) }
|
92
|
+
|
93
|
+
all_stages.each_with_object({}) do |stage, hash|
|
94
|
+
num = stage.number
|
95
|
+
num += 1 while hash.key?(num)
|
96
|
+
|
97
|
+
hash[num] = stage
|
98
|
+
stage.number = num if stage.number != num
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def paged_stages(page_num = 1, page_size = nil)
|
103
|
+
page_size ||= 20
|
104
|
+
page_size = page_size.to_i
|
105
|
+
page_size = 20 if page_size < 1
|
106
|
+
start = (page_num - 1) * page_size
|
107
|
+
start = 0 if start >= num_stages || start.negative?
|
108
|
+
|
109
|
+
stages.values[start..start + page_size - 1]
|
110
|
+
end
|
111
|
+
|
112
|
+
def stage(stage_number)
|
113
|
+
found_stage = stages[stage_number]
|
114
|
+
|
115
|
+
found_stage || create_stage(stage_number)
|
116
|
+
end
|
117
|
+
|
118
|
+
def delete
|
119
|
+
stages.each_value(&:delete)
|
120
|
+
|
121
|
+
Resque::Plugins::Stages::StagedGroupList.new.remove_group(self)
|
122
|
+
|
123
|
+
redis.del group_key
|
124
|
+
redis.del group_info_key
|
125
|
+
end
|
126
|
+
|
127
|
+
def stage_completed
|
128
|
+
initiate
|
129
|
+
end
|
130
|
+
|
131
|
+
def blank?
|
132
|
+
!redis.exists(group_key) && !redis.exists(group_info_key)
|
133
|
+
end
|
134
|
+
|
135
|
+
def verify_stage(stage)
|
136
|
+
ids = redis.lrange(group_key, 0, -1)
|
137
|
+
|
138
|
+
return if ids.include?(stage.group_stage_id)
|
139
|
+
|
140
|
+
redis.lpush(group_key, stage.group_stage_id)
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def next_stage
|
146
|
+
group_stages = stages
|
147
|
+
keys = group_stages.keys.sort
|
148
|
+
|
149
|
+
current_number = keys.detect do |key|
|
150
|
+
group_stages[key].status != :complete
|
151
|
+
end
|
152
|
+
|
153
|
+
group_stages[current_number]
|
154
|
+
end
|
155
|
+
|
156
|
+
def group_key
|
157
|
+
"StagedGroup::#{group_id}"
|
158
|
+
end
|
159
|
+
|
160
|
+
def group_info_key
|
161
|
+
"#{group_key}::Info"
|
162
|
+
end
|
163
|
+
|
164
|
+
def new_stage
|
165
|
+
next_stage_number = (last_stage&.number || 0) + 1
|
166
|
+
|
167
|
+
create_stage(next_stage_number)
|
168
|
+
end
|
169
|
+
|
170
|
+
def create_stage(stage_number)
|
171
|
+
stage = Resque::Plugins::Stages::StagedGroupStage.new(SecureRandom.uuid)
|
172
|
+
|
173
|
+
stage.staged_group = self
|
174
|
+
stage.number = stage_number
|
175
|
+
|
176
|
+
stage
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Plugins
|
5
|
+
module Stages
|
6
|
+
# A utility class that keeps track of all Groups and lists them.
|
7
|
+
class StagedGroupList
|
8
|
+
include Resque::Plugins::Stages::RedisAccess
|
9
|
+
|
10
|
+
def groups
|
11
|
+
redis.smembers(list_key).map { |group_id| Resque::Plugins::Stages::StagedGroup.new(group_id) }
|
12
|
+
end
|
13
|
+
|
14
|
+
def order_param(sort_option, current_sort, current_order)
|
15
|
+
current_order ||= "asc"
|
16
|
+
|
17
|
+
if sort_option == current_sort
|
18
|
+
current_order == "asc" ? "desc" : "asc"
|
19
|
+
else
|
20
|
+
"asc"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def paginated_groups(sort_key = :description,
|
25
|
+
sort_order = "asc",
|
26
|
+
page_num = 1,
|
27
|
+
queue_page_size = 20)
|
28
|
+
queue_page_size = queue_page_size.to_i
|
29
|
+
queue_page_size = 20 if queue_page_size < 1
|
30
|
+
|
31
|
+
group_list = sorted_groups(sort_key)
|
32
|
+
|
33
|
+
page_start = (page_num - 1) * queue_page_size
|
34
|
+
page_start = 0 if page_start > group_list.length || page_start.negative?
|
35
|
+
|
36
|
+
(sort_order == "desc" ? group_list.reverse : group_list)[page_start..(page_start + queue_page_size - 1)]
|
37
|
+
end
|
38
|
+
|
39
|
+
def num_groups
|
40
|
+
groups.length
|
41
|
+
end
|
42
|
+
|
43
|
+
def add_group(group)
|
44
|
+
redis.sadd list_key, group.group_id
|
45
|
+
end
|
46
|
+
|
47
|
+
def remove_group(group)
|
48
|
+
redis.srem list_key, group.group_id
|
49
|
+
end
|
50
|
+
|
51
|
+
def delete_all
|
52
|
+
groups.each(&:delete)
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def sorted_groups(sort_key)
|
58
|
+
groups.sort_by do |group|
|
59
|
+
group_sort_value(group, sort_key)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def group_sort_value(group, sort_key)
|
64
|
+
case sort_key.to_sym
|
65
|
+
when :description,
|
66
|
+
:num_stages
|
67
|
+
group.public_send(sort_key)
|
68
|
+
when :created_at
|
69
|
+
group.public_send(sort_key).to_s
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def list_key
|
74
|
+
"StagedGroupList"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,275 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Plugins
|
5
|
+
module Stages
|
6
|
+
# A class which represents a single stage and all of the jobs for that stage.
|
7
|
+
#
|
8
|
+
# Each stage has a status that it progresses through
|
9
|
+
# :pending - not started yet
|
10
|
+
# :running - currently running
|
11
|
+
# :complete - all jobs in the stage are completed
|
12
|
+
|
13
|
+
# rubocop:disable Metrics/ClassLength
|
14
|
+
class StagedGroupStage
|
15
|
+
include Resque::Plugins::Stages::RedisAccess
|
16
|
+
include Comparable
|
17
|
+
|
18
|
+
attr_reader :group_stage_id
|
19
|
+
|
20
|
+
def initialize(group_stage_id)
|
21
|
+
@group_stage_id = group_stage_id
|
22
|
+
end
|
23
|
+
|
24
|
+
def <=>(other)
|
25
|
+
return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroupStage)
|
26
|
+
|
27
|
+
group_stage_id <=> other.group_stage_id
|
28
|
+
end
|
29
|
+
|
30
|
+
def enqueue(klass, *args)
|
31
|
+
job = create_enqueue_job(klass, args)
|
32
|
+
|
33
|
+
return job if status == :pending
|
34
|
+
|
35
|
+
self.status = :running if status != :running
|
36
|
+
|
37
|
+
job.status = :queued
|
38
|
+
Resque.enqueue(*job.enqueue_args)
|
39
|
+
|
40
|
+
job
|
41
|
+
end
|
42
|
+
|
43
|
+
def enqueue_to(queue, klass, *args)
|
44
|
+
job = create_enqueue_job(klass, args)
|
45
|
+
|
46
|
+
return job if status == :pending
|
47
|
+
|
48
|
+
self.status = :running if status != :running
|
49
|
+
|
50
|
+
job.status = :queued
|
51
|
+
Resque.enqueue_to(queue, *job.enqueue_args)
|
52
|
+
|
53
|
+
job
|
54
|
+
end
|
55
|
+
|
56
|
+
def enqueue_at(timestamp, klass, *args)
|
57
|
+
job = create_enqueue_job(klass, args)
|
58
|
+
|
59
|
+
return job if status == :pending
|
60
|
+
|
61
|
+
self.status = :running if status != :running
|
62
|
+
|
63
|
+
job.status = :queued
|
64
|
+
Resque.enqueue_at(timestamp, *job.enqueue_args)
|
65
|
+
|
66
|
+
job
|
67
|
+
end
|
68
|
+
|
69
|
+
def enqueue_at_with_queue(queue, timestamp, klass, *args)
|
70
|
+
job = create_enqueue_job(klass, args)
|
71
|
+
|
72
|
+
return job if status == :pending
|
73
|
+
|
74
|
+
self.status = :running if status != :running
|
75
|
+
|
76
|
+
job.status = :queued
|
77
|
+
Resque.enqueue_at_with_queue(queue, timestamp, *job.enqueue_args)
|
78
|
+
|
79
|
+
job
|
80
|
+
end
|
81
|
+
|
82
|
+
def enqueue_in(number_of_seconds_from_now, klass, *args)
|
83
|
+
job = create_enqueue_job(klass, args)
|
84
|
+
|
85
|
+
return job if status == :pending
|
86
|
+
|
87
|
+
self.status = :running if status != :running
|
88
|
+
|
89
|
+
job.status = :queued
|
90
|
+
Resque.enqueue_in(number_of_seconds_from_now, *job.enqueue_args)
|
91
|
+
|
92
|
+
job
|
93
|
+
end
|
94
|
+
|
95
|
+
def enqueue_in_with_queue(queue, number_of_seconds_from_now, klass, *args)
|
96
|
+
job = create_enqueue_job(klass, args)
|
97
|
+
|
98
|
+
return job if status == :pending
|
99
|
+
|
100
|
+
self.status = :running if status != :running
|
101
|
+
|
102
|
+
job.status = :queued
|
103
|
+
Resque.enqueue_in_with_queue(queue, number_of_seconds_from_now, *job.enqueue_args)
|
104
|
+
|
105
|
+
job
|
106
|
+
end
|
107
|
+
|
108
|
+
def status
|
109
|
+
redis.hget(staged_group_key, "status")&.to_sym || :pending
|
110
|
+
end
|
111
|
+
|
112
|
+
def status=(value)
|
113
|
+
redis.hset(staged_group_key, "status", value.to_s)
|
114
|
+
|
115
|
+
staged_group&.stage_completed if status == :complete
|
116
|
+
end
|
117
|
+
|
118
|
+
def number
|
119
|
+
redis.hget(staged_group_key, "number")&.to_i || 1
|
120
|
+
end
|
121
|
+
|
122
|
+
def number=(value)
|
123
|
+
redis.hset(staged_group_key, "number", value)
|
124
|
+
end
|
125
|
+
|
126
|
+
def staged_group
|
127
|
+
return nil if staged_group_id.blank?
|
128
|
+
|
129
|
+
@staged_group ||= Resque::Plugins::Stages::StagedGroup.new(staged_group_id)
|
130
|
+
end
|
131
|
+
|
132
|
+
def staged_group=(value)
|
133
|
+
@staged_group = value
|
134
|
+
@staged_group_id = value.group_id
|
135
|
+
|
136
|
+
value.add_stage(self)
|
137
|
+
redis.hset(staged_group_key, "staged_group_id", value.group_id)
|
138
|
+
end
|
139
|
+
|
140
|
+
def jobs(start = 0, stop = -1)
|
141
|
+
redis.lrange(stage_key, start, stop).map { |id| Resque::Plugins::Stages::StagedJob.new(id) }
|
142
|
+
end
|
143
|
+
|
144
|
+
def jobs_by_status(status)
|
145
|
+
jobs.select { |job| job.status == status }
|
146
|
+
end
|
147
|
+
|
148
|
+
def paginated_jobs(sort_key = :class_name,
|
149
|
+
sort_order = "asc",
|
150
|
+
page_num = 1,
|
151
|
+
queue_page_size = 20)
|
152
|
+
queue_page_size = queue_page_size.to_i
|
153
|
+
queue_page_size = 20 if queue_page_size < 1
|
154
|
+
|
155
|
+
job_list = sorted_jobs(sort_key)
|
156
|
+
|
157
|
+
page_start = (page_num - 1) * queue_page_size
|
158
|
+
page_start = 0 if page_start > job_list.length || page_start.negative?
|
159
|
+
|
160
|
+
(sort_order == "desc" ? job_list.reverse : job_list)[page_start..(page_start + queue_page_size - 1)]
|
161
|
+
end
|
162
|
+
|
163
|
+
def order_param(sort_option, current_sort, current_order)
|
164
|
+
current_order ||= "asc"
|
165
|
+
|
166
|
+
if sort_option == current_sort
|
167
|
+
current_order == "asc" ? "desc" : "asc"
|
168
|
+
else
|
169
|
+
"asc"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def num_jobs
|
174
|
+
redis.llen(stage_key)
|
175
|
+
end
|
176
|
+
|
177
|
+
def add_job(staged_group_job)
|
178
|
+
redis.rpush stage_key, staged_group_job.job_id
|
179
|
+
end
|
180
|
+
|
181
|
+
def remove_job(staged_group_job)
|
182
|
+
redis.lrem(stage_key, 0, staged_group_job.job_id)
|
183
|
+
end
|
184
|
+
|
185
|
+
def delete
|
186
|
+
jobs.each(&:delete)
|
187
|
+
|
188
|
+
staged_group&.remove_stage self
|
189
|
+
|
190
|
+
redis.del stage_key
|
191
|
+
redis.del staged_group_key
|
192
|
+
end
|
193
|
+
|
194
|
+
def initiate
|
195
|
+
self.status = :running
|
196
|
+
|
197
|
+
jobs.each do |job|
|
198
|
+
next if job.completed?
|
199
|
+
next if job.queued?
|
200
|
+
|
201
|
+
job.enqueue_job
|
202
|
+
end
|
203
|
+
|
204
|
+
job_completed
|
205
|
+
end
|
206
|
+
|
207
|
+
def job_completed
|
208
|
+
self.status = :complete if jobs.all?(&:completed?)
|
209
|
+
end
|
210
|
+
|
211
|
+
def blank?
|
212
|
+
!redis.exists(stage_key) && !redis.exists(staged_group_key)
|
213
|
+
end
|
214
|
+
|
215
|
+
def verify
|
216
|
+
return build_new_structure if staged_group.blank?
|
217
|
+
|
218
|
+
staged_group.verify_stage(self)
|
219
|
+
end
|
220
|
+
|
221
|
+
def verify_job(job)
|
222
|
+
ids = redis.lrange(stage_key, 0, -1)
|
223
|
+
|
224
|
+
return if ids.include?(job.job_id)
|
225
|
+
|
226
|
+
redis.lpush(stage_key, job.job_id)
|
227
|
+
end
|
228
|
+
|
229
|
+
private
|
230
|
+
|
231
|
+
def build_new_structure
|
232
|
+
group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid)
|
233
|
+
|
234
|
+
self.staged_group = group
|
235
|
+
end
|
236
|
+
|
237
|
+
def create_enqueue_job(klass, args)
|
238
|
+
Resque::Plugins::Stages::StagedJob.create_job self, klass, *args
|
239
|
+
end
|
240
|
+
|
241
|
+
def stage_key
|
242
|
+
"StagedGroupStage::#{group_stage_id}"
|
243
|
+
end
|
244
|
+
|
245
|
+
def staged_group_key
|
246
|
+
"#{stage_key}::staged_group"
|
247
|
+
end
|
248
|
+
|
249
|
+
def staged_group_id
|
250
|
+
@staged_group_id ||= redis.hget(staged_group_key, "staged_group_id")
|
251
|
+
end
|
252
|
+
|
253
|
+
def sorted_jobs(sort_key)
|
254
|
+
jobs.sort_by do |job|
|
255
|
+
job_sort_value(job, sort_key)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
def job_sort_value(job, sort_key)
|
260
|
+
case sort_key.to_sym
|
261
|
+
when :class_name,
|
262
|
+
:status,
|
263
|
+
:status_message
|
264
|
+
job.public_send(sort_key)
|
265
|
+
|
266
|
+
when :queue_time
|
267
|
+
job.public_send(sort_key).to_s
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
# rubocop:enable Metrics/ClassLength
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|