resque-stages 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yml +103 -0
  5. data/.rubocop_todo.yml +34 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +6 -0
  8. data/CODE_OF_CONDUCT.md +74 -0
  9. data/Gemfile +9 -0
  10. data/Gemfile.lock +172 -0
  11. data/LICENSE.txt +21 -0
  12. data/README.md +250 -0
  13. data/Rakefile +8 -0
  14. data/bin/console +16 -0
  15. data/bin/setup +8 -0
  16. data/lib/resque-stages.rb +11 -0
  17. data/lib/resque/plugins/stages.rb +110 -0
  18. data/lib/resque/plugins/stages/cleaner.rb +36 -0
  19. data/lib/resque/plugins/stages/redis_access.rb +16 -0
  20. data/lib/resque/plugins/stages/staged_group.rb +181 -0
  21. data/lib/resque/plugins/stages/staged_group_list.rb +79 -0
  22. data/lib/resque/plugins/stages/staged_group_stage.rb +275 -0
  23. data/lib/resque/plugins/stages/staged_job.rb +271 -0
  24. data/lib/resque/plugins/stages/version.rb +9 -0
  25. data/lib/resque/server/public/stages.css +56 -0
  26. data/lib/resque/server/views/_group_stages_list_pagination.erb +67 -0
  27. data/lib/resque/server/views/_group_stages_list_table.erb +25 -0
  28. data/lib/resque/server/views/_stage_job_list_pagination.erb +72 -0
  29. data/lib/resque/server/views/_stage_job_list_table.erb +46 -0
  30. data/lib/resque/server/views/_staged_group_list_pagination.erb +67 -0
  31. data/lib/resque/server/views/_staged_group_list_table.erb +44 -0
  32. data/lib/resque/server/views/group_stages_list.erb +58 -0
  33. data/lib/resque/server/views/groups.erb +40 -0
  34. data/lib/resque/server/views/job_details.erb +91 -0
  35. data/lib/resque/server/views/stage.erb +64 -0
  36. data/lib/resque/stages_server.rb +240 -0
  37. data/read_me/groups_list.png +0 -0
  38. data/read_me/job.png +0 -0
  39. data/read_me/stage.png +0 -0
  40. data/read_me/stages.png +0 -0
  41. data/resque-stages.gemspec +49 -0
  42. data/spec/rails_helper.rb +40 -0
  43. data/spec/resque/plugins/stages/cleaner_spec.rb +82 -0
  44. data/spec/resque/plugins/stages/staged_group_list_spec.rb +96 -0
  45. data/spec/resque/plugins/stages/staged_group_spec.rb +226 -0
  46. data/spec/resque/plugins/stages/staged_group_stage_spec.rb +293 -0
  47. data/spec/resque/plugins/stages/staged_job_spec.rb +324 -0
  48. data/spec/resque/plugins/stages_spec.rb +369 -0
  49. data/spec/resque/server/public/stages.css_spec.rb +18 -0
  50. data/spec/resque/server/views/group_stages_list.erb_spec.rb +67 -0
  51. data/spec/resque/server/views/groups.erb_spec.rb +81 -0
  52. data/spec/resque/server/views/job_details.erb_spec.rb +100 -0
  53. data/spec/resque/server/views/stage.erb_spec.rb +68 -0
  54. data/spec/spec_helper.rb +104 -0
  55. data/spec/support/01_utils/fake_logger.rb +7 -0
  56. data/spec/support/config/redis-auth.yml +12 -0
  57. data/spec/support/fake_logger.rb +7 -0
  58. data/spec/support/jobs/basic_job.rb +17 -0
  59. data/spec/support/jobs/compressed_job.rb +18 -0
  60. data/spec/support/jobs/retry_job.rb +21 -0
  61. data/spec/support/purge_all.rb +15 -0
  62. metadata +297 -0
Binary file
Binary file
Binary file
Binary file
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/resque/plugins/stages/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "resque-stages"
7
+ spec.version = Resque::Plugins::Stages::VERSION
8
+ spec.authors = ["RealNobody"]
9
+ spec.email = ["RealNobody1@cox.net"]
10
+
11
+ spec.summary = "A Resque gem for executing batches of jobs in stages to ensure that some jobs complete execution before other jobs."
12
+ spec.description = "A Resque gem for executing batches of jobs in stages. All jobs in a stage must complete before any job in the next" \
13
+ " stage is started allowing you to be sure that jobs are not executed out of sequence."
14
+ spec.homepage = "https://github.com/RealNobody/resque-stages"
15
+ spec.license = "MIT"
16
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
17
+
18
+ # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
19
+
20
+ spec.metadata["homepage_uri"] = spec.homepage
21
+ spec.metadata["source_code_uri"] = "https://github.com/RealNobody/resque-stages"
22
+ spec.metadata["changelog_uri"] = "https://github.com/RealNobody/resque-stages"
23
+
24
+ # Specify which files should be added to the gem when it is released.
25
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
26
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
27
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
28
+ end
29
+
30
+ spec.test_files = Dir["spec/**/*"]
31
+
32
+ spec.bindir = "exe"
33
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
34
+ spec.require_paths = ["lib"]
35
+
36
+ spec.add_dependency "redis"
37
+ spec.add_dependency "redis-namespace"
38
+ spec.add_dependency "resque"
39
+
40
+ spec.add_development_dependency "activesupport"
41
+ spec.add_development_dependency "cornucopia"
42
+ spec.add_development_dependency "faker"
43
+ spec.add_development_dependency "gem-release"
44
+ spec.add_development_dependency "resque-compressible"
45
+ spec.add_development_dependency "resque-retry"
46
+ spec.add_development_dependency "rspec-rails", "> 3.9.1"
47
+ spec.add_development_dependency "rubocop"
48
+ spec.add_development_dependency "simplecov"
49
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "simplecov"
4
+ SimpleCov.start
5
+
6
+ require "active_support/testing/time_helpers"
7
+ require "spec_helper"
8
+ require "cornucopia/rspec_hooks"
9
+ require "resque-stages"
10
+ require "resque-compressible"
11
+ require "resque-retry"
12
+ require "faker"
13
+ require "rack/test"
14
+ require "resque/server"
15
+ require "resque/stages_server"
16
+ require "resque-scheduler"
17
+
18
+ Dir[File.expand_path("spec/support/**/*.rb"), File.dirname(__FILE__)].sort.each do |f|
19
+ require f unless File.directory?(f)
20
+ end
21
+
22
+ FileUtils.mkdir_p(File.expand_path("../log", File.dirname(__FILE__)))
23
+
24
+ redis_logger = Logger.new(File.expand_path("../log/redis.log", File.dirname(__FILE__)))
25
+ redis_logger.level = Logger::DEBUG
26
+ redis_logger.formatter = Logger::Formatter.new
27
+
28
+ redis_options = YAML.load_file(File.expand_path("support/config/redis-auth.yml", File.dirname(__FILE__)))
29
+ Redis.current = Redis.new(redis_options.merge(logger: redis_logger))
30
+
31
+ Resque.redis = Redis.new(redis_options)
32
+ Resque.inline = true
33
+
34
+ RSpec.configure do |config|
35
+ config.include ActiveSupport::Testing::TimeHelpers
36
+ end
37
+
38
+ # Cornucopia::Util::Configuration.context_seed = 1
39
+ # Cornucopia::Util::Configuration.seed = 1
40
+ # Cornucopia::Util::Configuration.order_seed = 1
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails_helper"
4
+
5
+ RSpec.describe Resque::Plugins::Stages::Cleaner do
6
+ let(:group) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid) }
7
+ let(:load_group) { Resque::Plugins::Stages::StagedGroup.new(group.group_id) }
8
+ let(:stage) { group.current_stage }
9
+ let(:load_stage) { Resque::Plugins::Stages::StagedGroupStage.new(stage.group_stage_id) }
10
+ let!(:job) { stage.enqueue BasicJob }
11
+ let(:load_job) { Resque::Plugins::Stages::StagedJob.new(job.job_id) }
12
+
13
+ describe "purge_all" do
14
+ it "deletes all values" do
15
+ Resque::Plugins::Stages::Cleaner.purge_all
16
+
17
+ expect(job).to be_blank
18
+ expect(stage).to be_blank
19
+ expect(group).to be_blank
20
+ end
21
+ end
22
+
23
+ describe "cleanup_jobs" do
24
+ it "does not recreate the group if the group info is deleted" do
25
+ job.redis.keys("StagedGroup::*::Info").each { |key| job.redis.del(key) }
26
+
27
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
28
+ expect(load_group.stages.values).to be_include stage
29
+ end
30
+
31
+ it "does not recreate the group if the group list is deleted" do
32
+ job.redis.keys("StagedGroup::*").each do |key|
33
+ next if key.include?("::Info")
34
+
35
+ job.redis.del(key)
36
+ end
37
+
38
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
39
+ expect(load_group.stages.values).to be_include stage
40
+ end
41
+
42
+ it "recreates the group if the group is deleted" do
43
+ job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
44
+
45
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
46
+ expect(load_group.stages.values).to be_include stage
47
+ end
48
+
49
+ it "creates a new group if the group is deleted and cannot be found" do
50
+ job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
51
+ job.redis.keys("StagedGroupStage::*::staged_group").each { |key| job.redis.del(key) }
52
+
53
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
54
+ expect(load_group.stages.values).not_to be_include stage
55
+ expect(load_stage.staged_group.stages.values).to be_include stage
56
+ expect(load_stage.jobs).to be_include(job)
57
+ end
58
+
59
+ it "does not create a new stage if it can be found" do
60
+ job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
61
+ job.redis.keys("StagedGroupStage::*").each do |key|
62
+ next if key.include?("::staged_group")
63
+
64
+ job.redis.del(key)
65
+ end
66
+
67
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
68
+ expect(load_group.stages.values).to be_include stage
69
+ expect(load_stage.jobs).to be_include(job)
70
+ end
71
+
72
+ it "create a new stage if it can be found" do
73
+ job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
74
+ job.redis.keys("StagedGroupStage::*").each { |key| job.redis.del(key) }
75
+
76
+ Resque::Plugins::Stages::Cleaner.cleanup_jobs
77
+ expect(load_stage.jobs).not_to be_include(job)
78
+ expect(load_job.staged_group_stage.jobs).to be_include(job)
79
+ expect(load_job.staged_group_stage.staged_group.stages.values).to be_include(load_job.staged_group_stage)
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails_helper"
4
+
5
+ RSpec.describe Resque::Plugins::Stages::StagedGroupList do
6
+ let(:groups) do
7
+ [travel_to(3.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "s Description 1") },
8
+ travel_to(0.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "a Description 2") },
9
+ travel_to(2.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "not A Description 3") },
10
+ travel_to(1.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid) }]
11
+ end
12
+ let(:group_list) { Resque::Plugins::Stages::StagedGroupList.new }
13
+
14
+ describe "groups" do
15
+ it "lists all groups" do
16
+ groups
17
+ expect(group_list.groups.map(&:group_id).sort).to eq groups.map(&:group_id).sort
18
+ end
19
+ end
20
+
21
+ describe "num_groups" do
22
+ it "counts the groups" do
23
+ groups
24
+ expect(group_list.num_groups).to eq 4
25
+ end
26
+ end
27
+
28
+ describe "paginated_groups" do
29
+ it "sorts groups by description" do
30
+ groups
31
+ expect(group_list.paginated_groups("description", "asc", 2, 2)).to eq [groups[2], groups[0]]
32
+ end
33
+
34
+ it "sorts groups by created_at" do
35
+ groups
36
+ expect(group_list.paginated_groups("created_at", "asc", 2, 2)).to eq [groups[3], groups[1]]
37
+ end
38
+
39
+ it "sorts groups by num_stages" do
40
+ groups.each.with_index do |group, index|
41
+ index.times { |count| group.stage(count) }
42
+ end
43
+
44
+ expect(group_list.paginated_groups("num_stages", "asc", 2, 2)).to eq [groups[2], groups[3]]
45
+ end
46
+ end
47
+
48
+ describe "delete_all" do
49
+ it "deletes all groups" do
50
+ groups
51
+ group_list.delete_all
52
+
53
+ expect(group_list.groups).to be_empty
54
+ end
55
+ end
56
+
57
+ describe "add_group" do
58
+ it "adds a group to the list when the group is created" do
59
+ expect(group_list.groups).to be_empty
60
+
61
+ group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "Description 1")
62
+
63
+ expect(group_list.groups).to eq [group]
64
+ end
65
+ end
66
+
67
+ describe "remove_group" do
68
+ it "removes a group from the list when the group is deleted" do
69
+ expect(group_list.groups).to be_empty
70
+
71
+ group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "Description 1")
72
+
73
+ expect(group_list.groups).to eq [group]
74
+
75
+ group.delete
76
+
77
+ expect(group_list.groups).to be_empty
78
+ end
79
+ end
80
+
81
+ describe "#order_param" do
82
+ it "returns asc for any column other than the current one" do
83
+ expect(group_list.order_param("sort_option",
84
+ "current_sort",
85
+ %w[asc desc].sample)).to eq "asc"
86
+ end
87
+
88
+ it "returns desc for the current column if it is asc" do
89
+ expect(group_list.order_param("sort_option", "sort_option", "asc")).to eq "desc"
90
+ end
91
+
92
+ it "returns asc for the current column if it is desc" do
93
+ expect(group_list.order_param("sort_option", "sort_option", "desc")).to eq "asc"
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,226 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails_helper"
4
+
5
+ RSpec.describe Resque::Plugins::Stages::StagedGroup do
6
+ let(:group) { Resque::Plugins::Stages::StagedGroup.new SecureRandom.uuid }
7
+ let(:load_group) { Resque::Plugins::Stages::StagedGroup.new group.group_id }
8
+
9
+ describe "within_a_grouping" do
10
+ it "yields a grouping" do
11
+ Resque::Plugins::Stages::StagedGroup.within_a_grouping do |grouping|
12
+ expect(grouping).to be_a Resque::Plugins::Stages::StagedGroup
13
+ end
14
+ end
15
+
16
+ it "initiates the grouping" do
17
+ test_grouping = nil
18
+
19
+ Resque::Plugins::Stages::StagedGroup.within_a_grouping do |grouping|
20
+ test_grouping = grouping
21
+ allow(grouping).to receive(:initiate)
22
+ end
23
+
24
+ expect(test_grouping).to have_received(:initiate)
25
+ end
26
+ end
27
+
28
+ describe "description" do
29
+ it "defaults the description to the group_id" do
30
+ group.description = nil
31
+
32
+ expect(load_group.description).to eq group.group_id.to_s
33
+ end
34
+
35
+ it "sets the description to anything" do
36
+ group.description = "This is a description"
37
+
38
+ expect(load_group.description).to eq "This is a description"
39
+ end
40
+ end
41
+
42
+ describe "created_at" do
43
+ it "returns the created_at time" do
44
+ time = Time.now
45
+
46
+ travel_to(time) do
47
+ time = Time.now
48
+ group
49
+ end
50
+
51
+ travel_to(time + 1.day) do
52
+ expect(load_group.created_at).to eq time
53
+ end
54
+ end
55
+ end
56
+
57
+ context "with stages" do
58
+ let(:stages) do
59
+ stage = group.stage(8)
60
+ Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
61
+ stage.redis.hset(stage.send(:staged_group_key), "status", :running.to_s)
62
+
63
+ stage = group.stage(1)
64
+ Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
65
+ stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
66
+
67
+ stage = group.stage(2)
68
+ Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
69
+ stage.redis.hset(stage.send(:staged_group_key), "status", :pending.to_s)
70
+
71
+ stage = group.stage(0)
72
+ Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
73
+ stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
74
+
75
+ group.stages
76
+ end
77
+
78
+ describe "initiate" do
79
+ it "initiates the first non-completed stage" do
80
+ allow(group).to receive(:stages).and_return stages
81
+
82
+ stages.each_value { |stage| allow(stage).to receive(:initiate) }
83
+
84
+ group.initiate
85
+
86
+ expect(stages[2]).to have_received(:initiate)
87
+ stages.each do |key, stage|
88
+ next if key == 2
89
+
90
+ expect(stage).not_to have_received(:initiate)
91
+ end
92
+ end
93
+ end
94
+
95
+ describe "current_stage" do
96
+ it "returns the first non-complete stage" do
97
+ stages
98
+
99
+ expect(load_group.current_stage).to eq stages[2]
100
+ end
101
+
102
+ it "returns the first non-complete stage" do
103
+ stage = stages[2]
104
+ stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
105
+
106
+ expect(load_group.current_stage).to eq stages[8]
107
+ end
108
+
109
+ it "builds a new stage if all current stages are complete" do
110
+ stages[2].status = :complete
111
+ stages[8].status = :complete
112
+
113
+ stage = load_group.current_stage
114
+ expect(stage.number).to eq 1
115
+ end
116
+ end
117
+
118
+ describe "stage_completed" do
119
+ it "initiates the next stage when a stage is compelted" do
120
+ allow(group).to receive(:stages).and_return stages
121
+
122
+ stages.each_value do |stage|
123
+ allow(stage).to receive(:initiate)
124
+ allow(stage).to receive(:staged_group).and_return group
125
+ end
126
+
127
+ stages[8].status = :complete
128
+
129
+ expect(stages[2]).to have_received(:initiate)
130
+ end
131
+
132
+ it "deletes the group if the last stage is completed" do
133
+ allow(group).to receive(:stages).and_return stages
134
+ allow(group).to receive(:delete)
135
+
136
+ stages.each_value do |stage|
137
+ allow(stage).to receive(:initiate)
138
+ allow(stage).to receive(:staged_group).and_return group
139
+ end
140
+
141
+ stages[8].status = :complete
142
+ stages[2].status = :complete
143
+
144
+ expect(group).to have_received(:delete)
145
+ end
146
+ end
147
+
148
+ describe "last_stage" do
149
+ it "returns the largest stage" do
150
+ stages
151
+
152
+ expect(load_group.last_stage).to eq stages[8]
153
+ end
154
+ end
155
+
156
+ describe "stages" do
157
+ it "returns a hash of all stages keyed by the stage number" do
158
+ stages
159
+
160
+ expect(load_group.stages).to eq stages
161
+ end
162
+ end
163
+
164
+ describe "paged_stages" do
165
+ it "a page of data" do
166
+ stages
167
+
168
+ expect(load_group.paged_stages(1, 2)).to eq [stages[8], stages[1]]
169
+ end
170
+
171
+ it "a mid page of data" do
172
+ stages
173
+
174
+ expect(load_group.paged_stages(2, 2)).to eq [stages[2], stages[0]]
175
+ end
176
+ end
177
+
178
+ describe "num_stages" do
179
+ it "a page of data" do
180
+ stages
181
+
182
+ expect(load_group.num_stages).to eq 4
183
+ end
184
+ end
185
+
186
+ describe "stage(value)" do
187
+ it "returns the stage with that number if it exists" do
188
+ stages
189
+
190
+ stage = load_group.stage(2)
191
+ expect(stage).to eq stages[2]
192
+ expect(stage.number).to eq 2
193
+ end
194
+
195
+ it "returns a new stage with that number if it does not exist" do
196
+ stages
197
+
198
+ stage = load_group.stage(6)
199
+ expect(stages).not_to be_include stage
200
+ expect(stage.number).to eq 6
201
+
202
+ expect(load_group.stages.length).to eq stages.length + 1
203
+ end
204
+ end
205
+
206
+ describe "delete" do
207
+ it "deletes all stages" do
208
+ allow(group).to receive(:stages).and_return stages
209
+
210
+ stages.each { |_key, stage| allow(stage).to receive(:delete) }
211
+
212
+ group.delete
213
+
214
+ stages.each_value do |stage|
215
+ expect(stage).to have_received(:delete)
216
+ end
217
+ end
218
+
219
+ it "removes a stage when it is deleted" do
220
+ stages[2].delete
221
+
222
+ expect(load_group.stages[2]).to be_nil
223
+ end
224
+ end
225
+ end
226
+ end