resque-stages 0.0.1

data/resque-stages.gemspec

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "lib/resque/plugins/stages/version"
+
+Gem::Specification.new do |spec|
+  spec.name    = "resque-stages"
+  spec.version = Resque::Plugins::Stages::VERSION
+  spec.authors = ["RealNobody"]
+  spec.email   = ["RealNobody1@cox.net"]
+
+  spec.summary               = "A Resque gem for executing batches of jobs in stages to ensure that some jobs complete execution before other jobs."
+  spec.description           = "A Resque gem for executing batches of jobs in stages.  All jobs in a stage must complete before any job in the next" \
+                               " stage is started allowing you to be sure that jobs are not executed out of sequence."
+  spec.homepage              = "https://github.com/RealNobody/resque-stages"
+  spec.license               = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"]    = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/RealNobody/resque-stages"
+  spec.metadata["changelog_uri"]   = "https://github.com/RealNobody/resque-stages"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.test_files = Dir["spec/**/*"]
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+  spec.add_dependency "redis-namespace"
+  spec.add_dependency "resque"
+
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "cornucopia"
+  spec.add_development_dependency "faker"
+  spec.add_development_dependency "gem-release"
+  spec.add_development_dependency "resque-compressible"
+  spec.add_development_dependency "resque-retry"
+  spec.add_development_dependency "rspec-rails", "> 3.9.1"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov"
+end

data/spec/rails_helper.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "simplecov"
+SimpleCov.start
+
+require "active_support/testing/time_helpers"
+require "spec_helper"
+require "cornucopia/rspec_hooks"
+require "resque-stages"
+require "resque-compressible"
+require "resque-retry"
+require "faker"
+require "rack/test"
+require "resque/server"
+require "resque/stages_server"
+require "resque-scheduler"
+
+Dir[File.expand_path("spec/support/**/*.rb"), File.dirname(__FILE__)].sort.each do |f|
+  require f unless File.directory?(f)
+end
+
+FileUtils.mkdir_p(File.expand_path("../log", File.dirname(__FILE__)))
+
+redis_logger           = Logger.new(File.expand_path("../log/redis.log", File.dirname(__FILE__)))
+redis_logger.level     = Logger::DEBUG
+redis_logger.formatter = Logger::Formatter.new
+
+redis_options = YAML.load_file(File.expand_path("support/config/redis-auth.yml", File.dirname(__FILE__)))
+Redis.current = Redis.new(redis_options.merge(logger: redis_logger))
+
+Resque.redis  = Redis.new(redis_options)
+Resque.inline = true
+
+RSpec.configure do |config|
+  config.include ActiveSupport::Testing::TimeHelpers
+end
+
+# Cornucopia::Util::Configuration.context_seed = 1
+# Cornucopia::Util::Configuration.seed         = 1
+# Cornucopia::Util::Configuration.order_seed   = 1

data/spec/resque/plugins/stages/cleaner_spec.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Resque::Plugins::Stages::Cleaner do
+  let(:group) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid) }
+  let(:load_group) { Resque::Plugins::Stages::StagedGroup.new(group.group_id) }
+  let(:stage) { group.current_stage }
+  let(:load_stage) { Resque::Plugins::Stages::StagedGroupStage.new(stage.group_stage_id) }
+  let!(:job) { stage.enqueue BasicJob }
+  let(:load_job) { Resque::Plugins::Stages::StagedJob.new(job.job_id) }
+
+  describe "purge_all" do
+    it "deletes all values" do
+      Resque::Plugins::Stages::Cleaner.purge_all
+
+      expect(job).to be_blank
+      expect(stage).to be_blank
+      expect(group).to be_blank
+    end
+  end
+
+  describe "cleanup_jobs" do
+    it "does not recreate the group if the group info is deleted" do
+      job.redis.keys("StagedGroup::*::Info").each { |key| job.redis.del(key) }
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_group.stages.values).to be_include stage
+    end
+
+    it "does not recreate the group if the group list is deleted" do
+      job.redis.keys("StagedGroup::*").each do |key|
+        next if key.include?("::Info")
+
+        job.redis.del(key)
+      end
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_group.stages.values).to be_include stage
+    end
+
+    it "recreates the group if the group is deleted" do
+      job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_group.stages.values).to be_include stage
+    end
+
+    it "creates a new group if the group is deleted and cannot be found" do
+      job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
+      job.redis.keys("StagedGroupStage::*::staged_group").each { |key| job.redis.del(key) }
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_group.stages.values).not_to be_include stage
+      expect(load_stage.staged_group.stages.values).to be_include stage
+      expect(load_stage.jobs).to be_include(job)
+    end
+
+    it "does not create a new stage if it can be found" do
+      job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
+      job.redis.keys("StagedGroupStage::*").each do |key|
+        next if key.include?("::staged_group")
+
+        job.redis.del(key)
+      end
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_group.stages.values).to be_include stage
+      expect(load_stage.jobs).to be_include(job)
+    end
+
+    it "create a new stage if it can be found" do
+      job.redis.keys("StagedGroup::*").each { |key| job.redis.del(key) }
+      job.redis.keys("StagedGroupStage::*").each { |key| job.redis.del(key) }
+
+      Resque::Plugins::Stages::Cleaner.cleanup_jobs
+      expect(load_stage.jobs).not_to be_include(job)
+      expect(load_job.staged_group_stage.jobs).to be_include(job)
+      expect(load_job.staged_group_stage.staged_group.stages.values).to be_include(load_job.staged_group_stage)
+    end
+  end
+end

data/spec/resque/plugins/stages/staged_group_list_spec.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Resque::Plugins::Stages::StagedGroupList do
+  let(:groups) do
+    [travel_to(3.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "s Description 1") },
+     travel_to(0.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "a Description 2") },
+     travel_to(2.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "not A Description 3") },
+     travel_to(1.hours.ago) { Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid) }]
+  end
+  let(:group_list) { Resque::Plugins::Stages::StagedGroupList.new }
+
+  describe "groups" do
+    it "lists all groups" do
+      groups
+      expect(group_list.groups.map(&:group_id).sort).to eq groups.map(&:group_id).sort
+    end
+  end
+
+  describe "num_groups" do
+    it "counts the groups" do
+      groups
+      expect(group_list.num_groups).to eq 4
+    end
+  end
+
+  describe "paginated_groups" do
+    it "sorts groups by description" do
+      groups
+      expect(group_list.paginated_groups("description", "asc", 2, 2)).to eq [groups[2], groups[0]]
+    end
+
+    it "sorts groups by created_at" do
+      groups
+      expect(group_list.paginated_groups("created_at", "asc", 2, 2)).to eq [groups[3], groups[1]]
+    end
+
+    it "sorts groups by num_stages" do
+      groups.each.with_index do |group, index|
+        index.times { |count| group.stage(count) }
+      end
+
+      expect(group_list.paginated_groups("num_stages", "asc", 2, 2)).to eq [groups[2], groups[3]]
+    end
+  end
+
+  describe "delete_all" do
+    it "deletes all groups" do
+      groups
+      group_list.delete_all
+
+      expect(group_list.groups).to be_empty
+    end
+  end
+
+  describe "add_group" do
+    it "adds a group to the list when the group is created" do
+      expect(group_list.groups).to be_empty
+
+      group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "Description 1")
+
+      expect(group_list.groups).to eq [group]
+    end
+  end
+
+  describe "remove_group" do
+    it "removes a group from the list when the group is deleted" do
+      expect(group_list.groups).to be_empty
+
+      group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: "Description 1")
+
+      expect(group_list.groups).to eq [group]
+
+      group.delete
+
+      expect(group_list.groups).to be_empty
+    end
+  end
+
+  describe "#order_param" do
+    it "returns asc for any column other than the current one" do
+      expect(group_list.order_param("sort_option",
+                                    "current_sort",
+                                    %w[asc desc].sample)).to eq "asc"
+    end
+
+    it "returns desc for the current column if it is asc" do
+      expect(group_list.order_param("sort_option", "sort_option", "asc")).to eq "desc"
+    end
+
+    it "returns asc for the current column if it is desc" do
+      expect(group_list.order_param("sort_option", "sort_option", "desc")).to eq "asc"
+    end
+  end
+end

data/spec/resque/plugins/stages/staged_group_spec.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Resque::Plugins::Stages::StagedGroup do
+  let(:group) { Resque::Plugins::Stages::StagedGroup.new SecureRandom.uuid }
+  let(:load_group) { Resque::Plugins::Stages::StagedGroup.new group.group_id }
+
+  describe "within_a_grouping" do
+    it "yields a grouping" do
+      Resque::Plugins::Stages::StagedGroup.within_a_grouping do |grouping|
+        expect(grouping).to be_a Resque::Plugins::Stages::StagedGroup
+      end
+    end
+
+    it "initiates the grouping" do
+      test_grouping = nil
+
+      Resque::Plugins::Stages::StagedGroup.within_a_grouping do |grouping|
+        test_grouping = grouping
+        allow(grouping).to receive(:initiate)
+      end
+
+      expect(test_grouping).to have_received(:initiate)
+    end
+  end
+
+  describe "description" do
+    it "defaults the description to the group_id" do
+      group.description = nil
+
+      expect(load_group.description).to eq group.group_id.to_s
+    end
+
+    it "sets the description to anything" do
+      group.description = "This is a description"
+
+      expect(load_group.description).to eq "This is a description"
+    end
+  end
+
+  describe "created_at" do
+    it "returns the created_at time" do
+      time = Time.now
+
+      travel_to(time) do
+        time = Time.now
+        group
+      end
+
+      travel_to(time + 1.day) do
+        expect(load_group.created_at).to eq time
+      end
+    end
+  end
+
+  context "with stages" do
+    let(:stages) do
+      stage = group.stage(8)
+      Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
+      stage.redis.hset(stage.send(:staged_group_key), "status", :running.to_s)
+
+      stage = group.stage(1)
+      Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
+      stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
+
+      stage = group.stage(2)
+      Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
+      stage.redis.hset(stage.send(:staged_group_key), "status", :pending.to_s)
+
+      stage = group.stage(0)
+      Resque::Plugins::Stages::StagedJob.create_job stage, BasicJob
+      stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
+
+      group.stages
+    end
+
+    describe "initiate" do
+      it "initiates the first non-completed stage" do
+        allow(group).to receive(:stages).and_return stages
+
+        stages.each_value { |stage| allow(stage).to receive(:initiate) }
+
+        group.initiate
+
+        expect(stages[2]).to have_received(:initiate)
+        stages.each do |key, stage|
+          next if key == 2
+
+          expect(stage).not_to have_received(:initiate)
+        end
+      end
+    end
+
+    describe "current_stage" do
+      it "returns the first non-complete stage" do
+        stages
+
+        expect(load_group.current_stage).to eq stages[2]
+      end
+
+      it "returns the first non-complete stage" do
+        stage = stages[2]
+        stage.redis.hset(stage.send(:staged_group_key), "status", :complete.to_s)
+
+        expect(load_group.current_stage).to eq stages[8]
+      end
+
+      it "builds a new stage if all current stages are complete" do
+        stages[2].status = :complete
+        stages[8].status = :complete
+
+        stage = load_group.current_stage
+        expect(stage.number).to eq 1
+      end
+    end
+
+    describe "stage_completed" do
+      it "initiates the next stage when a stage is compelted" do
+        allow(group).to receive(:stages).and_return stages
+
+        stages.each_value do |stage|
+          allow(stage).to receive(:initiate)
+          allow(stage).to receive(:staged_group).and_return group
+        end
+
+        stages[8].status = :complete
+
+        expect(stages[2]).to have_received(:initiate)
+      end
+
+      it "deletes the group if the last stage is completed" do
+        allow(group).to receive(:stages).and_return stages
+        allow(group).to receive(:delete)
+
+        stages.each_value do |stage|
+          allow(stage).to receive(:initiate)
+          allow(stage).to receive(:staged_group).and_return group
+        end
+
+        stages[8].status = :complete
+        stages[2].status = :complete
+
+        expect(group).to have_received(:delete)
+      end
+    end
+
+    describe "last_stage" do
+      it "returns the largest stage" do
+        stages
+
+        expect(load_group.last_stage).to eq stages[8]
+      end
+    end
+
+    describe "stages" do
+      it "returns a hash of all stages keyed by the stage number" do
+        stages
+
+        expect(load_group.stages).to eq stages
+      end
+    end
+
+    describe "paged_stages" do
+      it "a page of data" do
+        stages
+
+        expect(load_group.paged_stages(1, 2)).to eq [stages[8], stages[1]]
+      end
+
+      it "a mid page of data" do
+        stages
+
+        expect(load_group.paged_stages(2, 2)).to eq [stages[2], stages[0]]
+      end
+    end
+
+    describe "num_stages" do
+      it "a page of data" do
+        stages
+
+        expect(load_group.num_stages).to eq 4
+      end
+    end
+
+    describe "stage(value)" do
+      it "returns the stage with that number if it exists" do
+        stages
+
+        stage = load_group.stage(2)
+        expect(stage).to eq stages[2]
+        expect(stage.number).to eq 2
+      end
+
+      it "returns a new stage with that number if it does not exist" do
+        stages
+
+        stage = load_group.stage(6)
+        expect(stages).not_to be_include stage
+        expect(stage.number).to eq 6
+
+        expect(load_group.stages.length).to eq stages.length + 1
+      end
+    end
+
+    describe "delete" do
+      it "deletes all stages" do
+        allow(group).to receive(:stages).and_return stages
+
+        stages.each { |_key, stage| allow(stage).to receive(:delete) }
+
+        group.delete
+
+        stages.each_value do |stage|
+          expect(stage).to have_received(:delete)
+        end
+      end
+
+      it "removes a stage when it is deleted" do
+        stages[2].delete
+
+        expect(load_group.stages[2]).to be_nil
+      end
+    end
+  end
+end