resque-stages 0.0.1

data/lib/resque/plugins/stages/staged_group.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Stages
+      # This class represents the toplevel grouping of a set of staged jobs.
+      #
+      # The group defines individual numbered stages (starting with 0) and
+      # intiates subsequent stages as the current stage completes.
+      #
+      # There are methods on the group to initiate groups and add jobs to
+      # individual stages or get/create new stages
+      class StagedGroup
+        include Resque::Plugins::Stages::RedisAccess
+        include Comparable
+
+        attr_reader :group_id
+
+        def initialize(group_id, description: "")
+          @group_id = group_id
+
+          Resque::Plugins::Stages::StagedGroupList.new.add_group(self)
+
+          redis.hsetnx(group_info_key, "created_at", Time.now)
+          self.description = description if description.present?
+        end
+
+        def <=>(other)
+          return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroup)
+
+          group_id <=> other.group_id
+        end
+
+        class << self
+          def within_a_grouping(description = nil)
+            group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid, description: description)
+
+            yield group
+
+            group.initiate
+          end
+        end
+
+        def initiate
+          stage = next_stage
+
+          if stage
+            stage.initiate
+          else
+            delete
+          end
+        end
+
+        def description
+          @description ||= redis.hget(group_info_key, "description").presence || group_id
+        end
+
+        def created_at
+          @created_at ||= redis.hget(group_info_key, "created_at").presence.to_time || Time.now
+        end
+
+        def description=(value)
+          @description = value.presence
+          redis.hset(group_info_key, "description", description)
+        end
+
+        def current_stage
+          next_stage || new_stage
+        end
+
+        def last_stage
+          group_stages = stages
+          last_key     = group_stages.keys.max
+
+          group_stages[last_key]
+        end
+
+        def add_stage(staged_group_stage)
+          redis.rpush group_key, staged_group_stage.group_stage_id
+        end
+
+        def remove_stage(staged_group_stage)
+          redis.lrem(group_key, 0, staged_group_stage.group_stage_id)
+        end
+
+        def num_stages
+          redis.llen group_key
+        end
+
+        def stages
+          all_stages = redis.lrange(group_key, 0, -1).map { |stage_id| Resque::Plugins::Stages::StagedGroupStage.new(stage_id) }
+
+          all_stages.each_with_object({}) do |stage, hash|
+            num = stage.number
+            num += 1 while hash.key?(num)
+
+            hash[num]    = stage
+            stage.number = num if stage.number != num
+          end
+        end
+
+        def paged_stages(page_num = 1, page_size = nil)
+          page_size ||= 20
+          page_size = page_size.to_i
+          page_size = 20 if page_size < 1
+          start     = (page_num - 1) * page_size
+          start     = 0 if start >= num_stages || start.negative?
+
+          stages.values[start..start + page_size - 1]
+        end
+
+        def stage(stage_number)
+          found_stage = stages[stage_number]
+
+          found_stage || create_stage(stage_number)
+        end
+
+        def delete
+          stages.each_value(&:delete)
+
+          Resque::Plugins::Stages::StagedGroupList.new.remove_group(self)
+
+          redis.del group_key
+          redis.del group_info_key
+        end
+
+        def stage_completed
+          initiate
+        end
+
+        def blank?
+          !redis.exists(group_key) && !redis.exists(group_info_key)
+        end
+
+        def verify_stage(stage)
+          ids = redis.lrange(group_key, 0, -1)
+
+          return if ids.include?(stage.group_stage_id)
+
+          redis.lpush(group_key, stage.group_stage_id)
+        end
+
+        private
+
+        def next_stage
+          group_stages = stages
+          keys         = group_stages.keys.sort
+
+          current_number = keys.detect do |key|
+            group_stages[key].status != :complete
+          end
+
+          group_stages[current_number]
+        end
+
+        def group_key
+          "StagedGroup::#{group_id}"
+        end
+
+        def group_info_key
+          "#{group_key}::Info"
+        end
+
+        def new_stage
+          next_stage_number = (last_stage&.number || 0) + 1
+
+          create_stage(next_stage_number)
+        end
+
+        def create_stage(stage_number)
+          stage = Resque::Plugins::Stages::StagedGroupStage.new(SecureRandom.uuid)
+
+          stage.staged_group = self
+          stage.number       = stage_number
+
+          stage
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/stages/staged_group_list.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Stages
+      # A utility class that keeps track of all Groups and lists them.
+      class StagedGroupList
+        include Resque::Plugins::Stages::RedisAccess
+
+        def groups
+          redis.smembers(list_key).map { |group_id| Resque::Plugins::Stages::StagedGroup.new(group_id) }
+        end
+
+        def order_param(sort_option, current_sort, current_order)
+          current_order ||= "asc"
+
+          if sort_option == current_sort
+            current_order == "asc" ? "desc" : "asc"
+          else
+            "asc"
+          end
+        end
+
+        def paginated_groups(sort_key = :description,
+                             sort_order = "asc",
+                             page_num = 1,
+                             queue_page_size = 20)
+          queue_page_size = queue_page_size.to_i
+          queue_page_size = 20 if queue_page_size < 1
+
+          group_list = sorted_groups(sort_key)
+
+          page_start = (page_num - 1) * queue_page_size
+          page_start = 0 if page_start > group_list.length || page_start.negative?
+
+          (sort_order == "desc" ? group_list.reverse : group_list)[page_start..(page_start + queue_page_size - 1)]
+        end
+
+        def num_groups
+          groups.length
+        end
+
+        def add_group(group)
+          redis.sadd list_key, group.group_id
+        end
+
+        def remove_group(group)
+          redis.srem list_key, group.group_id
+        end
+
+        def delete_all
+          groups.each(&:delete)
+        end
+
+        private
+
+        def sorted_groups(sort_key)
+          groups.sort_by do |group|
+            group_sort_value(group, sort_key)
+          end
+        end
+
+        def group_sort_value(group, sort_key)
+          case sort_key.to_sym
+            when :description,
+                :num_stages
+              group.public_send(sort_key)
+            when :created_at
+              group.public_send(sort_key).to_s
+          end
+        end
+
+        def list_key
+          "StagedGroupList"
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/stages/staged_group_stage.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Stages
+      # A class which represents a single stage and all of the jobs for that stage.
+      #
+      # Each stage has a status that it progresses through
+      #   :pending  - not started yet
+      #   :running  - currently running
+      #   :complete - all jobs in the stage are completed
+
+      # rubocop:disable Metrics/ClassLength
+      class StagedGroupStage
+        include Resque::Plugins::Stages::RedisAccess
+        include Comparable
+
+        attr_reader :group_stage_id
+
+        def initialize(group_stage_id)
+          @group_stage_id = group_stage_id
+        end
+
+        def <=>(other)
+          return nil unless other.is_a?(Resque::Plugins::Stages::StagedGroupStage)
+
+          group_stage_id <=> other.group_stage_id
+        end
+
+        def enqueue(klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue(*job.enqueue_args)
+
+          job
+        end
+
+        def enqueue_to(queue, klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue_to(queue, *job.enqueue_args)
+
+          job
+        end
+
+        def enqueue_at(timestamp, klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue_at(timestamp, *job.enqueue_args)
+
+          job
+        end
+
+        def enqueue_at_with_queue(queue, timestamp, klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue_at_with_queue(queue, timestamp, *job.enqueue_args)
+
+          job
+        end
+
+        def enqueue_in(number_of_seconds_from_now, klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue_in(number_of_seconds_from_now, *job.enqueue_args)
+
+          job
+        end
+
+        def enqueue_in_with_queue(queue, number_of_seconds_from_now, klass, *args)
+          job = create_enqueue_job(klass, args)
+
+          return job if status == :pending
+
+          self.status = :running if status != :running
+
+          job.status = :queued
+          Resque.enqueue_in_with_queue(queue, number_of_seconds_from_now, *job.enqueue_args)
+
+          job
+        end
+
+        def status
+          redis.hget(staged_group_key, "status")&.to_sym || :pending
+        end
+
+        def status=(value)
+          redis.hset(staged_group_key, "status", value.to_s)
+
+          staged_group&.stage_completed if status == :complete
+        end
+
+        def number
+          redis.hget(staged_group_key, "number")&.to_i || 1
+        end
+
+        def number=(value)
+          redis.hset(staged_group_key, "number", value)
+        end
+
+        def staged_group
+          return nil if staged_group_id.blank?
+
+          @staged_group ||= Resque::Plugins::Stages::StagedGroup.new(staged_group_id)
+        end
+
+        def staged_group=(value)
+          @staged_group    = value
+          @staged_group_id = value.group_id
+
+          value.add_stage(self)
+          redis.hset(staged_group_key, "staged_group_id", value.group_id)
+        end
+
+        def jobs(start = 0, stop = -1)
+          redis.lrange(stage_key, start, stop).map { |id| Resque::Plugins::Stages::StagedJob.new(id) }
+        end
+
+        def jobs_by_status(status)
+          jobs.select { |job| job.status == status }
+        end
+
+        def paginated_jobs(sort_key = :class_name,
+                           sort_order = "asc",
+                           page_num = 1,
+                           queue_page_size = 20)
+          queue_page_size = queue_page_size.to_i
+          queue_page_size = 20 if queue_page_size < 1
+
+          job_list = sorted_jobs(sort_key)
+
+          page_start = (page_num - 1) * queue_page_size
+          page_start = 0 if page_start > job_list.length || page_start.negative?
+
+          (sort_order == "desc" ? job_list.reverse : job_list)[page_start..(page_start + queue_page_size - 1)]
+        end
+
+        def order_param(sort_option, current_sort, current_order)
+          current_order ||= "asc"
+
+          if sort_option == current_sort
+            current_order == "asc" ? "desc" : "asc"
+          else
+            "asc"
+          end
+        end
+
+        def num_jobs
+          redis.llen(stage_key)
+        end
+
+        def add_job(staged_group_job)
+          redis.rpush stage_key, staged_group_job.job_id
+        end
+
+        def remove_job(staged_group_job)
+          redis.lrem(stage_key, 0, staged_group_job.job_id)
+        end
+
+        def delete
+          jobs.each(&:delete)
+
+          staged_group&.remove_stage self
+
+          redis.del stage_key
+          redis.del staged_group_key
+        end
+
+        def initiate
+          self.status = :running
+
+          jobs.each do |job|
+            next if job.completed?
+            next if job.queued?
+
+            job.enqueue_job
+          end
+
+          job_completed
+        end
+
+        def job_completed
+          self.status = :complete if jobs.all?(&:completed?)
+        end
+
+        def blank?
+          !redis.exists(stage_key) && !redis.exists(staged_group_key)
+        end
+
+        def verify
+          return build_new_structure if staged_group.blank?
+
+          staged_group.verify_stage(self)
+        end
+
+        def verify_job(job)
+          ids = redis.lrange(stage_key, 0, -1)
+
+          return if ids.include?(job.job_id)
+
+          redis.lpush(stage_key, job.job_id)
+        end
+
+        private
+
+        def build_new_structure
+          group = Resque::Plugins::Stages::StagedGroup.new(SecureRandom.uuid)
+
+          self.staged_group = group
+        end
+
+        def create_enqueue_job(klass, args)
+          Resque::Plugins::Stages::StagedJob.create_job self, klass, *args
+        end
+
+        def stage_key
+          "StagedGroupStage::#{group_stage_id}"
+        end
+
+        def staged_group_key
+          "#{stage_key}::staged_group"
+        end
+
+        def staged_group_id
+          @staged_group_id ||= redis.hget(staged_group_key, "staged_group_id")
+        end
+
+        def sorted_jobs(sort_key)
+          jobs.sort_by do |job|
+            job_sort_value(job, sort_key)
+          end
+        end
+
+        def job_sort_value(job, sort_key)
+          case sort_key.to_sym
+            when :class_name,
+                :status,
+                :status_message
+              job.public_send(sort_key)
+
+            when :queue_time
+              job.public_send(sort_key).to_s
+          end
+        end
+      end
+
+      # rubocop:enable Metrics/ClassLength
+    end
+  end
+end