resque-approve 0.0.7 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 346c8fcef056427d3dd16e9f6c3075d66ba2a209
-  data.tar.gz: 961923c6b5fa71e78763dc9e3b29791ad737cba8
+SHA256:
+  metadata.gz: a4031a8e2f3c82719ba8c7ebc6d0444cab625394a00f5f92baa83d6998ee62a4
+  data.tar.gz: b0e3ec9f2c9d067b19fc89a8fd08b4988b8c4d592928e513034c0cbe9150ab5e
 SHA512:
-  metadata.gz: 7c0b34d32004e25dee56d9e71d1bb6d681e308847267e2fbca0d8f7c098f75c48d094d03862430bd31a7b67606941741ef9160e83986752301793135951cd9d2
-  data.tar.gz: e18cfc1e00817eaaf2b45adaed87640022818076e1fbef703cbe0efc308b5efc0755d2174f382185871d2ee47567c6272aad5ee8f8113a84273451e4ff0e307a
+  metadata.gz: aa99e9a452542df8269c799f8ad7a54d14cf0dfd2c80c12cb9d6ec91a2cbbdeb1afc81110e2e803123fbba182aff598faada2a2b43097fdc6b6416132a2fb629
+  data.tar.gz: 270799be2ba9decfeb6d110259a261c533cbe23f6ca5b586fa3f3328d379b356de11091928b083a3dddf2f4bde537671910db76ba3710aae9e454e352cf02308

data/lib/resque-approve.rb

@@ -6,5 +6,8 @@ require File.expand_path(File.join("resque", "plugins", "approve", "pending_job"
 require File.expand_path(File.join("resque", "plugins", "approve", "pending_job_queue"), File.dirname(__FILE__))
 require File.expand_path(File.join("resque", "plugins", "approve", "approval_key_list"), File.dirname(__FILE__))
 require File.expand_path(File.join("resque", "plugins", "approve", "cleaner"), File.dirname(__FILE__))
+require File.expand_path(File.join("resque", "plugins", "approve", "auto_approve_next"), File.dirname(__FILE__))
+require File.expand_path(File.join("resque", "plugins", "approve", "compressable_auto_approve_next"), File.dirname(__FILE__))
+require File.expand_path(File.join("resque", "plugins", "version"), File.dirname(__FILE__))
 
 require File.expand_path(File.join("resque", "plugins", "approve"), File.dirname(__FILE__))

data/lib/resque/approve_server.rb

@@ -107,6 +107,7 @@ module Resque
         delete_all_queues(base)
         approve_all_queues(base)
         delete_queue(base)
+        reset_running(base)
         delete_one_queue(base)
         approve_queue(base)
         approve_one_queue(base)
@@ -167,6 +168,16 @@ module Resque
         end
       end
 
+      def reset_running(base)
+        base.class_eval do
+          post "/approve/reset_running" do
+            Resque::Plugins::Approve::PendingJobQueue.new(params[:approval_key]).reset_running
+
+            redirect u("approve/job_list?#{{ approval_key: params[:approval_key] }.to_param}")
+          end
+        end
+      end
+
       def delete_one_queue(base)
         base.class_eval do
           post "/approve/delete_one_queue" do

data/lib/resque/plugins/approve.rb

@@ -11,6 +11,8 @@ module Resque
 
       included do
         self.auto_delete_approval_key = false
+        self.max_active_jobs          = -1
+        self.default_queue_name       = nil
       end
 
       class << self
@@ -46,20 +48,60 @@ module Resque
           @auto_delete_approval_key
         end
 
+        def max_active_jobs=(value)
+          @max_active_jobs = value
+        end
+
+        def max_active_jobs
+          @max_active_jobs
+        end
+
+        def default_queue_name=(value)
+          @default_queue_name = value
+        end
+
+        def default_queue_name
+          @default_queue_name || Resque.queue_from_class(self)
+        end
+
+        def approve
+          Resque::Plugins::Approve.approve(default_queue_name)
+        end
+
+        def approve_one
+          Resque::Plugins::Approve.approve_one(default_queue_name)
+        end
+
+        def approve_num(num_approve)
+          Resque::Plugins::Approve.approve_num(num_approve, default_queue_name)
+        end
+
+        def remove
+          Resque::Plugins::Approve.remove(default_queue_name)
+        end
+
+        def remove_one
+          Resque::Plugins::Approve.remove_one(default_queue_name)
+        end
+
         # It is possible to run a job immediately using `Resque.push`.  This will bypass the queue and run
         # the job immediately.  This will prevent such a job from enqueuing, and instead pause it for approval
         #
         # The primary reason for this is to prevent the job from receiving the approval parameters it is not
         # supposed to have when actually run/enqueued.
+        #
+        # NOTE:  This does not validate running counts.
         def before_perform_approve(*args)
           # Check if the job needs to be approved, and if so, do not enqueue it.
           job = PendingJob.new(SecureRandom.uuid, class_name: name, args: args)
 
-          if job.requires_approval?
+          if job.approval_keys? && !job.max_active_jobs?
             ApprovalKeyList.new.add_job(job)
 
             raise Resque::Job::DontPerform, "The job has not been approved yet."
           else
+            job.max_jobs_perform_args(args)
+
             true
           end
         end
@@ -69,9 +111,14 @@ module Resque
           job = PendingJob.new(SecureRandom.uuid, class_name: name, args: args)
 
           if job.requires_approval?
+            return false if job.compressable? && !job.compressed?(args)
+
             ApprovalKeyList.new.add_job(job)
+
             false
           else
+            job.max_jobs_perform_args(args)
+
             true
           end
         end

data/lib/resque/plugins/approve/auto_approve_next.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Approve
+      # This module adds a new class level perform method that overrides the jobs classes
+      # class method perform that is used by Resque.
+      #
+      # The new overloaded method uses super to perform the original perform functionality
+      # but ensures that when the job is complete that it approves the next job in the queue.
+      #
+      # The reason for this class is to manage the maximum number of jobs that are allowed to
+      # run at the same time for a particular job.  When the maximum number of jobs is reached
+      # when a job completes, it will approve the next job in the queue automatically.
+      module AutoApproveNext
+        extend ActiveSupport::Concern
+
+        class_methods do
+          def perform(*args)
+            job          = Resque::Plugins::Approve::PendingJob.new(SecureRandom.uuid, class_name: name, args: args)
+            dup_args     = job.args
+            del_options  = dup_args.extract_options!.with_indifferent_access
+            approval_key = del_options.delete(:approval_key) || job.approve_options[:approval_key]
+
+            dup_args << del_options if del_options.present?
+
+            begin
+              super(*dup_args)
+            ensure
+              Resque::Plugins::Approve::PendingJobQueue.new(approval_key).decrement_running
+              Resque::Plugins::Approve.approve_one approval_key
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/approve/compressable_auto_approve_next.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Approve
+      # This module adds a new class level perform method that overrides the jobs classes
+      # class method perform that is used by Resque.
+      #
+      # The new overloaded method uses super to perform the original perform functionality
+      # but ensures that when the job is complete that it approves the next job in the queue.
+      #
+      # The reason for this class is to manage the maximum number of jobs that are allowed to
+      # run at the same time for a particular job.  When the maximum number of jobs is reached
+      # when a job completes, it will approve the next job in the queue automatically.
+      module CompressableAutoApproveNext
+        def perform_with_auto_approve(*args)
+          job          = Resque::Plugins::Approve::PendingJob.new(SecureRandom.uuid, class_name: name, args: args)
+          dup_args     = job.uncompressed_args
+          del_options  = dup_args.extract_options!.with_indifferent_access
+          approval_key = del_options.delete(:approval_key) || job.approve_options[:approval_key]
+
+          dup_args << del_options if del_options.present?
+
+          begin
+            perform_without_auto_approve(*dup_args)
+          ensure
+            Resque::Plugins::Approve::PendingJobQueue.new(approval_key).decrement_running
+            Resque::Plugins::Approve.approve_one approval_key
+          end
+        end
+
+        def self.extended(base)
+          class << base
+            alias_method :perform_without_auto_approve, :perform
+            alias_method :perform, :perform_with_auto_approve
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/approve/pending_job.rb

@@ -12,31 +12,35 @@ module Resque
       #   class_name  - The name of the job class to be enqueued
       #   args        - The arguments for the job to be enqueued
       #
-      #   approval_key    - The approval key for the pending job that will be used to release/approve the job.
-      #                     This will default to nil.
-      #   approval_queue  - The queue that the pending job will be enqueued into.
-      #                     This will default to the queue for the job.
-      #   approval_at     - The time when the job is to be enqueued.
-      #                     This will call `enqueue_at` to enqueue the job, so this option will only work
-      #                     properly if you use the `resque-scheduler` gem.
+      #   requires_approval - If a class defines a default approval queue (similar to defining
+      #                       the queue for Resque), then instead of passing `approval_key: "key name"` in the
+      #                       parameters, you would pass `requires_approval: true`.  This way the key name
+      #                       appears only in the class and not scattered through the code.
+      #   approval_key      - The approval key for the pending job that will be used to release/approve the job.
+      #                       This will default to nil.
+      #   approval_queue    - The queue that the pending job will be enqueued into.
+      #                       This will default to the queue for the job.
+      #   approval_at       - The time when the job is to be enqueued.
+      #                       This will call `enqueue_at` to enqueue the job, so this option will only work
+      #                       properly if you use the `resque-scheduler` gem.
       #
-      #                     When using resque-scheduler, there are two ways to delay enqueue a job
-      #                     and how you do this depends on your use case.
+      #                       When using resque-scheduler, there are two ways to delay enqueue a job
+      #                       and how you do this depends on your use case.
       #
-      #                     Resque.enqueue YourJob, params, approval_key: "your key", approval_at: time
-      #                     This will enqueue the job for approval, which will pause the job until it is approved.
-      #                     Once approved, the job will delay enqueue for time, and will execute immediately or at
-      #                     that time depending on if the time has passed.
+      #                       Resque.enqueue YourJob, params, approval_key: "your key", approval_at: time
+      #                       This will enqueue the job for approval, which will pause the job until it is approved.
+      #                       Once approved, the job will delay enqueue for time, and will execute immediately or at
+      #                       that time depending on if the time has passed.
       #
-      #                     This is the recommended method to use as it will not run the job early, and it will allow
-      #                     you to release it without knowing if it is still delayed or not.
+      #                       This is the recommended method to use as it will not run the job early, and it will allow
+      #                       you to release it without knowing if it is still delayed or not.
       #
-      #                     You can also do:
-      #                     Resque.enqueue_at time, YourJob, params, approval_key: "your key"
-      #                     This will delay enqueue the job - because it has not been enqueued yet, the job
-      #                     cannot be releaed until the time has passed and the job is actually enqueued.
-      #                     Any time after that point, it can be released.  Releasing the key before this
-      #                     time has no effect on this job.
+      #                       You can also do:
+      #                       Resque.enqueue_at time, YourJob, params, approval_key: "your key"
+      #                       This will delay enqueue the job - because it has not been enqueued yet, the job
+      #                       cannot be releaed until the time has passed and the job is actually enqueued.
+      #                       Any time after that point, it can be released.  Releasing the key before this
+      #                       time has no effect on this job.
       class PendingJob
         include Resque::Plugins::Approve::RedisAccess
         include Comparable
@@ -60,6 +64,10 @@ module Resque
           @class_name ||= stored_values[:class_name]
         end
 
+        def uncompressed_args
+          decompress_args(args)
+        end
+
         def args
           @args = if @args.present?
                     @args
@@ -86,8 +94,12 @@ module Resque
                              end
         end
 
+        def approval_keys?
+          @approval_keys ||= (approve_options.key?(:approval_key) || approve_options[:requires_approval])
+        end
+
         def requires_approval?
-          @requires_approval ||= approve_options.key?(:approval_key) || approve_options[:requires_approval]
+          @requires_approval ||= approval_keys? && too_many_running?
         end
 
         def approval_key
@@ -98,6 +110,21 @@ module Resque
           @approval_queue ||= approve_options[:approval_queue] || Resque.queue_from_class(klass)
         end
 
+        def max_active_jobs?
+          return @max_active_jobs if defined?(@max_active_jobs)
+
+          @max_active_jobs = max_active_jobs.positive?
+        end
+
+        def max_active_jobs
+          max_val = ((klass.respond_to?(:max_active_jobs) && klass.max_active_jobs) ||
+              (klass.instance_variable_defined?(:@max_active_jobs) && klass.instance_variable_get(:@max_active_jobs)))
+
+          return -1 if max_val.blank?
+
+          max_val.to_i
+        end
+
         def approval_at
           @approval_at ||= approve_options[:approval_at]&.to_time
         end
@@ -108,9 +135,9 @@ module Resque
 
         def enqueue_job
           return_value = if approval_at.present?
-                           Resque.enqueue_at_with_queue approval_queue, approval_at, klass, *args
+                           Resque.enqueue_at_with_queue approval_queue, approval_at, klass, *compressed_args(args)
                          else
-                           Resque.enqueue_to approval_queue, klass, *args
+                           Resque.enqueue_to approval_queue, klass, *compressed_args(args)
                          end
 
           delete
@@ -132,10 +159,10 @@ module Resque
           # Make sure the job is loaded into memory so we can use it even though we are going to delete it.
           stored_values
 
-          return if class_name.blank?
-
           redis.del(job_key)
 
+          return if class_name.blank?
+
           queue.remove_job(self)
         end
 
@@ -143,8 +170,43 @@ module Resque
           @queue ||= Resque::Plugins::Approve::PendingJobQueue.new(approval_key)
         end
 
+        def max_jobs_perform_args(args)
+          return compressed_max_jobs_perform_args(args) if compressed?(args)
+
+          return unless args.last.is_a?(Hash)
+
+          remove_options = args[-1]
+
+          %i[approval_queue approval_at requires_approval].each do |key|
+            remove_options.delete(key)
+            remove_options.delete(key.to_s)
+          end
+        end
+
+        def compressed?(basic_args)
+          return false unless compressable?
+
+          described_class.compressed?(basic_args)
+        end
+
+        def compressable?
+          !described_class.blank? &&
+              described_class.singleton_class.included_modules.map(&:name).include?("Resque::Plugins::Compressible")
+        end
+
         private
 
+        def compressed_max_jobs_perform_args(args)
+          decompressed = decompress_args(args)
+
+          max_jobs_perform_args(decompressed)
+          decompressed = compressed_args(decompressed)[0]
+
+          compress_arg            = args[0]
+          compress_arg[:payload]  = decompressed[:payload] if compress_arg.key?(:payload)
+          compress_arg["payload"] = decompressed[:payload] if compress_arg.key?("payload")
+        end
+
         def klass
           @klass ||= class_name.constantize
         end
@@ -158,15 +220,30 @@ module Resque
         end
 
         def extract_approve_options
-          return if args.blank? || !args[-1].is_a?(Hash)
+          extract_args = uncompressed_args
+
+          return if extract_args.blank? || !extract_args[-1].is_a?(Hash)
 
-          self.approve_options = args.pop
+          self.approve_options = extract_args.pop.with_indifferent_access
 
-          options = approve_options.slice!(:approval_key, :approval_queue, :approval_at)
+          options = slice_approval_options
 
-          args << options.to_hash if options.present?
+          extract_args << options.to_hash if options.present?
+          @args = compressed_args(extract_args)
         end
 
+        # rubocop:disable Layout/SpaceAroundOperators
+        def slice_approval_options
+          options                        = approve_options.slice!(:approval_key, :approval_queue, :approval_at, :requires_approval)
+          approve_options[:approval_key] ||= klass.default_queue_name if approve_options[:requires_approval]
+
+          options_approval_key(options)
+
+          options
+        end
+
+        # rubocop:enable Layout/SpaceAroundOperators
+
         def encode_args(*args)
           Resque.encode(args)
         end
@@ -180,7 +257,57 @@ module Resque
         def approve_options=(value)
           @approve_options = (value&.dup || {}).with_indifferent_access
         end
+
+        def too_many_running?
+          return true if queue.paused?
+          return true unless max_active_jobs?
+          return @too_many_running if defined?(@too_many_running)
+
+          num_running = queue.increment_running
+          max_jobs    = max_active_jobs
+
+          queue.decrement_running if num_running > max_jobs
+          @too_many_running = num_running > max_jobs
+        end
+
+        def options_approval_key(options)
+          return unless max_active_jobs?
+          return if options.key?(:approval_key)
+
+          if compressable?
+            unless klass.singleton_class.included_modules.include?(Resque::Plugins::Approve::CompressableAutoApproveNext)
+              klass.extend Resque::Plugins::Approve::CompressableAutoApproveNext
+            end
+          else
+            klass.include Resque::Plugins::Approve::AutoApproveNext unless klass.included_modules.include?(Resque::Plugins::Approve::AutoApproveNext)
+          end
+
+          options[:approval_key] = approve_options.fetch(:approval_key) { klass.default_queue_name }
+        end
+
+        def described_class
+          return if class_name.blank?
+
+          class_name.constantize
+        rescue StandardError
+          nil
+        end
+
+        def compressed_args(compress_args)
+          return compress_args unless compressable?
+          return compress_args if described_class.compressed?(compress_args)
+
+          [{ :resque_compressed => true, :payload => described_class.compressed_args(compress_args) }]
+        end
+
+        def decompress_args(basic_args)
+          return basic_args unless compressable?
+          return basic_args unless described_class.compressed?(basic_args)
+
+          described_class.uncompressed_args(basic_args.first[:payload] || basic_args.first["payload"])
+        end
       end
+
       # rubocop:enable Metrics/ClassLength
     end
   end