resque-unique_by_arity 0.2.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5082a9bd4cfded2e55fe7436b007c8f38d5e2604
-  data.tar.gz: 30758aea39db2962f319e0274bdda29fbf4e7ba7
+  metadata.gz: 80b0037698ecc3c3fb68119077dfadb27c266201
+  data.tar.gz: 810f868a334ae08dc806f335f434d5d52fc09639
 SHA512:
-  metadata.gz: ee2e12f6728c2801048e46d43a0526a5c9292c59ac37906ddff94d1ff75527270709c73f589aad6f97cd0bc177578792c379a993a55f0274136b499957888d08
-  data.tar.gz: 82fa706dca697a8b9634b418a3b1b55a89f20a735d010fea737f2af7fcfc9f459ebb4de6d6215f673bb90b7a6373523f72d2cd41228b06253ed3c07bb5eb0426
+  metadata.gz: ce9d5ff1591847f915a3ee57f2418cb337cb9933f7a7bf8e4f28cf99b5ed52d6a58c6b5e11f410753b8df0797787bb9a3b3ec1dcb9f8e05f19b48bd2ec11f5c4
+  data.tar.gz: f4fd53779a94981d4d8335c322bc6adead29a61394e12e44d167a06258b05e79e2350e68bbff528793c43241e07c4a100b8517934cd9cae21ba0c641577c4527

data/README.md

@@ -2,14 +2,16 @@
 
 NOTE:
 
-Requires the pboling fork of resque-lonely_job at https://github.com/pboling/resque-lonely_job as the standard one is just very incompatible with resque-solo (or vice versa, either direction is true; they step on each other).
+Requires `resque_solo` gem, and `resque-unique_at_runtime` gem; the latter is a fork of `resque-lonely_job`.
+Why? `resque-lonely_job` and `resque_solo` can't be used together, because their `redis_key` methods conflict.
 
-Usage:
+Example:
 
 ```ruby
 class MyJob
   include UniqueByArity::Cop.new(
     arity_for_uniqueness: 1,
+    lock_after_execution_period: 60,
     unique_at_runtime: true,
     unique_in_queue: true
   )
@@ -34,6 +36,84 @@ Or install it yourself as:
 
 ## Usage
 
+### Arity For Uniqueness
+
+Some jobs have parameters that you do not want to consider for determination of uniqueness.  Resque jobs should use simple parameters, **not named parameters**, so you can just specify the number of parameters, counting from the left, you want to be considered for uniqueness.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 3,
+    unique_at_runtime: true
+  )
+  def self.perform(my, cat, is, the, best, opts = {})
+    # Only the first 3: [my, cat, is] will be considered for determination of uniqueness
+  end
+end
+```
+
+### Lock After Execution
+
+Give the job a break after it finishes running, and don't allow another of the same, with matching args @ configured arity, to start within X seconds.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 1,
+    lock_after_execution_period: 60,
+    unique_at_runtime: true
+  )
+end
+```
+
+### Unique At Runtime (across all queues)
+
+Prevent your app from running a job that is already running.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 1,
+    unique_at_runtime: true
+  )
+end
+```
+
+### Unique At Queue Time
+
+#### Unique In Job's Specific Queue
+
+Prevent your app from queueing a job that is already queued in the same queue.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 1,
+    unique_in_queue: true
+  )
+end
+```
+
+#### Unique Across All Queues
+
+Prevent your app from queueing a job that is already queued in *any* queue.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 1,
+    unique_across_queues: true
+  )
+end
+```
+
+### All Together Now
+
+#### Unique At Runtime (across all queues) AND Unique In Job's Specific Queue
+
+Prevent your app from running a job that is already running, **and**
+prevent your app from queueing a job that is already queued in the same queue.
+
 ```ruby
 class MyJob
   include UniqueByArity::Cop.new(
@@ -44,6 +124,73 @@ class MyJob
 end
 ```
 
+#### Unique At Runtime (across all queues) AND Unique Across All Queues
+
+Prevent your app from running a job that is already running, **and**
+prevent your app from queueing a job that is already queued in *any* queue.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    arity_for_uniqueness: 1,
+    unique_at_runtime: true,
+    unique_across_queues: true
+  )
+end
+```
+
+### Debugging
+
+Run your worker with `RESQUE_DEBUG=true` to see payloads printed before they are used to determine uniqueness.
+
+### Customize Unique Keys Per Job
+
+Redefine methods to customize all the things.  Warning: This might be crazy-making.
+
+```ruby
+class MyJob
+  include UniqueByArity::Cop.new(
+    #...
+  )
+
+  # Core hashing algorithm for a job used for *all 3 types* of uniqueness 
+  # @return [Array<String, arguments>], where the string is the unique digest, and arguments are the specific args that were used to calculate the digest 
+  def self.redis_unique_hash(payload)
+    # ... See source @ lib/resque/unique_by_arity/cop_modulizer.rb 
+    #       for how the built-in version works
+    # uniqueness_args = payload["args"] # over simplified - this ignores the arity
+    # args = { class: job, args: uniqueness_args }
+    # return [Digest::MD5.hexdigest(Resque.encode(args)), uniqueness_args]
+  end
+
+  # Prefix to the unique key for a job for resque_solo, queue time uniqueness 
+  def self.solo_redis_key_prefix
+    # "unique_job:#{self}" # <= default value
+  end
+
+  # Prefix to the unique redis key for a job for resque_solo, queue time uniqueness 
+  def self.solo_key_namespace(queue = nil)
+    # definition depends on which type of uniqueness is chosen, be careful if you customize
+    # "solo:queue:#{queue}:job" # <= is for unique within queue at queue time
+    # "solo:across_queues:job" # <= is for unique across all queues at queue time
+  end
+  
+  def self.unique_at_queue_time_redis_key(queue, payload)
+    # unique_hash, _args_for_uniqueness = redis_unique_hash(payload)
+    # "#{solo_key_namespace(queue)}:#{solo_redis_key_prefix}:#{unique_hash}"
+  end
+  
+  def self.runtime_key_namespace
+    # "unique_at_runtime:#{self}"
+  end
+  
+  def self.unique_at_runtime_redis_key(*args)
+    # unique_hash, _args_for_uniqueness = redis_unique_hash({"class" => self.to_s, "args" => args})
+    # key = "#{runtime_key_namespace}:#{unique_hash}" # <= simplified default
+  end
+end
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/resque/resque_solo/queue.rb

@@ -0,0 +1,9 @@
+module ResqueSoloUniqueHack
+  def unique_key(queue, item)
+    # "solo:queue:#{queue}:job:#{const_for(item).redis_key(item)}"
+    const_for(item).unique_at_queue_time_redis_key(queue, item)
+    # NOTE: DOES NOT CALL SUPER
+  end
+end
+
+ResqueSolo::Queue.singleton_class.prepend ResqueSoloUniqueHack

data/lib/resque/unique_by_arity/configuration.rb

@@ -8,12 +8,7 @@ module Resque
       attr_accessor :lock_after_execution_period
       attr_accessor :unique_at_runtime
       attr_accessor :unique_in_queue
-      alias :debug_logger :logger
-      alias :debug_log_level :log_level
-      alias :debug_arity_for_uniqueness :arity_for_uniqueness
-      alias :debug_lock_after_execution_period :lock_after_execution_period
-      alias :debug_unique_at_runtime :unique_at_runtime
-      alias :debug_unique_in_queue :unique_in_queue
+      attr_accessor :unique_across_queues
       def initialize(**options)
         @logger = options.key?(:logger) ? options[:logger] : Logger.new(STDOUT)
         @log_level = options.key?(:log_level) ? options[:log_level] : :debug
@@ -21,6 +16,10 @@ module Resque
         @lock_after_execution_period = options.key?(:lock_after_execution_period) ? options[:lock_after_execution_period] : nil
         @unique_at_runtime = options.key?(:unique_at_runtime) ? options[:unique_at_runtime] : false
         @unique_in_queue = options.key?(:unique_in_queue) ? options[:unique_in_queue] : false
+        @unique_across_queues = options.key?(:unique_across_queues) ? options[:unique_across_queues] : false
+        warn ":arity_for_uniqueness is set to #{@arity_for_uniqueness}, but no uniqueness enforcement was turned on [:unique_at_runtime, :unique_in_queue, :unique_across_queues]" unless @unique_at_runtime || @unique_in_queue || @unique_across_queues
+        warn ":lock_after_execution_period is set to #{@lock_after_execution_period}, but :unique_at_runtime is not set" if @lock_after_execution_period && !@unique_at_runtime
+        warn ":unique_in_queue and :unique_across_queues should not be set at the same time, as :unique_across_queues will always supercede :unique_in_queue" if @unique_in_queue && @unique_across_queues
       end
       def to_hash
         {
@@ -29,7 +28,8 @@ module Resque
             arity_for_uniqueness: arity_for_uniqueness,
             lock_after_execution_period: lock_after_execution_period,
             unique_at_runtime: unique_at_runtime,
-            unique_in_queue: unique_in_queue
+            unique_in_queue: unique_in_queue,
+            unique_across_queues: unique_across_queues
         }
       end
     end

data/lib/resque/unique_by_arity/cop.rb

@@ -11,12 +11,12 @@ module Resque
 
         # gem is resque_solo, which is a rewrite of resque-loner
         # see: https://github.com/neighborland/resque_solo
-        # defines a redis_key method, which if we are not careful, conflicts with a custom redis_key we set here
-        base.send(:include, Resque::Plugins::UniqueJob) if @configuration.unique_in_queue
+        # defines a redis_key method, which we have to override.
+        base.send(:include, Resque::Plugins::UniqueJob) if @configuration.unique_in_queue || @configuration.unique_across_queues
 
-        # gem is resque-lonely_job
-        # see: https://github.com/wallace/resque-lonely_job
-        base.send(:extend, Resque::Plugins::LonelyJob) if @configuration.unique_at_runtime
+        # gem is resque-unique_at_runtime
+        # see: https://github.com/pboling/resque-unique_at_runtime
+        base.send(:extend, Resque::Plugins::UniqueAtRuntime) if @configuration.unique_at_runtime
 
         uniqueness_cop_module = Resque::UniqueByArity::CopModulizer.to_mod(@configuration)
         # This will override methods from both plugins above, if configured for both

data/lib/resque/unique_by_arity/cop_modulizer.rb

@@ -3,10 +3,11 @@ module Resque
     module CopModulizer
       def self.to_mod(configuration)
         Module.new do
-          if configuration.unique_in_queue || configuration.unique_at_runtime
+          if configuration.unique_in_queue || configuration.unique_at_runtime || configuration.unique_across_queues
             # @return [Array<String, arguments>] the key base hash used to enforce uniqueness, and the arguments from the payload used to calculate it
             define_method(:redis_unique_hash) do |payload|
               payload = Resque.decode(Resque.encode(payload))
+              puts "payload is #{payload.inspect}" if ENV['RESQUE_DEBUG'] == 'true'
               job  = payload["class"]
               # It seems possible that some jobs may not have an "args" key in the payload.
               args = payload["args"] || []
@@ -20,28 +21,64 @@ module Resque
               return [Digest::MD5.hexdigest(Resque.encode(args)), uniqueness_args]
             end
           end
-          if configuration.unique_in_queue
-            # Returns a string, used by LonelyJob, that will be used as the redis key
+
+          if configuration.lock_after_execution_period
+            self.instance_variable_set(:@lock_after_execution_period, configuration.lock_after_execution_period)
+          end
+          
+          if configuration.unique_in_queue || configuration.unique_across_queues
+            ### Gem: resque_solo
+            ### Plugin Name: Resque::Plugins::UniqueJob
+            ### Provides: Queue-time uniqueness for a single queue, or across queues
+            #
+            # Returns a string, used by Resque::Plugins::UniqueJob, that will be used as the prefix to the redis key
+            define_method(:solo_redis_key_prefix) do
+              "unique_job:#{self}"
+            end
+            #
+            # Returns a string, used by Resque::Plugins::UniqueJob, that will be used as the redis key
             # The example in the readme is bad.  The args passed to this method are like:
             # [{:class=>"MakeCompanyReport", :args=>[1]}]
-            # This is identical to the version from Resque::Plugins::UniqueJob
-            # and we do not want the version from Resque::Plugins::LonelyJob to be used.
             # Payload is what Resque stored for this job along with the job's class name:
             #   a hash containing :class and :args
             # @return [String] the key used to enforce uniqueness (at queue-time)
-            define_method(:redis_key) do |payload|
+            define_method(:unique_at_queue_time_redis_key) do |queue, payload|
               unique_hash, args_for_uniqueness = redis_unique_hash(payload)
-              key = "unique_job:#{self}:#{unique_hash}"
-              puts "#{self}.redis_key for #{args_for_uniqueness} is: #{key.green}"
+              key = "#{solo_key_namespace(queue)}:#{solo_redis_key_prefix}:#{unique_hash}"
+              puts "#{ColorizedString['[Queue-Time]'].green} #{self}.unique_at_queue_time_redis_key for #{args_for_uniqueness} is: #{ColorizedString[key].green}" if ENV['RESQUE_DEBUG'] == 'true'
               key
             end
+            #
             # @return [Fixnum] number of keys that were deleted
-            define_method(:purge_unique_job_redis_keys) do
-              keys = Resque.redis.keys("unique_job:#{self}:*")
+            define_method(:purge_unique_queued_redis_keys) do
+              # solo_key_namespace may or may not ignore the queue passed in, depending on config.
+              key_match = "#{solo_key_namespace(self.instance_variable_get(:@queue))}:#{solo_redis_key_prefix}:*"
+              keys = Resque.redis.keys(key_match)
+              puts "Purging #{keys.length} keys from #{ColorizedString[key_match].red}"
               Resque.redis.del keys if keys.length > 0
             end
+            if configuration.unique_in_queue
+              # @return [String] the Redis namespace of the key used to enforce uniqueness (at queue-time)
+              define_method(:solo_key_namespace) do |queue = nil|
+                "solo:queue:#{queue}:job"
+              end
+            elsif configuration.unique_across_queues
+              # @return [String] the Redis namespace of the key used to enforce uniqueness (at queue-time)
+              define_method(:solo_key_namespace) do |_queue = nil|
+                "solo:across_queues:job"
+              end
+            end
           end
+
+          ### Gem: resque-unique_at_runtime
+          ### Plugin Name: Resque::Plugins::UniqueAtRuntime
+          ### Provides: Runtime uniqueness across queues
           if configuration.unique_at_runtime
+            # @return [String] the Redis namespace of the key used to enforce uniqueness (at runtime)
+            define_method(:runtime_key_namespace) do
+              "unique_at_runtime:#{self}"
+            end
+            # Returns a string, used by Resque::Plugins::UniqueAtRuntime, that will be used as the redis key
             # The versions of redis_key from resque_solo and resque-lonely_job are incompatible.
             # So we forked resque-lonely_job, change the name of the method so it would not conflict,
             #   and now we can override it, and fix the params to be compatible with the redis_key
@@ -49,15 +86,17 @@ module Resque
             # Does not need any customization for arity, because it funnels down to redis_key,
             #   and we handle the arity option there
             # @return [String] the key used to enforce loneliness (uniqueness at runtime)
-            define_method(:lonely_job_redis_key) do |*args|
+            define_method(:unique_at_runtime_redis_key) do |*args|
               unique_hash, args_for_uniqueness = redis_unique_hash({"class" => self.to_s, "args" => args})
-              key = "lonely_job:#{self}:#{unique_hash}"
-              puts "#{self}.lonely_job_redis_key for #{args_for_uniqueness} is: #{key.yellow}"
+              key = "#{runtime_key_namespace}:#{unique_hash}"
+              puts "#{ColorizedString['[Run-Time]'].yellow} #{self}.unique_at_runtime_redis_key for #{args_for_uniqueness} is: #{ColorizedString[key].yellow}" if ENV['RESQUE_DEBUG'] == 'true'
               key
             end
             # @return [Fixnum] number of keys that were deleted
-            define_method(:purge_lonely_job_redis_keys) do
-              keys = Resque.redis.keys("lonely_job:#{self}:*")
+            define_method(:purge_unique_at_runtime_redis_keys) do
+              key_match = "#{runtime_key_namespace}:*"
+              keys = Resque.redis.keys(key_match)
+              puts "Purging #{keys.length} keys from #{ColorizedString[key_match].red}"
               Resque.redis.del keys if keys.length > 0
             end
           end

data/lib/resque/unique_by_arity/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module UniqueByArity
-    VERSION = "0.2.1"
+    VERSION = "1.0.0"
   end
 end

data/lib/resque/unique_by_arity.rb

@@ -1,5 +1,9 @@
-require "resque-lonely_job"
+require "colorized_string"
+
+require "resque-unique_at_runtime"
 require "resque_solo"
+# Little hack of the resque_solo gem
+require "resque/resque_solo/queue"
 
 require "resque/unique_by_arity/version"
 require "resque/unique_by_arity/configuration"
@@ -72,6 +76,12 @@ module Resque
     def uniqueness_unique_in_queue=(unique_in_queue)
       @uniqueness_configuration.unique_in_queue = unique_in_queue
     end
+    def uniqueness_unique_across_queues
+      @uniqueness_configuration.unique_across_queues
+    end
+    def uniqueness_unique_across_queues=(unique_across_queues)
+      @uniqueness_configuration.unique_across_queues = unique_across_queues
+    end
     self.uniqueness_configuration = Configuration.new # setup defaults
   end
 end

data/resque-unique_by_arity.gemspec

@@ -9,9 +9,10 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Peter Boling"]
   spec.email         = ["peter.boling@gmail.com"]
 
-  spec.summary       = %q{Magic hacks which allow integration of resque_solo  and resque-lonely_simlutaneously into Resque jobs}
-  spec.description   = %q{resque_solo  and resque-lonely are incompatible - fixes that}
+  spec.summary       = %q{Magic hacks which allow integration of resque_solo and resque-unique_at_runtime_simultaneously into Resque jobs}
+  spec.description   = %q{fixes incompatibilities between resque_solo and resque-unique_at_runtime}
   spec.homepage      = "https://github.com/pboling/resque-unique_by_arity"
+  spec.required_ruby_version = ">= 2.0.0"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -20,8 +21,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "resque-lonely_job"
-  spec.add_runtime_dependency "resque_solo"
+  spec.add_runtime_dependency "resque-unique_at_runtime", "~> 2.0"
+  spec.add_runtime_dependency "resque_solo", "~> 0.3"
+  spec.add_runtime_dependency "colorize", "~> 0.8"
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: resque-unique_by_arity
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Peter Boling
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-30 00:00:00.000000000 Z
+date: 2017-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: resque-lonely_job
+  name: resque-unique_at_runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: resque_solo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +94,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: resque_solo  and resque-lonely are incompatible - fixes that
+description: fixes incompatibilities between resque_solo and resque-unique_at_runtime
 email:
 - peter.boling@gmail.com
 executables: []
@@ -95,6 +109,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- lib/resque/resque_solo/queue.rb
 - lib/resque/unique_by_arity.rb
 - lib/resque/unique_by_arity/configuration.rb
 - lib/resque/unique_by_arity/cop.rb
@@ -112,7 +127,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -123,6 +138,6 @@ rubyforge_project:
 rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
-summary: Magic hacks which allow integration of resque_solo  and resque-lonely_simlutaneously
+summary: Magic hacks which allow integration of resque_solo and resque-unique_at_runtime_simultaneously
   into Resque jobs
 test_files: []