active_job_resque_solo 0.2.0 → 0.3.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f760861754e12e2de29870716638d38d93ceaa09
-  data.tar.gz: 553f61b9c2b9653afc9217dc45f8439273311a06
+  metadata.gz: b9c434bd9ef7b517ac674178db86450d4220458e
+  data.tar.gz: 7ad486c09cd6dcd27b068172fcf4888975d7ecdf
 SHA512:
-  metadata.gz: e558f3fef406f0bf9b39740b697c05e889dda701a6aecc24684c6dd5106ce9b28ffd163da394ede37bcd972d9f9590033f1ed51ff786dc7f0ae4fe7c965fe7ab
-  data.tar.gz: f671a673aa14aacece35e289639331fd3dee4b5a723c7fa4efdce1ec10c806d71e194f2b0820aa6f8dc11e457e4c668bfb8cf998fad43c3c0c7829c05c15b1f2
+  metadata.gz: ab4a3fc4b9a73e1ff7687cd00e648a71a7f7e1206294d453c44dbdde61bf6a29d03e3de2b7544175c1380fca79d5a3755fdc9449eb700146dcd228adaf97fe0a
+  data.tar.gz: a93fcdacfbfa92cb621ad935062171b5b3437dc452756773e487f87e005fc21ca1d60672b315aabefc237c9a4294d44834551181cf3c0bb3cc53a67a234b9c7b

data/.travis.yml

@@ -1,5 +1,6 @@
 sudo: false
 language: ruby
 rvm:
+  - 2.3.4
   - 2.4.1
 before_install: gem install bundler -v 1.15.4

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# 0.3.0.pre
+
+* Adds locking to prevent two or more processes in a race condition from enquing multiple copies of a job.
+
+* Adds the `solo_lock_key_prefix` directive to set the lock key prefix for your job.
+
+# 0.2.0
+
+* Removes internal methods used by Solo from your Job class' namespace.
+
+# 0.1.0
+
+* Initial version of ActiveJob::Plugins::Resque::Solo.

data/README.md

@@ -1,6 +1,8 @@
 # ActiveJobResqueSolo
 
-A plugin for ActiveJob with Resque to prevent duplicate enqueing of jobs.
+A plugin for ActiveJob with Resque to prevent duplicate enqueuing of jobs.
+
+[![Build Status](https://travis-ci.org/kinkade/active_job_resque_solo.svg?branch=master)](https://travis-ci.org/kinkade/active_job_resque_solo)
 
 ## Installation
 
@@ -72,21 +74,41 @@ class MyJob < ActiveJob::Base
   end
 end
 ```
+## Locking
 
-## Duplicate enqueues are still possible
+Solo uses an internal locking mechanism to prevent multiple processes from
+enqueuing the same job during race conditions.  This gem does not perform any
+locking around job execution.
 
-While this plugin will greatly reduce duplicate instances of a job from being
-enqueued, there are two scenarios where duplicates may still be enqueued,
-so be sure to check out other gems for locking if your job is not idempotent.
+The lock prevents competing jobs of the same class and arguments from being
+enqueued, complying with the argument filtering programmed with `solo_only_args`
+and `solo_except_args`.
+
+The default Redis key prefix is "ajr_solo".  It can be set to a different,
+arbitrary string of your choice using `solo_lock_key_prefix`:
+
+```ruby
+class MyJob < ActiveJob::Base
+
+  include ActiveJob::Plugins::Resque::Solo
 
-1. When multiple processes simultaneously attempt to enqueue the same job, two or
-more instances may be enqueued.
-2. If your queue has many jobs, and workers remove a job while Solo scans
-the queue, it's possible for the original enqueued job to be missed. Solo will allow
-the new instance of the job to be enqueued.
+  solo_lock_key_prefix "my_lock_prefix"
+
+  def perform(*args)
+  end
+end
+```
+
+## Duplicate enqueues are possible
+
+While this plugin will greatly reduce duplicate instances of a job from being
+enqueued, a job may be enqueued multiple times if the Redis response times are
+very slow. Slowness could be caused by extremely high load on Redis or networking
+issues.
 
-Solo errors towards allowing a duplicate job instance to be enqueued rather than
-prevent a job from being enqueued at all.
+The locks are acquired for dynamic amounts of time, but expire quickly, typically
+in one second. Killed workers will not leave long-lived, orphaned locks to
+adversely block jobs from being enqueued.
 
 ## Contributing
 
@@ -98,4 +120,4 @@ The gem is available as open source under the terms of the [MIT License](http://
 
 ## Code of Conduct
 
-Everyone interacting in the ActiveJobResqueSolo project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/active_job_resque_solo/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the ActiveJobResqueSolo project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kinkade/active_job_resque_solo/blob/master/CODE_OF_CONDUCT.md).

data/active_job_resque_solo.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Prevents duplicate ActiveJob+Resque jobs from being enqueued.}
   spec.description   = %q{If you are using ActiveJob with the Resque Adapter, this gem will help prevent duplicate jobs, based on arguments, from being enqueued to Resque.}
   spec.license       = "MIT"
+  spec.homepage      = "https://github.com/kinkade/active_job_resque_solo"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})

data/lib/active_job/plugins/resque/solo/inspector.rb

@@ -1,14 +1,19 @@
+require_relative "lock"
+require 'json'
+require 'digest/sha1'
+
 module ActiveJob
   module Plugins
     module Resque
       module Solo
         class Inspector
 
-          def initialize(only_args, except_args)
+          def initialize(only_args, except_args, lock_key_prefix)
             @only_args = only_args
             @except_args = except_args || []
             # always ignore the ActiveJob symbol hash key.
             @except_args << "_aj_symbol_keys" unless @except_args.include?("_aj_symbol_keys")
+            @lock_key_prefix = lock_key_prefix.present? ? lock_key_prefix : "ajr_solo"
           end
 
           def self.resque_present?
@@ -18,8 +23,13 @@ module ActiveJob
           def around_enqueue(job, block)
             if Inspector::resque_present?
 
-              if !job_enqueued?(job) && !job_executing?(job)
-                block.call
+              Lock.try_acquire_release(lock_key(job)) do |lock, extend_at|
+                @lock = lock
+                @extend_lock_at = extend_at
+
+                if !job_enqueued?(job) && !job_executing?(job)
+                  block.call
+                end
               end
             else
               # if resque is not present, always enqueue
@@ -31,25 +41,18 @@ module ActiveJob
             size = ::Resque.size(job.queue_name)
             return false if size.zero?
 
-            page_size = 250
-            pages = (size/page_size).to_i + 1
-            jobs = []
-
-            # It's possible for this loop to skip jobs if they
-            # are dequeued while the loop is in progress.
-            (0..pages).each do |i|
-              page_start = i * page_size
-              page = ::Resque.peek(job.queue_name, page_start, page_size)
-              break if page.empty?
-              jobs += page
-            end
+            scheduled_jobs = ::Resque.peek(job.queue_name, 0, 0)
+
+            extend_lock
 
             job_class, job_arguments = job(job)
 
-            (jobs.size-1).downto(0) do |i|
-              scheduled_job = jobs[i]
+            (scheduled_jobs.size-1).downto(0) do |i|
+              scheduled_job = scheduled_jobs[i]
               return true if job_enqueued_with_args?(job_class, job_arguments, scheduled_job)
+              extend_lock
             end
+
             false
           end
 
@@ -62,6 +65,8 @@ module ActiveJob
               args = processing["payload"]["args"][0]
               job_with_args_eq?(job_class, job_arguments, args)
             end
+
+            extend_lock
           end
 
           def job_enqueued_with_args?(job_class, job_arguments, scheduled_job)
@@ -97,6 +102,18 @@ module ActiveJob
 
             args
           end
+
+          def lock_key(job)
+            job_class, job_arguments = job(job)
+            sha1 = Digest::SHA1.hexdigest(job_arguments.to_json)
+            "#{@lock_key_prefix}:#{job_class}:#{sha1}"
+          end
+
+          def extend_lock
+            if Time.now.utc >= @extend_lock_at
+              @extend_lock_at = @lock.extend
+            end
+          end
         end
       end
     end

data/lib/active_job/plugins/resque/solo/lock.rb

@@ -0,0 +1,86 @@
+module ActiveJob
+  module Plugins
+    module Resque
+      module Solo
+        class Lock
+          # TTLs in seconds
+          ACQUIRE_TTL = 1.0
+          EXECUTE_TTL = 5.0
+
+          def initialize(key)
+            @redis = ::Resque.redis
+            @uuid = ::SecureRandom.uuid
+            @acquired = nil
+            @key = key
+          end
+
+          # Attempts to acquire a lock named by the key in Redis.  If the lock
+          # is acquired, the block is executed.  If the lock is not acquired,
+          # the block is not executed.
+          #
+          # @param [String] key the key used to articulate the lock
+          # @yield the block to execute if the lock can be acquired
+          # @return [Boolean] x if the block was executed, false if the block was not executed
+          def self.try_acquire_release(key)
+            lock = Lock.new(key)
+
+            extend_at = lock.try_acquire
+            return false if extend_at.nil?
+
+            begin
+              yield(lock, extend_at)
+            ensure
+              lock.release
+            end
+
+            true
+          end
+
+          def try_acquire
+            extend_at = Time.now.utc + (ACQUIRE_TTL.to_f / 2)
+            px = (ACQUIRE_TTL.to_f * 1000).to_i
+
+            if @redis.set(@key, @uuid, px: px, nx: true)
+              @acquired = @uuid
+            else
+              extend_at = Time.now.utc
+              @acquired = @redis.get(@key)
+            end
+
+            # Consider the lock not acquired if it is proven
+            # that another process has acquired the lock.
+            #
+            # It is unlikely that acquired will be nil, but
+            # it is possible if Redis is slow due to extreme load.
+            (@acquired.nil? || @acquired == @uuid) ? extend_at : nil
+          end
+
+          def extend
+            extend_at = 1.year.from_now
+
+            @redis.watch(@key) do
+              if @redis.get(@key) == @uuid
+                extend_at = Time.now.utc + (EXECUTE_TTL.to_f / 2)
+                @redis.multi do |multi|
+                  multi.expire(@key, EXECUTE_TTL.to_i)
+                end
+              end
+            end
+
+            extend_at
+          end
+
+          def release
+            @redis.watch(@key) do
+              if @redis.get(@key) == @uuid
+                @redis.multi do |multi|
+                  multi.del(@key)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_job/plugins/resque/solo.rb

@@ -1,4 +1,5 @@
 require_relative 'solo/inspector'
+require_relative 'solo/lock'
 
 module ActiveJob
   module Plugins
@@ -24,7 +25,13 @@ module ActiveJob
           end
 
           def solo_inspector
-            @solo_inspector ||= Inspector.new(@solo_only_args, @solo_except_args)
+            @solo_inspector ||= Inspector.new(@solo_only_args, @solo_except_args, @solo_lock_key_prefix)
+          end
+
+          def solo_lock_key_prefix(key_prefix)
+            @solo_lock_key_prefix = key_prefix.strip
+            raise ArgumentError, "solo_lock_key_prefix cannot be blank or only spaces." if @solo_lock_key_prefix.blank?
+
           end
         end
       end

data/lib/active_job_resque_solo/version.rb

@@ -1,3 +1,3 @@
 module ActiveJobResqueSolo
-  VERSION = "0.2.0"
+  VERSION = "0.3.0.pre"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_job_resque_solo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0.pre
 platform: ruby
 authors:
 - Phillip Kinkade
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-03 00:00:00.000000000 Z
+date: 2017-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -97,6 +97,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -107,9 +108,10 @@ files:
 - bin/setup
 - lib/active_job/plugins/resque/solo.rb
 - lib/active_job/plugins/resque/solo/inspector.rb
+- lib/active_job/plugins/resque/solo/lock.rb
 - lib/active_job_resque_solo.rb
 - lib/active_job_resque_solo/version.rb
-homepage: 
+homepage: https://github.com/kinkade/active_job_resque_solo
 licenses:
 - MIT
 metadata: {}
@@ -124,9 +126,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.6