sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1

data/redis/create.lua

@@ -0,0 +1,58 @@
+-- redis.replicate_commands();
+
+local exists_key    = KEYS[1]
+local grabbed_key   = KEYS[2]
+local available_key = KEYS[3]
+local version_key   = KEYS[4]
+local unique_digest = KEYS[5]
+
+local job_id        = ARGV[1]
+local expiration    = tonumber(ARGV[2])
+local api_version   = ARGV[3]
+local concurrency   = tonumber(ARGV[4])
+
+local stored_token  = redis.call('GETSET', exists_key, job_id)
+
+redis.log(redis.LOG_DEBUG, "create.lua - starting...")
+
+if stored_token then
+  redis.log(redis.LOG_DEBUG, "create.lua - stored_token: " .. stored_token)
+  return stored_token
+end
+
+----------------------------------------------------------------
+-- TODO: Legacy support (Remove in v6.1)
+local old_token  = redis.call('GET', unique_digest)
+if old_token then
+  -- redis.log(redis.LOG_DEBUG, "create.lua - " .. unique_digest .. " with "  .. old_token)
+  if old_token == job_id or old_token == '2' then
+    -- redis.log(redis.LOG_DEBUG, "create.lua - " .. unique_digest .. " with "  .. old_token .. " MATCH with " .. job_id)
+    redis.call('DEL', unique_digest)
+  else
+    -- redis.log(redis.LOG_DEBUG, "create.lua - " .. unique_digest .. " with "  .. old_token .. " MISMATCH with " .. job_id)
+    return old_token
+  end
+end
+----------------------------------------------------------------
+
+redis.call('EXPIRE', exists_key, 10)
+redis.call('DEL', grabbed_key)
+redis.call('DEL', available_key)
+
+if concurrency and concurrency > 1 then
+  for index = 1, concurrency do
+    redis.call('RPUSH', available_key, index)
+  end
+else
+  redis.call('RPUSH', available_key, job_id)
+end
+redis.call('GETSET', version_key, api_version)
+redis.call('PERSIST', exists_key)
+
+if expiration then
+  redis.call('EXPIRE', available_key, expiration)
+  redis.call('EXPIRE', exists_key, expiration)
+  redis.call('EXPIRE', version_key, expiration)
+end
+
+return job_id

data/redis/delete.lua

@@ -0,0 +1,11 @@
+local exists_key    = KEYS[1]
+local grabbed_key   = KEYS[2]
+local available_key = KEYS[3]
+local version_key   = KEYS[4]
+local unique_digest = KEYS[5] -- TODO: Legacy support (Remove in v6.1)
+
+redis.call('DEL', exists_key)
+redis.call('DEL', grabbed_key)
+redis.call('DEL', available_key)
+redis.call('DEL', version_key)
+redis.call('DEL', unique_digest) -- TODO: Legacy support (Remove in v6.1)

data/redis/release_stale_locks.lua

@@ -0,0 +1,90 @@
+redis.replicate_commands();
+
+local exists_key           = KEYS[1]
+local grabbed_key          = KEYS[2]
+local available_key        = KEYS[3]
+local release_key          = KEYS[4]
+local version_key          = KEYS[5]
+
+local expires_in           = tonumber(ARGV[1])
+local stale_client_timeout = tonumber(ARGV[2])
+local expiration           = tonumber(ARGV[3])
+
+local function current_time()
+  local time = redis.call('time')
+  local s = time[1]
+  local ms = time[2]
+  local number = tonumber((s .. '.' .. ms))
+
+  return number
+end
+
+local hgetall = function (key)
+  local bulk = redis.call('HGETALL', key)
+  local result = {}
+  local nextkey
+  for i, v in ipairs(bulk) do
+    if i % 2 == 1 then
+      nextkey = v
+    else
+      result[nextkey] = v
+    end
+  end
+  return result
+end
+
+local cached_current_time = current_time()
+redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - started at : " .. cached_current_time)
+
+local my_lock_expires_at = cached_current_time + expires_in + 1
+redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - my_lock_expires_at: " .. my_lock_expires_at)
+
+if not redis.call('SETNX', release_key, my_lock_expires_at) then
+  -- Check if expired
+  local other_lock_expires_at = tonumber(redis.call('GET', release_key))
+  redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - other_lock_expires_at: " .. other_lock_expires_at)
+
+  if other_lock_expires_at < cached_current_time then
+    local old_expires_at = tonumber(redis.call('GETSET', release_key, my_lock_expires_at))
+    redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - old_expires_at: " .. old_expires_at)
+
+    -- Check if another client started cleanup yet. If not,
+    -- then we now have the lock.
+    if not old_expires_at == other_lock_expires_at then
+      redis.log(redis.LOG_DEBUG, "release_stale_locks.lua -could not retrieve lock: exiting 0")
+      return 0
+    end
+  end
+end
+
+local keys = hgetall(grabbed_key)
+for key, locked_at in pairs(keys) do
+  local timed_out_at = tonumber(locked_at) + stale_client_timeout
+  redis.log(redis.LOG_DEBUG,
+    "release_stale_locks.lua - processing: " .. grabbed_key .. " key: " .. key .. " locked_at: " .. locked_at
+  )
+
+  if timed_out_at < current_time() then
+    redis.log(redis.LOG_DEBUG, "HDEL " .. grabbed_key .. ":" .. key)
+    redis.call('HDEL', grabbed_key, key)
+    redis.log(redis.LOG_DEBUG, "LPUSH " .. available_key .. ":" .. key)
+    redis.call('LPUSH', available_key, key)
+
+    if expiration then
+      redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - EXPIRE " .. available_key .. " with " .. expiration)
+      redis.call('EXPIRE', available_key, expiration)
+      redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - EXPIRE " .. exists_key .. " with " .. expiration)
+      redis.call('EXPIRE', exists_key, expiration)
+    end
+  end
+end
+
+-- Make sure not to delete the lock in case someone else already expired
+-- our lock, with one second in between to account for some lag.
+if my_lock_expires_at > (current_time() - 1) then
+  redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - DEL " .. release_key)
+  redis.call('DEL', release_key)
+end
+
+redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - comleted at : " .. current_time())
+return 1

data/redis/signal.lua

@@ -0,0 +1,21 @@
+local exists_key    = KEYS[1]
+local grabbed_key   = KEYS[2]
+local available_key = KEYS[3]
+local version_key   = KEYS[4]
+local unique_digest = KEYS[5] -- TODO: Legacy support (Remove in v6.1)
+
+local token      = ARGV[1]
+local expiration = tonumber(ARGV[2])
+
+redis.call('HDEL', grabbed_key, token)
+local available_count = redis.call('LPUSH', available_key, token)
+
+if expiration then
+  redis.log(redis.LOG_DEBUG, "signal_locks.lua - expiring stale locks")
+  redis.call('EXPIRE', exists_key, expiration)
+  redis.call('EXPIRE', available_key, expiration)
+  redis.call('EXPIRE', version_key, expiration)
+  redis.call('EXPIRE', unique_digest, expiration) -- TODO: Legacy support (Remove in v6.1)
+end
+
+return available_count

data/sidekiq-unique-jobs.gemspec

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
-require File.expand_path('../lib/sidekiq_unique_jobs/version', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'sidekiq_unique_jobs/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'sidekiq-unique-jobs'
@@ -15,18 +18,22 @@ Gem::Specification.new do |spec|
 
   spec.bindir        = 'bin'
   spec.executables   = %w[jobs]
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features|gemfiles|pkg|rails_example|tmp)/})
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject do |file|
+      file.match(%r{^(test|spec|features|gemfiles|pkg|rails_example|tmp)/})
+    end
   end
 
   spec.require_paths = ['lib']
+  spec.add_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.5'
   spec.add_dependency 'sidekiq', '>= 4.0', '< 6.0'
-  spec.add_dependency 'concurrent-ruby', '>= 1.0.2'
   spec.add_dependency 'thor', '~> 0'
 
-  spec.add_development_dependency 'rspec', '~> 3.1'
-  spec.add_development_dependency 'rake', '~> 12.0'
-  spec.add_development_dependency 'timecop', '~> 0.8'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rspec', '~> 3.7'
+  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'timecop', '~> 0.9'
   spec.add_development_dependency 'yard', '~> 0.9'
-  spec.add_development_dependency 'gem-release', '~> 0.7'
+  spec.add_development_dependency 'gem-release', '~> 1.0'
+  spec.add_development_dependency 'awesome_print', '~> 1.8'
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 5.0.11
+  version: 6.0.0.rc1
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-19 00:00:00.000000000 Z
+date: 2018-06-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -31,75 +51,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '1.16'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -120,14 +140,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '1.8'
 description: Handles various types of unique jobs for Sidekiq
 email:
 - mikael@zoolutions.se
@@ -144,6 +178,7 @@ files:
 - ".eslintrc"
 - ".fasterer.yml"
 - ".gitignore"
+- ".reek"
 - ".rspec"
 - ".rubocop.yml"
 - ".simplecov"
@@ -152,44 +187,85 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/bench
 - bin/jobs
+- examples/another_unique_job.rb
+- examples/custom_queue_job.rb
+- examples/custom_queue_job_with_filter_method.rb
+- examples/custom_queue_job_with_filter_proc.rb
+- examples/expiring_job.rb
+- examples/inline_worker.rb
+- examples/just_a_worker.rb
+- examples/long_running_job.rb
+- examples/main_job.rb
+- examples/my_job.rb
+- examples/my_unique_job.rb
+- examples/my_unique_job_with_filter_method.rb
+- examples/my_unique_job_with_filter_proc.rb
+- examples/notify_worker.rb
+- examples/plain_class.rb
+- examples/simple_worker.rb
+- examples/spawn_simple_worker.rb
+- examples/test_class.rb
+- examples/unique_across_workers_job.rb
+- examples/unique_job_with_conditional_parameter.rb
+- examples/unique_job_with_filter_method.rb
+- examples/unique_job_with_nil_unique_args.rb
+- examples/unique_job_with_no_unique_args_method.rb
+- examples/unique_job_withthout_unique_args_parameter.rb
+- examples/unique_on_all_queues_job.rb
+- examples/until_and_while_executing_job.rb
+- examples/until_executed_2_job.rb
+- examples/until_executed_job.rb
+- examples/until_executing_job.rb
+- examples/until_expired_job.rb
+- examples/until_global_expired_job.rb
+- examples/while_executing_job.rb
+- examples/while_executing_reject_job.rb
+- examples/without_argument_job.rb
 - lib/sidekiq-unique-jobs.rb
 - lib/sidekiq/simulator.rb
+- lib/sidekiq_unique_jobs.rb
 - lib/sidekiq_unique_jobs/cli.rb
 - lib/sidekiq_unique_jobs/client/middleware.rb
-- lib/sidekiq_unique_jobs/config.rb
+- lib/sidekiq_unique_jobs/connection.rb
 - lib/sidekiq_unique_jobs/constants.rb
 - lib/sidekiq_unique_jobs/core_ext.rb
-- lib/sidekiq_unique_jobs/lock.rb
+- lib/sidekiq_unique_jobs/exceptions.rb
+- lib/sidekiq_unique_jobs/lock/base_lock.rb
 - lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb
 - lib/sidekiq_unique_jobs/lock/until_executed.rb
 - lib/sidekiq_unique_jobs/lock/until_executing.rb
-- lib/sidekiq_unique_jobs/lock/until_timeout.rb
+- lib/sidekiq_unique_jobs/lock/until_expired.rb
 - lib/sidekiq_unique_jobs/lock/while_executing.rb
+- lib/sidekiq_unique_jobs/lock/while_executing_reject.rb
+- lib/sidekiq_unique_jobs/lock/while_executing_requeue.rb
+- lib/sidekiq_unique_jobs/locksmith.rb
+- lib/sidekiq_unique_jobs/logging.rb
 - lib/sidekiq_unique_jobs/middleware.rb
 - lib/sidekiq_unique_jobs/normalizer.rb
 - lib/sidekiq_unique_jobs/options_with_fallback.rb
-- lib/sidekiq_unique_jobs/run_lock_failed.rb
-- lib/sidekiq_unique_jobs/script_mock.rb
 - lib/sidekiq_unique_jobs/scripts.rb
-- lib/sidekiq_unique_jobs/scripts/acquire_lock.rb
-- lib/sidekiq_unique_jobs/scripts/release_lock.rb
 - lib/sidekiq_unique_jobs/server/middleware.rb
 - lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
+- lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb
 - lib/sidekiq_unique_jobs/testing.rb
-- lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb
-- lib/sidekiq_unique_jobs/timeout_calculator.rb
+- lib/sidekiq_unique_jobs/timeout.rb
+- lib/sidekiq_unique_jobs/timeout/calculator.rb
 - lib/sidekiq_unique_jobs/unique_args.rb
 - lib/sidekiq_unique_jobs/unlockable.rb
 - lib/sidekiq_unique_jobs/util.rb
 - lib/sidekiq_unique_jobs/version.rb
 - redis/acquire_lock.lua
+- redis/create.lua
+- redis/delete.lua
 - redis/release_lock.lua
-- redis/synchronize.lua
+- redis/release_stale_locks.lua
+- redis/signal.lua
 - sidekiq-unique-jobs.gemspec
 homepage: https://github.com/mhenrixon/sidekiq-unique-jobs
 licenses:
@@ -206,12 +282,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Uniqueness for Sidekiq Jobs