sqeduler 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 510d5754df3875e36f052b971668cf58ccdcdde1
-  data.tar.gz: dd0b216fe36b27a4df8f76dbd243e4f088b45fe8
+  metadata.gz: 9331eead88bcd85bcd8e564ce22b5de81b3a43ee
+  data.tar.gz: b2ae466592349be888d1ff930f344e1b32702587
 SHA512:
-  metadata.gz: 64eb353667433b507b725db2b9b92adc5dde0ec71a140ddc640a5c74a52f42dcbfa1dc63ddeba27b4c8ebd0b188711e5256cac9f89d2edd45cff0f5d635d1e7c
-  data.tar.gz: 1e7cde947da623040ee8f07218fc7b02be92b0c4bec6f54b9688068fc662b090eaadd8f6a3e8466493244aa43ed9e395f552be5443f3dee36bb13a19857c43e0
+  metadata.gz: 0b7af459f6a789931ad6a078650449e3efe51641f3b863206a9c99283740bb636010ecc581abd946027773f14fa16f820f1fb6df2cfb90b9476b8f9c2a913109
+  data.tar.gz: 416c4d0f6c7059fbd40e5aeb8495b46c25f67f919e314196c1df4a8313dd0475d6ea159c229dede39015b282f9ce987c7078bdbf9894494704096412caf47019

data/CHANGES.md

@@ -1,3 +1,7 @@
+### 0.3.0 / 2016-01-25
+
+* Added lock refresh to maintain exclusive locks until long running jobs finish
+
 ### 0.2.2 / 2015-11-11
 
 * Support ERB in job schedules

data/README.md

@@ -19,11 +19,10 @@ Provides loosely-coupled helpers for Sidekiq workers. Provides highly available
 
 ## Examples
 
-To install this gem with necessary forks:
+To install this gem, add it to your Gemfile:
 
 ```ruby
 gem 'sqeduler'
-gem 'sidekiq-scheduler', :github => 'ecin/sidekiq-scheduler', :branch => 'ecin/redis-lock' # https://github.com/Moove-it/sidekiq-scheduler/pull/38
 ```
 
 ### Scheduling
@@ -48,6 +47,8 @@ config.on_server_start = proc {|config| ... }
 config.on_client_start = proc {|config| ... }
 # required if you want to start the Sidekiq::Scheduler
 config.schedule_path = Rails.root.join('config').join('sidekiq_schedule.yml')
+# optional to maintain locks for exclusive jobs, see "Lock Maintainer" below
+config.maintain_locks = true
 
 Sqeduler::Service.config = config
 # Starts Sidekiq and Sidekiq::Scheduler
@@ -57,6 +58,12 @@ Sqeduler::Service.start
 See documentation for [Sidekiq::Scheduler](https://github.com/Moove-it/sidekiq-scheduler#scheduled-jobs-recurring-jobs)
 for specifics on how to construct your schedule YAML file.
 
+### Lock Maintainer
+
+Exclusive locks only last for the expiration you set. If your expiration is 30 seconds and the job runs for 60 seconds, you can have multiple jobs running at once. Rather than having to set absurdly high lock expirations, you can enable the `maintain_locks` option which handles this for you.
+
+Every 30 seconds, Sqeduler will look for any exclusive Sidekiq jobs that have been running for more than 30 seconds, and have a lock expiration of more than 30 seconds and refresh the lock.
+
 ### Worker Helpers
 
 To use `Sqeduler::Worker` modules:

data/lib/sqeduler/config.rb

@@ -3,7 +3,7 @@ module Sqeduler
   # Simple config for Sqeduler::Service
   class Config
     attr_accessor :logger, :redis_hash, :schedule_path,
-                  :on_server_start, :on_client_start
+                  :on_server_start, :on_client_start, :maintain_locks
 
     def initialize(opts = {})
       self.redis_hash = opts[:redis_hash]
@@ -11,6 +11,7 @@ module Sqeduler
       self.on_server_start = opts[:on_server_start]
       self.on_client_start = opts[:on_client_start]
       self.logger = opts[:logger]
+      self.maintain_locks = opts[:maintain_locks]
     end
   end
 end

data/lib/sqeduler/lock_maintainer.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+module Sqeduler
+  # This is to ensure that if you set your jobs to run one a time and something goes wrong
+  # causing a job to run for a long time, your lock won't expire.
+  # This doesn't stop long running jobs, it just ensures you only end up with one long running job
+  # rather than 20 of them.
+  class LockMaintainer
+    RUN_INTERVAL = 30
+    RUN_JITTER = 1..5
+
+    def initialize
+      @class_with_locks = {}
+    end
+
+    # This is only done when we initialize Sqeduler, don't need to worry about threading
+    def run
+      @maintainer_thread ||= Thread.new do
+        loop do
+          begin
+            synchronize
+          rescue => ex
+            Service.logger.error "[SQEDULER LOCK MAINTAINER] #{ex.class}, #{ex.message}"
+          end
+
+          sleep RUN_INTERVAL + rand(RUN_JITTER)
+        end
+      end
+    end
+
+    private
+
+    def synchronize
+      # Not great, but finding our identity in Sidekiq is a pain, and we already have locks in Sqeduler.
+      # Easier to just try and grab a lock each time and whichever server wins gets to do it.
+      return unless redis_lock.send(:take_lock)
+
+      now = Time.now.to_i
+
+      Service.redis_pool do |redis|
+        redis.pipelined do
+          workers.each do |_worker, _tid, args|
+            # No sense in pinging if it's not been running long enough to matter
+            next if (now - args["run_at"]) < RUN_INTERVAL
+
+            klass = str_to_class(args["payload"]["class"])
+            next unless klass
+
+            lock_key = klass.sync_lock_key(*args["payload"]["args"])
+
+            # This works because EXPIRE does not recreate the key, it only resets the expiration.
+            # We don't have to worry about atomic operations or anything like that.
+            # If the job finishes in the interim and deletes the key nothing will happen.
+            redis.expire(lock_key, klass.synchronize_jobs_expiration)
+          end
+        end
+      end
+    end
+
+    # Not all classes will use exclusive locks
+    def str_to_class(class_name)
+      return @class_with_locks[class_name] unless @class_with_locks[class_name].nil?
+
+      klass = class_name.constantize
+      if klass.respond_to?(:synchronize_jobs_mode)
+        # We only care about exclusive jobs that are long running
+        if klass.synchronize_jobs_mode == :one_at_a_time && klass.synchronize_jobs_timeout >= RUN_INTERVAL
+          return @class_with_locks[class_name] = klass
+        end
+      end
+
+      @class_with_locks[class_name] = false
+    end
+
+    def redis_lock
+      @redis_lock ||= RedisLock.new("sqeduler-lock-maintainer", :expiration => 60, :timeout => 0)
+    end
+
+    def workers
+      @workers ||= Sidekiq::Workers.new
+    end
+  end
+end

data/lib/sqeduler/redis_scripts.rb

@@ -23,12 +23,13 @@ module Sqeduler
     def load_sha(redis, script_name)
       @redis_sha_cache ||= {}
       @redis_sha_cache[script_name] ||= begin
-        script = if script_name == :refresh
-          refresh_lock_script
-        elsif script_name == :release
-          release_lock_script
-        else
-          fail "No script for #{script_name}"
+        script = case script_name
+                 when :refresh
+                   refresh_lock_script
+                 when :release
+                   release_lock_script
+                 else
+                   fail "No script for #{script_name}"
         end
         # strip leading whitespace of 8 characters
         redis.script(:load, script.gsub(/^ {8}/, ""))

data/lib/sqeduler/service.rb

@@ -5,7 +5,7 @@ module Sqeduler
   # Singleton class for configuring this Gem.
   class Service
     SCHEDULER_TIMEOUT = 60
-    MINIMUM_REDIS_VERSION = "2.6.12"
+    MINIMUM_REDIS_VERSION = "2.6.12".freeze
 
     class << self
       attr_accessor :config
@@ -44,6 +44,7 @@ module Sqeduler
             chain.add(Sqeduler::Middleware::KillSwitch)
           end
 
+          LockMaintainer.new.run if Service.config.maintain_locks
           Service.config.on_server_start.call(config) if Service.config.on_server_start
         end
       end
@@ -104,11 +105,8 @@ module Sqeduler
 
       def logger
         return config.logger if config.logger
-        if defined?(Rails)
-          Rails.logger
-        else
-          fail ArgumentError, "No logger provided and Rails.logger cannot be inferred"
-        end
+        return Rails.logger if defined?(Rails)
+        fail ArgumentError, "No logger provided and Rails.logger cannot be inferred"
       end
     end
   end

data/lib/sqeduler/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Sqeduler
-  VERSION = "0.2.2"
+  VERSION = "0.3.0".freeze
 end

data/lib/sqeduler/worker/kill_switch.rb

@@ -3,7 +3,7 @@ module Sqeduler
   module Worker
     # Uses Redis hashes to enabled and disable workers across multiple hosts.
     module KillSwitch
-      SIDEKIQ_DISABLED_WORKERS = "sidekiq.disabled-workers"
+      SIDEKIQ_DISABLED_WORKERS = "sidekiq.disabled-workers".freeze
 
       def self.prepended(base)
         if base.ancestors.include?(Sqeduler::Worker::Callbacks)

data/lib/sqeduler/worker/synchronization.rb

@@ -27,12 +27,20 @@ module Sqeduler
           return if synchronize_jobs_expiration
           fail ArgumentError, ":expiration is required!"
         end
+
+        def sync_lock_key(*args)
+          if args.empty?
+            name
+          else
+            "#{name}-#{args.join}"
+          end
+        end
       end
       # rubocop:enable Style/Documentation
 
       def perform(*args)
         if self.class.synchronize_jobs_mode == :one_at_a_time
-          perform_locked(sync_lock_key(*args)) do
+          perform_locked(self.class.sync_lock_key(*args)) do
             perform_timed do
               super
             end
@@ -44,14 +52,6 @@ module Sqeduler
 
       private
 
-      def sync_lock_key(*args)
-        if args.empty?
-          self.class.name
-        else
-          "#{self.class.name}-#{args.join}"
-        end
-      end
-
       # callback for when a lock cannot be obtained
       def on_lock_timeout(key)
         Service.logger.warn(

data/lib/sqeduler.rb

@@ -6,6 +6,7 @@ require "sqeduler/redis_scripts"
 require "sqeduler/lock_value"
 require "sqeduler/redis_lock"
 require "sqeduler/trigger_lock"
+require "sqeduler/lock_maintainer"
 require "sqeduler/middleware/kill_switch"
 require "sqeduler/service"
 require "sqeduler/worker/callbacks"

data/spec/fixtures/empty_schedule.yaml

@@ -0,0 +1 @@
+{}

data/spec/fixtures/env.rb

@@ -1,15 +1,14 @@
 require "sqeduler"
 require_relative "fake_worker"
 
-REDIS_CONFIG = {
-  :host => "localhost",
-  :db => 1,
-  :namespace => "sqeduler-tests"
-}
 Sidekiq.logger = Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG }
 
 Sqeduler::Service.config = Sqeduler::Config.new(
-  :redis_hash => REDIS_CONFIG,
+  :redis_hash => {
+    :host => "localhost",
+    :db => 1,
+    :namespace => "sqeduler-tests"
+  },
   :logger => Sidekiq.logger,
   :schedule_path => File.expand_path(File.dirname(__FILE__)) + "/schedule.yaml",
   :on_server_start => proc do |_config|

data/spec/fixtures/fake_worker.rb

@@ -1,12 +1,12 @@
 # encoding: utf-8
 # Sample worker for specs
 class FakeWorker
-  JOB_RUN_PATH =            "/tmp/job_run"
-  JOB_BEFORE_START_PATH =   "/tmp/job_before_start"
-  JOB_SUCCESS_PATH =        "/tmp/job_success"
-  JOB_FAILURE_PATH =        "/tmp/job_failure"
-  JOB_LOCK_FAILURE_PATH =   "/tmp/lock_failure"
-  SCHEDULE_COLLISION_PATH = "/tmp/schedule_collision"
+  JOB_RUN_PATH =            "/tmp/job_run".freeze
+  JOB_BEFORE_START_PATH =   "/tmp/job_before_start".freeze
+  JOB_SUCCESS_PATH =        "/tmp/job_success".freeze
+  JOB_FAILURE_PATH =        "/tmp/job_failure".freeze
+  JOB_LOCK_FAILURE_PATH =   "/tmp/lock_failure".freeze
+  SCHEDULE_COLLISION_PATH = "/tmp/schedule_collision".freeze
   include Sidekiq::Worker
   include Sqeduler::Worker::Everything
 

data/spec/lock_maintainer_spec.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+require "spec_helper"
+
+RSpec.describe Sqeduler::LockMaintainer do
+  let(:instance) { described_class.new }
+
+  before do
+    stub_const(
+      "SyncExclusiveWorker",
+      Class.new do
+        include Sidekiq::Worker
+        prepend Sqeduler::Worker::Synchronization
+        synchronize :one_at_a_time, :expiration => 300, :timeout => 30
+
+        def perform(*_args)
+          yield
+        end
+      end
+    )
+
+    stub_const(
+      "SyncWhateverWorker",
+      Class.new do
+        include Sidekiq::Worker
+        prepend Sqeduler::Worker::Synchronization
+
+        def perform
+          fail "This shouldn't be called"
+        end
+      end
+    )
+
+    Sqeduler::Service.config = Sqeduler::Config.new(
+      :redis_hash => REDIS_CONFIG,
+      :logger => Logger.new("/dev/null"),
+      :schedule_path => Pathname.new("./spec/fixtures/empty_schedule.yaml")
+    )
+  end
+
+  context "#run" do
+    let(:run) { instance.run }
+
+    it "calls into the synchronizer" do
+      expect(instance).to receive(:synchronize).at_least(1)
+
+      run.join(1)
+      run.terminate
+    end
+
+    it "doesn't die on errors" do
+      expect(instance).to receive(:synchronize).and_raise(StandardError, "Boom")
+
+      run.join(1)
+      expect(run.status).to_not be_falsy
+      run.terminate
+    end
+  end
+
+  context "#synchronize" do
+    subject(:sync) { instance.send(:synchronize) }
+
+    let(:run_at) { Time.now.to_i }
+    let(:job_args) { [1, { "a" => "b" }] }
+
+    let(:workers) do
+      [
+        [
+          "process-key",
+          "worker-tid-1234",
+          {
+            "run_at" => run_at,
+            "payload" => {
+              "class" => "SyncExclusiveWorker",
+              "args" => job_args
+            }
+          }
+        ],
+        [
+          "process-key",
+          "worker-tid-4321",
+          {
+            "run_at" => run_at,
+            "payload" => {
+              "class" => "SyncWhateverWorker",
+              "args" => []
+            }
+          }
+        ]
+      ]
+    end
+
+    before { allow(instance).to receive(:workers).and_return(workers) }
+
+    it "does nothing if the jobs just started" do
+      expect(instance).to_not receive(:str_to_class)
+      sync
+    end
+
+    context "when outside the run threshold" do
+      let(:run_at) { Time.now - described_class::RUN_INTERVAL - 5 }
+
+      let(:lock_key) { SyncExclusiveWorker.sync_lock_key(job_args) }
+
+      it "refresh the lock" do
+        SyncExclusiveWorker.new.perform(job_args) do
+          Sqeduler::Service.redis_pool do |redis|
+            # Change the lock timing to make sure ours works
+            redis.expire(lock_key, 10)
+            expect(redis.ttl(lock_key)).to eq(10)
+
+            # Run the re-locker
+            sync
+
+            # Confirm it reset
+            expect(redis.ttl(lock_key)).to eq(300)
+          end
+        end
+
+        # Shouldn't be around once the job finished
+        Sqeduler::Service.redis_pool do |redis|
+          expect(redis.exists(lock_key)).to eq(false)
+        end
+      end
+
+      it "obeys the exclusive lock" do
+        instance.send(:redis_lock).send(:take_lock)
+        expect(instance).to_not receive(:str_to_class)
+
+        sync
+      end
+    end
+  end
+
+  context "#str_to_class" do
+    it "only returns exclusive lock classes" do
+      expect(instance.send(:str_to_class, "SyncExclusiveWorker")).to eq(SyncExclusiveWorker)
+      expect(instance.send(:str_to_class, "SyncWhateverWorker")).to eq(false)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -4,11 +4,7 @@ require "rspec"
 require "sqeduler"
 require "timecop"
 
-REDIS_CONFIG = {
-  :host => "localhost",
-  :db => 1
-}
-TEST_REDIS = Redis.new(REDIS_CONFIG)
+TEST_REDIS = Redis.new(:host => "localhost", :db => 1)
 
 Timecop.safe_mode = true
 
@@ -16,6 +12,7 @@ RSpec.configure do |config|
   config.before(:each) do
     TEST_REDIS.flushdb
     Sqeduler::Service.config = nil
+    REDIS_CONFIG = { :host => "localhost", :db => 1 }.clone
   end
   config.disable_monkey_patching!
 end

data/sqeduler.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "pry", "~> 0"
   gem.add_development_dependency "rake", "~> 10"
   gem.add_development_dependency "rspec", "~> 3.3"
-  gem.add_development_dependency "rubocop", "~> 0.24"
+  gem.add_development_dependency "rubocop", "~> 0.36.0"
   gem.add_development_dependency "timecop", "~> 0"
   gem.add_development_dependency "yard", "~> 0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqeduler
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Jared Jenkins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-11 00:00:00.000000000 Z
+date: 2016-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.24'
+        version: 0.36.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.24'
+        version: 0.36.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -172,6 +172,7 @@ files:
 - Rakefile
 - lib/sqeduler.rb
 - lib/sqeduler/config.rb
+- lib/sqeduler/lock_maintainer.rb
 - lib/sqeduler/lock_value.rb
 - lib/sqeduler/middleware/kill_switch.rb
 - lib/sqeduler/redis_lock.rb
@@ -184,10 +185,12 @@ files:
 - lib/sqeduler/worker/kill_switch.rb
 - lib/sqeduler/worker/synchronization.rb
 - spec/config_spec.rb
+- spec/fixtures/empty_schedule.yaml
 - spec/fixtures/env.rb
 - spec/fixtures/fake_worker.rb
 - spec/fixtures/schedule.yaml
 - spec/integration_spec.rb
+- spec/lock_maintainer_spec.rb
 - spec/middleware/kill_switch_spec.rb
 - spec/service_spec.rb
 - spec/spec_helper.rb
@@ -216,16 +219,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.0
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Common Sidekiq infrastructure for multi-host applications.
 test_files:
 - spec/config_spec.rb
+- spec/fixtures/empty_schedule.yaml
 - spec/fixtures/env.rb
 - spec/fixtures/fake_worker.rb
 - spec/fixtures/schedule.yaml
 - spec/integration_spec.rb
+- spec/lock_maintainer_spec.rb
 - spec/middleware/kill_switch_spec.rb
 - spec/service_spec.rb
 - spec/spec_helper.rb