sidekiq-dejavu 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec3f881a31547703f392561450f5377d121bd295
-  data.tar.gz: 9b329b0b2bc47ee1265713e66dbaf84f3872661c
+  metadata.gz: 90086538bf6bc73e5b5a0b27034c72189c1f2221
+  data.tar.gz: 1a58258aef38bbf3f912cb74132c0620a337c1dd
 SHA512:
-  metadata.gz: e1fcd1f0d38a22fab9a7f0375ba943760a9445e51d3eb7a91b67bc73c5e30d69faeaadcf29557d09b381dc160f00c73e1f824bca5ccfa5f72cc8ef435c5bff37
-  data.tar.gz: 7b0f0e51af703b03fd560a51dad5ff9a76c6ab43913d0e6c4f5a0d8ac41ae5fb30824eb1507cc0a9ed61bf11b3005977e7035ed0835a01769596bf4a6e634de9
+  metadata.gz: 98f4470432dfa3579a25f9ce56eb19ae56a10155c31dfc02de1d0eca73ea33950f77502f07f68aa890e280e42e5e6a8b8ddc3dca7b7bfa60b7a1b0d900203018
+  data.tar.gz: 75357647e815519951176b27b8ef9d1bfa58e546ce7a358f7eb6097c508c2333ec5d567c2b62a377517f655f2fda02a8803513fe880f7e98fe1c5b9ac8ca1ba4

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+sudo: false
+cache: bundler
+services:
+  - redis-server
+rvm:
+  - 2.1
+  - 2.0.0
+  - 2.2
+  - jruby
+  - jruby-head
+  - rbx-2
+env:
+  - SIDEKIQ='~> 3.5'
+  - SIDEKIQ='~> 4.0'
+matrix:
+  allow_failures:
+    - rvm: rbx-2
+    - rvm: jruby-head

data/Gemfile

@@ -2,3 +2,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in sidekiq-dejavu.gemspec
 gemspec
+
+gem "sidekiq", ENV['SIDEKIQ'] if ENV['SIDEKIQ']
+gem "minitest"
+gem "minitest-utils"
+gem "redis-namespace"
+gem "pry" unless ENV['CI']

data/README.md

@@ -1,5 +1,8 @@
 # Sidekiq::Dejavu
 
+[![Gem Version](https://badge.fury.io/rb/sidekiq-dejavu.svg)](https://rubygems.org/gems/sidekiq-dejavu)
+[![Build Status](https://travis-ci.org/felixbuenemann/sidekiq-dejavu.svg)](https://travis-ci.org/felixbuenemann/sidekiq-dejavu)
+
 Dejavu is a clockless scheduler that uses Sidekiq's built-in scheduling.
 
 Most scheduling solutions for Sidekiq require either a separate cron like process
@@ -27,6 +30,8 @@ and all re-scheduling themselves. This is because jobs get re-scheduled if there
 schedule with the same name, but only the scheduled jobs queue is inspected.
 Depending on the intervals used it is possible that the job gets scheduled and run again before
 the other jobs can see the scheduled job, so they get re-scheduled as well.
+- It's possible for jobs to be scheduled multiple times, if multiple processes load the same schedule
+at the same time due to missing locks around loading the schedule.
 
 ## Installation
 
@@ -46,25 +51,27 @@ Or install it yourself as:
 
 You can configure schedule tasks in your config/sidekiq.yml:
 
-    :schedule:
-      hello_world:
-        interval: 10
-        class: HelloWorldWorker
-        queue: hello
-        retry: false
-        backtrace: false
-        args:
-          - Hello
-          - World
-      cleanup:
-        interval: '30 5 * * *' # every day at 5:30
-        class: HelloWorldWorker
-        queue: hello
-        retry: false
-        backtrace: false
-        args:
-          - Hello
-          - World
+```yaml
+:schedule:
+  hello_world:
+    interval: 10
+    class: HelloWorldWorker
+    queue: hello
+    retry: false
+    backtrace: false
+    args:
+      - Hello
+      - World
+  cleanup:
+    interval: '30 5 * * *' # every day at 5:30
+    class: HelloWorldWorker
+    queue: hello
+    retry: false
+    backtrace: false
+    args:
+      - Hello
+      - World
+```
 
 Interval can be specified in seconds or as a cron expression, including vixie cron syntax.
 
@@ -72,6 +79,8 @@ If a job fails or takes longer than an interval it will be retried at the next i
 
 You also need to set all options for the job, because currently a workers `sidekiq_options` are ignored. The minium required options for a schedule are `interval` and `class`. The same worker can be scheduled multiple times by using different schedule names (keys) in the config.
 
+Note that by default sidekiq polls scheduled jobs roughly every 15 seconds, which means in order to run jobs at shorter intervals youd'd have to tune `config.average_scheduled_poll_interval` to a lower value.
+
 ## Contributing
 
 1. Fork it ( https://github.com/felixbuenemann/sidekiq-dejavu/fork )

data/Rakefile

@@ -1,2 +1,9 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
 
+Rake::TestTask.new(:test) do |test|
+  # test.warning = true
+  test.pattern = "test/**/test_*.rb"
+end
+
+task :default => :test

data/lib/sidekiq-dejavu.rb

@@ -1,3 +1,4 @@
+require "sidekiq"
 require "sidekiq/dejavu"
 
 Sidekiq.configure_server do |config|

data/lib/sidekiq/dejavu/helper.rb

@@ -3,6 +3,13 @@ require 'parse-cron'
 module Sidekiq
   module Dejavu
     module Helper
+      def valid_cron?(interval)
+        CronParser.new(interval)
+        true
+      rescue ArgumentError
+        false
+      end
+
       def next_timestamp(interval, time = Time.now)
         CronParser.new(interval).next(time).to_f
       rescue ArgumentError
@@ -10,7 +17,8 @@ module Sidekiq
       end
 
       def next_randomized_timestamp(interval, time = Time.now)
-        CronParser.new(interval).next(time).to_f
+        diff = CronParser.new(interval).next(time).to_f - time.to_f
+        time.to_f + Random.rand(diff)
       rescue ArgumentError
         time.to_f + Random.rand(interval.to_f)
       end

data/lib/sidekiq/dejavu/manager.rb

@@ -3,10 +3,11 @@ module Sidekiq
     class Manager
       include Helper
 
-      attr_reader :schedules
+      attr_accessor :schedules, :scheduled_set
 
-      def initialize(schedules)
+      def initialize(schedules = {}, scheduled_set = Sidekiq::ScheduledSet.new)
         @schedules = schedules
+        @scheduled_set = scheduled_set
       end
 
       def reload_schedule!
@@ -14,12 +15,23 @@ module Sidekiq
         add_new_schedules
       end
 
+      def scheduled_jobs
+        scheduled_set.select { |job| job.item.has_key? 'schedule' }
+      end
+
       private
 
       def clear_changed_schedules
         scheduled_jobs.each do |job|
           item = job.item
           name = item['schedule']
+
+          unless schedules.has_key? name
+            Sidekiq.logger.info "Clearing schedule #{name} (not listed in config)."
+            job.delete
+            next
+          end
+
           schedule_options = schedules[name]
           schedule_options['args'] = Array(schedule_options['args'])
           item_options = item.select { |k,v| schedule_options.keys.include? k }
@@ -27,27 +39,25 @@ module Sidekiq
           if item_options != schedule_options
             Sidekiq.logger.info "Clearing schedule #{name} (config changed)."
             job.delete
-          else
-            schedules.delete(name)
           end
         end
       end
 
       def add_new_schedules
+        existing = scheduled_jobs.map { |job| job.item['schedule'] }
+
         schedules.each do |name, options|
+          next if existing.include? name
+
           args = Array(options['args'])
           interval = options['interval']
-          first_run = next_randomized_timestamp(interval)
+          first_run = valid_cron?(interval) ? next_timestamp(interval) : next_randomized_timestamp(interval)
           job = options.merge('args' => args, 'schedule' => name, 'at' => first_run)
 
           Sidekiq.logger.info "Scheduling #{name} for first run at #{Time.at first_run}."
           Sidekiq::Client.push(job)
         end
       end
-
-      def scheduled_jobs
-        Sidekiq::ScheduledSet.new.select { |job| schedules.keys.include? job.item['schedule'] }
-      end
     end
   end
 end

data/lib/sidekiq/dejavu/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Dejavu
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/sidekiq-dejavu.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", ">= 3", "< 5"
+  spec.add_dependency "sidekiq", ">= 3", "< 6"
   spec.add_dependency "parse-cron", "~> 0.1.4"
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"

data/test/fixtures/dummy_worker.rb

@@ -0,0 +1,6 @@
+class DummyWorker
+  include Sidekiq::Worker
+  sidekiq_options :queue => :dejavu_test
+  def perform(*)
+  end
+end

data/test/helper.rb

@@ -0,0 +1,52 @@
+# disable minitest/parallel threads
+ENV["N"] = "0"
+
+require "bundler/setup"
+
+require "sidekiq"
+require "sidekiq/api"
+require "sidekiq/testing"
+
+require "minitest/autorun"
+require "yaml"
+require "pry" unless ENV['CI']
+
+require "sidekiq/dejavu"
+
+require_relative "fixtures/dummy_worker"
+
+require 'sidekiq/redis_connection'
+REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
+REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'dejavu_test')
+
+Sidekiq.configure_client do |config|
+  config.redis = { :url => REDIS_URL, :namespace => 'dejavu_test' }
+end
+
+Sidekiq::Testing.disable!
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add Sidekiq::Dejavu::Middleware::Server::Scheduler
+end
+
+class Sidekiq::Dejavu::Test < MiniTest::Test
+  attr_reader :log, :scheduled_set
+
+  def setup
+    # capture logger
+    @old_logger = Sidekiq.logger
+    @log = StringIO.new
+    Sidekiq.logger = Logger.new(@log)
+    # clear scheduled set
+    @scheduled_set = Sidekiq::ScheduledSet.new
+    @scheduled_set.clear
+  end
+
+  def teardown
+    Sidekiq::Worker.drain_all
+    Sidekiq.logger = @old_logger
+  end
+
+  def assert_logged(regex)
+    assert_match log, regex
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,61 @@
+require_relative "helper"
+
+class TestHelper < Sidekiq::Dejavu::Test
+  include Sidekiq::Dejavu::Helper
+
+  def time
+    @time ||= Time.now
+  end
+
+  def test_valid_cron_with_valid_cron
+    interval = '* * * * *'
+    assert_equal true, valid_cron?(interval)
+  end
+
+  def test_valid_cron_with_interval
+    [10, 10.0, '10', '10.0'].each do |interval|
+      assert_equal false, valid_cron?(interval)
+    end
+  end
+
+  def test_valid_cron_with_garbage
+    interval = 'garbage'
+    assert_equal false, valid_cron?(interval)
+  end
+
+  def test_next_timestamp_cron
+    interval = '* * * * *'
+    expected = time.to_f + (60 - time.to_f % 60)
+    assert_equal expected, next_timestamp(interval, time)
+  end
+
+  def test_next_timestamp_numeric
+    expected = time.to_f + 10.0
+    [10, 10.0, '10', '10.0'].each do |interval|
+      assert_equal expected, next_timestamp(interval, time)
+    end
+  end
+
+  def test_next_randomized_timestamp_cron
+    interval = '* * * * *'
+    expected = time.to_f + (60 - time.to_f % 60)
+    timestamp = next_randomized_timestamp(interval, time)
+    another_timestamp = next_randomized_timestamp(interval, time)
+
+    assert_in_delta expected, timestamp, 60.0
+    refute_equal expected, timestamp
+    refute_equal timestamp, another_timestamp
+  end
+
+  def test_next_randomized_timestamp_numeric
+    expected = time.to_f + 10.0
+    [10, 10.0, '10', '10.0'].each do |interval|
+      timestamp = next_randomized_timestamp(interval, time)
+      another_timestamp = next_randomized_timestamp(interval, time)
+
+      assert_in_delta expected, timestamp, 10.0
+      refute_equal expected, timestamp
+      refute_equal timestamp, another_timestamp
+    end
+  end
+end

data/test/test_manager.rb

@@ -0,0 +1,121 @@
+require_relative "helper"
+
+class TestManager < Sidekiq::Dejavu::Test
+  def schedules
+    @schedules ||= YAML.load(<<-YAML).freeze
+      foo:
+        interval: 10
+        class: DummyWorker
+        args:
+          - fooarg
+      bar:
+        interval: '30 5 * * *'
+        class: DummyWorker
+        args:
+          - bararg
+    YAML
+  end
+
+  def test_reload_schedule!
+    manager = Sidekiq::Dejavu::Manager.new schedules
+
+    assert_equal 0, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 2, scheduled_set.size
+
+    foo = scheduled_set.find { |job| job.item['schedule'] == 'foo' }
+    assert foo
+    assert_equal 10, foo.item['interval']
+    assert_equal 'DummyWorker', foo.item['class']
+    assert_equal %w(fooarg), foo.item['args']
+
+    bar = scheduled_set.find { |job| job.item['schedule'] == 'bar' }
+    assert bar
+    assert_equal '30 5 * * *', bar.item['interval']
+    assert_equal 'DummyWorker', bar.item['class']
+    assert_equal %w(bararg), bar.item['args']
+  end
+
+  def test_reload_schedule_clears_old_schedules
+    old_schedules = YAML.load <<-YAML
+      old:
+        interval: 15
+        class: DummyWorker
+    YAML
+
+    manager = Sidekiq::Dejavu::Manager.new old_schedules
+    assert_equal 0, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 1, scheduled_set.size
+
+    assert scheduled_set.find { |job| job.item['schedule'] == 'old' }
+
+    manager.schedules = {}
+    assert_equal 1, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 0, scheduled_set.size
+
+    refute scheduled_set.find { |job| job.item['schedule'] == 'old' }
+  end
+
+  def test_reload_schedule_updates_existing_schedules
+    old_schedules = YAML.load <<-YAML
+      foo:
+        interval: 15
+        class: DummyWorker
+        args:
+          - oldfooarg
+      bar:
+        interval: '15 4 * * *'
+        class: DummyWorker
+        args:
+          - oldbararg
+    YAML
+
+    manager = Sidekiq::Dejavu::Manager.new old_schedules
+
+    assert_equal 0, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 2, scheduled_set.size
+
+    foo = scheduled_set.find { |job| job.item['schedule'] == 'foo' }
+    assert foo
+    assert_equal 15, foo.item['interval']
+    assert_equal 'DummyWorker', foo.item['class']
+    assert_equal %w(oldfooarg), foo.item['args']
+
+    bar = scheduled_set.find { |job| job.item['schedule'] == 'bar' }
+    assert bar
+    assert_equal '15 4 * * *', bar.item['interval']
+    assert_equal 'DummyWorker', bar.item['class']
+    assert_equal %w(oldbararg), bar.item['args']
+
+    manager.schedules = schedules
+    assert_equal 2, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 2, scheduled_set.size
+
+    foo = scheduled_set.find { |job| job.item['schedule'] == 'foo' }
+    assert foo
+    assert_equal 10, foo.item['interval']
+    assert_equal 'DummyWorker', foo.item['class']
+    assert_equal %w(fooarg), foo.item['args']
+
+    bar = scheduled_set.find { |job| job.item['schedule'] == 'bar' }
+    assert bar
+    assert_equal '30 5 * * *', bar.item['interval']
+    assert_equal 'DummyWorker', bar.item['class']
+    assert_equal %w(bararg), bar.item['args']
+  end
+
+  def test_reload_schedule_keeps_unchanged_schedules
+    manager = Sidekiq::Dejavu::Manager.new schedules
+    assert_equal 0, scheduled_set.size
+    manager.reload_schedule!
+    assert_equal 2, scheduled_set.size
+
+    manager.schedules = schedules
+    manager.reload_schedule!
+    assert_equal 2, scheduled_set.size
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-dejavu
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Felix Buenemann
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-15 00:00:00.000000000 Z
+date: 2017-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -19,7 +19,7 @@ dependencies:
         version: '3'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '3'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: parse-cron
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +81,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -92,6 +93,10 @@ files:
 - lib/sidekiq/dejavu/middleware/server/scheduler.rb
 - lib/sidekiq/dejavu/version.rb
 - sidekiq-dejavu.gemspec
+- test/fixtures/dummy_worker.rb
+- test/helper.rb
+- test/test_helper.rb
+- test/test_manager.rb
 homepage: https://github.com/felixbuenemann/sidekiq-dejavu
 licenses:
 - MIT
@@ -112,8 +117,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Dejavu is a clockless scheduler that uses Sidekiq's built-in scheduling
-test_files: []
+test_files:
+- test/fixtures/dummy_worker.rb
+- test/helper.rb
+- test/test_helper.rb
+- test/test_manager.rb