periodically 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6b138f5577f4135c5cc3b9114624be617202245cbb66419cea0d2fe9a65e32c
-  data.tar.gz: bc2d6f5942c6188aa6054975613794481605c49bc58409e2437a474c8e727a3b
+  metadata.gz: 1f35124533240ee8deb6be9e68e68886950898319c94b6dbdccd9be8f515bc0d
+  data.tar.gz: 1ff62610c643f3e9e226d5ed7747c5d5297b88d486516f40d57a0381dc777cc1
 SHA512:
-  metadata.gz: 8ea4cd75750d13a4e9efa42f28ad8c2a680d5ec43ce2b1b03451d1e2ecf0a7a020c81dd8ca74725d1792db74ef7269456072db9ea253675fc64602aa770f1769
-  data.tar.gz: e7ccc2c21b2be69296a826bb6106564d8412e528d533ba53483fba17b23a53e260fea865c7abd6226ceddf04c2390cad364563b8d498edbf8639b99b91a3e52b
+  metadata.gz: 23fef90a350aa3ba3e7786236c1fcd1b89576c79f6757538e0424328c90daa1ffb3d583e1fed8f0d0fec0e09bf1c8b2115168c565ba2ebe6312813d507402f6f
+  data.tar.gz: 9eff6deaafd19e8211bfa7142bf0ae7a99906ff8c30e4155dd6ff0aebda52516c87ebdf2009631f8e0a5d291498a5207d6d3bd787ef167d2e316ff35b37d0194

data/.editorconfig

@@ -4,6 +4,6 @@ root = true
 end_of_line = lf
 insert_final_newline = true
 
-[*.{rb,gemspec}}]
+[*.{rb,gemspec}]
 indent_style = space
 indent_size = 2

data/Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 gem 'rake'
 gem 'activerecord'
 gem 'redis-namespace'
-gem 'rufus-scheduler'
 
 group :test do
   gem 'minitest'

data/Gemfile.lock

@@ -14,11 +14,6 @@ GEM
       zeitwerk (~> 2.2)
     ast (2.4.0)
     concurrent-ruby (1.1.5)
-    et-orbi (1.2.2)
-      tzinfo
-    fugit (1.3.3)
-      et-orbi (~> 1.1, >= 1.1.8)
-      raabro (~> 1.1)
     i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
@@ -26,7 +21,6 @@ GEM
     parallel (1.19.1)
     parser (2.7.0.1)
       ast (~> 2.4.0)
-    raabro (1.1.6)
     rainbow (3.0.0)
     rake (13.0.1)
     redis (4.1.3)
@@ -42,8 +36,6 @@ GEM
     rubocop-performance (1.5.2)
       rubocop (>= 0.71.0)
     ruby-progressbar (1.10.1)
-    rufus-scheduler (3.6.0)
-      fugit (~> 1.1, >= 1.1.6)
     standard (0.1.7)
       rubocop (~> 0.77.0)
       rubocop-performance (~> 1.5.1)
@@ -61,7 +53,6 @@ DEPENDENCIES
   minitest
   rake
   redis-namespace
-  rufus-scheduler
   standard
 
 BUNDLED WITH

data/README.md

@@ -95,9 +95,14 @@ def update_method
   return if status == "pending"
 
   # Log error and defer execution
-  #   This unique instance will be deferred for later execution (using exponential backoff) and the error logged
+  #   This unique instance will be deferred for later execution (using exponential backoff) and the error is logged
   raise "something went wrong" if status == "error"
 
+  # Defer any further calls to :update_method (on any instance)
+  #   This is perfect for short-time rate limiting, but highly discouraged as a method of timing updates!
+  #   You should use database columns (e.g. last_synced like this method) instead, which causes less stress on Redis
+  return Periodically::Defer.job_by(60.minutes) if status == "rate_limited"
+
   # Update checked delay
   #   Updates the property we check against, thus making this instance not pass the Periodically condition
   #   Note that this line is normal Rails code: Periodically conditions are database/anything-agnostic
@@ -105,7 +110,30 @@ def update_method
 end
 ```
 
-The job method's return value can be used to defer further execution of the same model instance, even if it still passes the condition: `return Periodically::Defer(60.minutes)`
+The job method's return value can be used to defer further execution of either the model instance, the specific job within the model or any job within the model:
+
+- `Periodically::Defer.instance_by(60.minutes) # calling :update_method on this instance will be delayed`
+- `Periodically::Defer.job_by(60.minutes) # calling :update_method on any instance of this class will be delayed`
+- `Periodically::Defer.class_by(60.minutes) # calling anything on any instance of this class will be delayed`
+
+## Testing
+
+What do you want to test?
+
+**Behavior of my update callbacks**
+
+Just call them by yourself in your tests.
+
+**The periodically :on condition**
+
+(TODO this does not exist yet)
+
+You can call `Periodically.would_execute?(MyModel, :object)` to statically check the condition.
+
+## Debugging
+
+As part of your application you might want to be able to get a quick snapshot of how Periodically is doing.
+You can do that by calling `Periodically::Debug.total_debug_dump`, which returns a hash containing bunch of debug information.
 
 ## Dashboard
 

data/lib/periodically/debug.rb

@@ -9,9 +9,26 @@ module Periodically
         Hash[keys.zip(values)]
       end
       locks = Periodically.redis do |conn|
-        conn.scan_each(:match => "locks:*").to_a
+        keys = conn.scan_each(:match => "locks:*").to_a
+        ttls = conn.eval(%{
+          local matcher = KEYS[1] .. ":*"
+          local ttls = {}
+
+          local cursor = "0"
+          repeat
+            local result = redis.call("SCAN", cursor, "MATCH", matcher)
+            cursor = result[1]
+
+            local keys   = result[2]
+            for i, key in ipairs(keys) do
+              ttls[#ttls + 1] = redis.call('ttl', key)
+            end
+          until cursor == "0"
+          return ttls
+        }, :keys => ['locks'])
+        Hash[keys.zip(ttls)]
       end
-      { job_count: Periodically._registered_jobs.size, error_counts: error_counts, locks: locks }
+      { job_count: Periodically._registered_jobs.size, error_counts: error_counts, lock_ttls: locks }
     end
   end
 end

data/lib/periodically/defer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Periodically
+  module Defer
+    def self.instance_by(duration)
+      DeferInstance.new(duration)
+    end
+    def self.job_by(duration)
+      DeferJob.new(duration)
+    end
+    def self.class_by(duration)
+      DeferClass.new(duration)
+    end
+
+    class DeferInstance
+      attr_reader :duration
+      def initialize(duration)
+        @duration = duration
+      end
+    end
+    class DeferJob
+      attr_reader :duration
+      def initialize(duration)
+        @duration = duration
+      end
+    end
+    class DeferClass
+      attr_reader :duration
+      def initialize(duration)
+        @duration = duration
+      end
+    end
+  end
+end

data/lib/periodically/job.rb

@@ -7,11 +7,14 @@ module Periodically
     def initialize(klass, method, opts)
       @klass = klass
       @method = method
-      @job_key = "#{klass.name}/#{method.to_s}"
+      @class_key = klass.name
+      @job_key = "#{@class_key}/#{method.to_s}"
       @opts = opts
     end
 
     def poll_next_instance
+      return if job_or_class_locked?
+
       ActiveRecord::Base.uncached do
         where = @opts[:on].call()
         where.to_a.find do |obj|
@@ -25,43 +28,59 @@ module Periodically
         begin
           instance.send(@method)
         rescue => e
-          Periodically.logger.error("Job instance[#{instance}] execution raised an error: #{e}")
-
+          Periodically.logger.error("Job instance[#{instance}] execution raised an exception\n#{e.message}\n#{e.backtrace.join("\n")}")
           new_error_count = increase_instance_error_count(instance)
           lock_instance(instance, DEFAULT_ERROR_DELAY.call(new_error_count))
           return
         end
 
+      if return_value.is_a?(Periodically::Defer::DeferInstance)
+        lock_instance(instance, return_value.duration)
+      elsif return_value.is_a?(Periodically::Defer::DeferJob)
+        Job.lock_key("locks:#{@job_key}", return_value.duration)
+      elsif return_value.is_a?(Periodically::Defer::DeferClass)
+        Job.lock_key("locks:#{@class_key}", return_value.duration)
+      end
+
       clear_instance_error_count(instance)
     end
 
     private
 
-    def full_instance_key(instance)
+    def instance_key(instance)
       "#{@job_key}/#{instance.id}"
     end
 
     def increase_instance_error_count(instance)
-      error_count_key = "errors:#{full_instance_key(instance)}"
+      error_count_key = "errors:#{instance_key(instance)}"
       Periodically.redis {|conn| conn.incr(error_count_key)}
     end
 
     def clear_instance_error_count(instance)
-      error_count_key = "errors:#{full_instance_key(instance)}"
+      error_count_key = "errors:#{instance_key(instance)}"
       Periodically.redis {|conn| conn.del(error_count_key)}
     end
 
+    def job_or_class_locked?
+      Periodically.redis do |conn|
+        # TODO can this be optimized with exists?(Array)
+        conn.exists("locks:#{@job_key}") || conn.exists("locks:#{@class_key}")
+      end
+    end
+
     def instance_locked?(instance)
-      Periodically.redis {|conn| conn.exists("locks:#{full_instance_key(instance)}")}
+      Periodically.redis {|conn| conn.exists("locks:#{instance_key(instance)}")}
     end
 
     def lock_instance(instance, seconds)
-      lock_key = "locks:#{full_instance_key(instance)}"
+      Job.lock_key("locks:#{instance_key(instance)}", seconds)
+    end
 
+    def self.lock_key(key, seconds)
       Periodically.redis do |conn|
         conn.multi do |multi|
-          multi.set(lock_key, "1")
-          multi.expire(lock_key, seconds)
+          multi.set(key, "1")
+          multi.expire(key, seconds)
         end
       end
     end

data/lib/periodically.rb

@@ -2,6 +2,7 @@
 
 require "periodically/job"
 require "periodically/debug"
+require "periodically/defer"
 require "periodically/redis"
 require "periodically/model"
 
@@ -22,7 +23,12 @@ module Periodically
       [job, instance] if instance
     end.first
 
-    job.execute_instance(instance) if job && instance
+    if job && instance
+      job.execute_instance(instance)
+      true
+    else
+      false
+    end
   end
 
   def self.register(klass, method, opts)
@@ -45,13 +51,17 @@ module Periodically
   end
 
   def self.start
-    require "rufus-scheduler"
-    scheduler = Rufus::Scheduler.new
-
     Periodically.redis { |conn| conn.ping }
     
-    scheduler.interval "10s" do
-      Periodically.execute_next
+    Thread.new do
+      while true
+        executed = Periodically.execute_next
+        if executed
+          sleep 1.seconds
+        else
+          sleep 10.seconds
+        end
+      end
     end
   end
 end

data/periodically.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
 
   gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
   gem.name = "periodically"
-  gem.version = "0.0.4"
+  gem.version = "0.0.5"
   gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "redis", ">= 4.1.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: periodically
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - wyozi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-06 00:00:00.000000000 Z
+date: 2020-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -66,6 +66,7 @@ files:
 - lib/periodically.rb
 - lib/periodically/cli.rb
 - lib/periodically/debug.rb
+- lib/periodically/defer.rb
 - lib/periodically/job.rb
 - lib/periodically/model.rb
 - lib/periodically/redis.rb