periodically 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f35124533240ee8deb6be9e68e68886950898319c94b6dbdccd9be8f515bc0d
-  data.tar.gz: 1ff62610c643f3e9e226d5ed7747c5d5297b88d486516f40d57a0381dc777cc1
+  metadata.gz: db828c2104d540e27fda8d4896b3ea91620c6b40034ef7be09678938bd57eba3
+  data.tar.gz: 4dad188a4afe02239af267042e3fea818c7544889dcfa17962a83f7fa00735c9
 SHA512:
-  metadata.gz: 23fef90a350aa3ba3e7786236c1fcd1b89576c79f6757538e0424328c90daa1ffb3d583e1fed8f0d0fec0e09bf1c8b2115168c565ba2ebe6312813d507402f6f
-  data.tar.gz: 9eff6deaafd19e8211bfa7142bf0ae7a99906ff8c30e4155dd6ff0aebda52516c87ebdf2009631f8e0a5d291498a5207d6d3bd787ef167d2e316ff35b37d0194
+  metadata.gz: 30e5029889fc90ff67fd12cc9e2be6eedf438474130d03ba41474998415747230f57a7e0aea424afd5c32274c3d2b89fa1b0eb61a4f2a5ee640e17cb5b30853c
+  data.tar.gz: 010cbdfd28fe74a99a9d4ed7ce3ea1062f14726fd9a92239a7c8f389c7a393bcebdea5d26af83fa289d6bac7a4391506c21ec85c732d0e3ffbcd805d8e163ea7

data/.github/workflows/ci.yml

@@ -0,0 +1,18 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6.x
+      - name: Build and test with Rake
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake test

data/Gemfile

@@ -3,9 +3,11 @@ source 'https://rubygems.org'
 gem 'rake'
 gem 'activerecord'
 gem 'redis-namespace'
+gem 'connection_pool'
 
 group :test do
   gem 'minitest'
+  gem 'mock_redis'
 end
 
 group :development, :test do

data/Gemfile.lock

@@ -14,10 +14,12 @@ GEM
       zeitwerk (~> 2.2)
     ast (2.4.0)
     concurrent-ruby (1.1.5)
+    connection_pool (2.2.2)
     i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     minitest (5.13.0)
+    mock_redis (0.22.0)
     parallel (1.19.1)
     parser (2.7.0.1)
       ast (~> 2.4.0)
@@ -50,7 +52,9 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord
+  connection_pool
   minitest
+  mock_redis
   rake
   redis-namespace
   standard

data/README.md

@@ -1,15 +1,19 @@
 # periodically
 
-Redis-backed Ruby library for periodically running tasks, whose execution depends can depend on a custom lambda block (e.g. weekly syncs).
-The job execution won't be guaranteed to happen exactly as the condition is fulfilled, hence periodically is best for
-nonfrequent and noncritical jobs.
+Redis-backed Ruby library for tasks that need to run once in a while. "Once in a while" is defined more accurately by a custom lambda
+block by the library user.
+
+Since task execution is done in a single threa using non-accurate and non-time-based conditions, periodically is best for
+infrequent and noncritical jobs, such as weekly syncs.
 
 Example usecases:
 
-- Sync a Rails model's data once per week
-  - Achievable by e.g. a `last_synced` value in the database
-- Launch a non-important sync operation depending on a specific condition (e.g. NULL value in some database column)
-  - Achievable by checking the column against NULL inside the `on` condition
+- Sync a Rails model's data once per week from an external source. External source can be unstable, so you'd like to repeat failing jobs but some amount of marginally outdated data won't be a problem
+  - Great fit for periodically! Can be achieved for example by a `last_synced` value in the database and a condition based on it
+  - Additionally, periodically supports deferring jobs (meaning one line rate limiting!)
+- Apply asynchronous corrections to data added to the database. For example automatically fetch title for user-submitted links.
+  - Achievable by adding a condition against the correctable column (`where(title: nil)`)
+  - However, a background job processor like Sidekiq would likely be more efficient
 
 ## Getting started with Rails
 
@@ -21,6 +25,8 @@ Example usecases:
 
 ```rb
 require "periodically"
+
+# Launches a background thread. For production usage you may want to do this in another process
 Periodically.start
 ```
 
@@ -37,12 +43,13 @@ class Item < ApplicationRecord
   include Periodically::Model
 
   periodically :refresh_price,
-                  on: -> { Item.where("last_synced < ?", 7.days.ago) }
+    on: -> { Item.where("last_synced < ?", 7.days.ago) }
 
   private
 
   def refresh_price
     self.price = PriceFetcher.fetch(item_id)
+    self.last_synced = Time.now! # Remember to update the condition by yourself!
     save!
   end
 end
@@ -74,7 +81,7 @@ include Periodically::Model
 periodically :update_method, # call instance method "update_method" for found instances
   on: -> { where("last_synced < ?", 7.days.ago) }, # Condition that must be true for update method to be called
   min_class_interval: 5.minutes, # (Optional) The minimum interval between calls to this specific class (TODO not implemented)
-  max_retries: 25, # (Optional) Maximum number of retries. Periodically uses exponential backoff
+  max_retries: 25, # (Optional) Maximum number of retries. Periodically uses exponential backoff (TODO not implemented)
   instance_id: -> { cache_key_with_version }, # (Optional) Returns this instance's unique identifying key. Used for e.g. deferring jobs and marking them as erroring (TODO not implemented)
 
 
@@ -126,29 +133,28 @@ Just call them by yourself in your tests.
 
 **The periodically :on condition**
 
-(TODO this does not exist yet)
-
-You can call `Periodically.would_execute?(MyModel, :object)` to statically check the condition.
+You can call `Periodically.would_execute?(MyModel, :object)` to statically check the condition. (TODO not implemented)
 
 ## Debugging
 
-As part of your application you might want to be able to get a quick snapshot of how Periodically is doing.
+As part of your application you might want to be able to get a quick snapshot of how periodically is doing.
 You can do that by calling `Periodically::Debug.total_debug_dump`, which returns a hash containing bunch of debug information.
 
 ## Dashboard
 
-(When implemented :D) Dashboard contains recently succeeded executions, failed executions (with stacktrace) and deferred executions.
+Dashboard contains recently succeeded executions, failed executions (with stacktrace) and deferred executions. (TODO not implemented)
 
 ## Why not Sidekiq?
 
-With Sidekiq you can achieve something almost similar by combining a scheduled job that enqueues further unique jobs based on the condition.
+With Sidekiq you can achieve something almost similar by combining a scheduled job that enqueues further unique jobs based on the condition. (see https://github.com/mperham/sidekiq/wiki/Ent-Periodic-Jobs#dynamic-jobs)
 
-However, there are few advantages Periodically has for the specific usecase of per-instance non-critical jobs:
+However, there are few advantages periodically has for the specific usecase of per-instance non-critical jobs:
 
-- **Improved backpressure handling.** Due to knowing the conditions, we are able to track the number of pending jobs at all times. This enables early warnings for the developers in case of job buildup. (TODO)
-- **Better observability.** We know exactly how many items fulfill the condition and how many don't; therefore, we can visualize success rates and current status as a percentage of the total. (TODO)
-- **Cleaner per-instance retrying.** If we start executing a job, but suddenly want to defer execution by some time in Sidekiq, it is definitely doable with scheduled jobs. However, this may entrap you in a "scheduled unique job" hell: if some job keeps getting mistakenly deferred, it might be hard to find out about this behavior without some complex job tracking logic. In contrast, Periodically delivers this functionality for free due to more explicit control over job scheduling and rescheduling. (TODO)
-- **More clever polling.** Since we know the exact condition for new periodic jobs, we can deduce the next execution time and sleep accordingly. (TODO)
-- **Easier priority escalation.** Periodically selects jobs in order of the given condition and maintains no queue of its own; therefore it is trivial to prioritize certain jobs by adding a new query condition.
+- **Improved backpressure handling.** Since we know the conditions for executing jobs, we are in better control of the job producer and able to balance between different jobs. This enables for instance early warnings for the developers in case of job buildup. (TODO not implemented)
+- **Closer to the source.** Periodic callbacks are defined inside the models, so it is always easy to find which jobs are affecting which models.
+- **Cleaner per-instance retrying.** If we start executing a job, but suddenly want to defer execution by some time in Sidekiq, it is definitely doable with scheduled jobs. However, this may entrap you in a "scheduled unique job" hell: if some job keeps getting mistakenly deferred, it might be hard to find out about the repeated erroneous behavior without some complex job tracking logic. In contrast, periodically delivers this functionality for free due to more explicit control over job scheduling and rescheduling. (TODO not implemented)
+- **More flexible delaying.** In periodically, you can delay job executions per-instance, per-job or per-class.
+- **More clever polling.** Since we know the exact condition for new periodic jobs, we can deduce the next execution time and sleep accordingly. (TODO not implemented)
+- **Easier priority escalation.** Periodically selects jobs in order of the given condition and maintains no queue of its own; therefore prioritizing certain jobs by adding a new query condition comes by design.
 
-Importantly, Sidekiq and Periodically aim to solve different problems. Nothing prevents one from using both at the same time.
+Importantly, Sidekiq and periodically aim to solve different problems. Nothing prevents one from using both at the same time.

data/lib/periodically.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "logger"
+
 require "periodically/job"
 require "periodically/debug"
 require "periodically/defer"
@@ -9,8 +11,11 @@ require "periodically/model"
 module Periodically
   @@registered = []
 
+  def self.logger=(new_logger)
+    @logger ||= new_logger
+  end
   def self.logger
-    Rails.logger # TODO
+    @logger ||= Logger.new(STDOUT)
   end
 
   REDIS_DEFAULTS = {
@@ -18,13 +23,13 @@ module Periodically
   }
 
   def self.execute_next
-    job, instance = @@registered.lazy.filter_map do |job|
+    found_job, instance = @@registered.lazy.filter_map do |job|
       instance = job.poll_next_instance
       [job, instance] if instance
     end.first
 
-    if job && instance
-      job.execute_instance(instance)
+    if found_job && instance
+      found_job.execute_instance(instance)
       true
     else
       false

data/lib/periodically/debug.rb

@@ -8,17 +8,32 @@ module Periodically
         values = conn.mget(keys)
         Hash[keys.zip(values)]
       end
-      locks = Periodically.redis do |conn|
+      error_messages = Periodically.redis do |conn|
+        keys = conn.scan_each(:match => "errormessages:*").to_a
+        values = conn.mget(keys)
+        Hash[keys.zip(values)]
+      end
+      locks = lock_ttls
+      {
+        job_count: Periodically._registered_jobs.size,
+        error_counts: error_counts,
+        error_messages: error_messages,
+        lock_ttls: locks
+      }
+    end
+
+    def self.lock_ttls
+      Periodically.redis do |conn|
         keys = conn.scan_each(:match => "locks:*").to_a
         ttls = conn.eval(%{
           local matcher = KEYS[1] .. ":*"
           local ttls = {}
-
+  
           local cursor = "0"
           repeat
             local result = redis.call("SCAN", cursor, "MATCH", matcher)
             cursor = result[1]
-
+  
             local keys   = result[2]
             for i, key in ipairs(keys) do
               ttls[#ttls + 1] = redis.call('ttl', key)
@@ -28,7 +43,6 @@ module Periodically
         }, :keys => ['locks'])
         Hash[keys.zip(ttls)]
       end
-      { job_count: Periodically._registered_jobs.size, error_counts: error_counts, lock_ttls: locks }
     end
   end
 end

data/lib/periodically/job.rb

@@ -15,11 +15,9 @@ module Periodically
     def poll_next_instance
       return if job_or_class_locked?
 
-      ActiveRecord::Base.uncached do
-        where = @opts[:on].call()
-        where.to_a.find do |obj|
-          !instance_locked?(obj)
-        end
+      where = @opts[:on].call()
+      where.to_a.find do |obj|
+        !instance_locked?(obj)
       end
     end
 
@@ -28,9 +26,11 @@ module Periodically
         begin
           instance.send(@method)
         rescue => e
-          Periodically.logger.error("Job instance[#{instance}] execution raised an exception\n#{e.message}\n#{e.backtrace.join("\n")}")
+          stored_error = "#{e.message}\n#{e.backtrace.join("\n")}"
+          Periodically.logger.error("Job instance[#{instance}] execution raised an exception\n#{stored_error}")
           new_error_count = increase_instance_error_count(instance)
           lock_instance(instance, DEFAULT_ERROR_DELAY.call(new_error_count))
+          store_instance_error(instance, stored_error)
           return
         end
 
@@ -61,6 +61,11 @@ module Periodically
       Periodically.redis {|conn| conn.del(error_count_key)}
     end
 
+    def store_instance_error(instance, error)
+      error_message_key = "errormessages:#{instance_key(instance)}"
+      Periodically.redis {|conn| conn.set(error_message_key, error)}
+    end
+
     def job_or_class_locked?
       Periodically.redis do |conn|
         # TODO can this be optimized with exists?(Array)

data/lib/periodically/redis.rb

@@ -35,7 +35,8 @@ module Periodically
       def build_client(options)
         namespace = options[:namespace]
 
-        client = Redis.new client_opts(options)
+        client = $PERIODICALLY_MOCK_REDIS ? MockRedis.new : Redis.new(client_opts(options))
+
         if namespace
           begin
             require "redis-namespace"

data/periodically.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
 
   gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
   gem.name = "periodically"
-  gem.version = "0.0.5"
+  gem.version = "0.0.6"
   gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "redis", ">= 4.1.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: periodically
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - wyozi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -59,6 +59,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/ci.yml"
 - Gemfile
 - Gemfile.lock
 - README.md