periodically 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 39405716f55a3c85a4674d38d0440ba293d9bea7e44a260e062b2e631dc230b8
+  data.tar.gz: f51dbc7e4d9a033778107700e6c82c8743374e9705e8526f84e7900ac56e6cdc
+SHA512:
+  metadata.gz: f65b70c130c7dffd6cb01da0c02992a4ff4bec0bf763a46ed996057a047709871aabd07f0efb5fa79708a40e5c2d4c33da263c4fdcb0726644af8ad3494517c0
+  data.tar.gz: fe4a38609d1b958f748cbeaae53f571ce0d005318599d49d96a6ae0d788db1bde0c96a05b32fb9908c33864b7529aa4930d09d2806d843b1948cd6cad81026ee

data/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.rb,*.gemspec]
+indent_style = space
+indent_size = 2

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'activerecord'
+gem 'redis-namespace'
+gem 'rufus-scheduler'

data/README.md

@@ -0,0 +1,125 @@
+# periodically
+
+Redis-backed Ruby library for periodically running tasks, whose execution depends can depend on a custom lambda block (e.g. weekly syncs).
+The job execution won't be guaranteed to happen exactly as the condition is fulfilled, hence periodically is best for
+nonfrequent and noncritical jobs.
+
+Example usecases:
+
+- Sync a Rails model's data once per week
+  - Achievable by e.g. a `last_synced` value in the database
+- Launch a non-important sync operation depending on a specific condition (e.g. NULL value in some database column)
+  - Achievable by checking the column against NULL inside the `on` condition
+
+### Example usage with Rails
+
+```rb
+# config/initializers/periodically.rb
+require "periodically"
+
+# Note: in development mode classes are loaded lazily, so periodical jobs only start once classes have been loaded
+# In production classes are loaded eagerly, so this is no problem
+
+Periodically.start
+```
+
+```rb
+# app/models/item.rb
+
+class Item < ApplicationRecord
+  include Periodically::Model
+
+  periodically :refresh_price,
+                  on: -> { Item.where("last_synced < ?", 7.days.ago) }
+
+  private
+
+  def refresh_price
+    self.price = PriceFetcher.fetch(item_id)
+    save!
+  end
+end
+```
+
+### Example usage with pure Ruby
+
+```rb
+# TODO
+```
+
+## Execution model
+
+Periodically launches a single background thread, which executes registered queries every x seconds. If a pending query is found, the registered callback method is called in the same thread. Hence, a blocking callback method will also block execution of other pending queries.
+
+By default everything happens in the same process as the main Rails web server.
+To parallelize processing a bit, you can do `bundle exec periodically` to start a new process for jobs. (Remember to remove `Periodically.start` from initializer!)
+
+## API
+
+### Terminology
+
+- **Job**: something enqueued to be called using the `periodically` method
+- **Instance job**: a single Job execution concerning a specific instance of the class
+
+### Inside a Model
+
+#### Definitions
+
+```rb
+# Add Periodically context to this class
+include Periodically::Model
+
+# Enqueue a Periodically job
+periodically :update_method, # call instance method "update_method" for found instances
+  on: -> { where("last_synced < ?", 7.days.ago) }, # Condition that must be true for update method to be called
+  min_class_interval: 5.minutes, # (Optional) The minimum interval between calls to this specific class (TODO not implemented)
+  max_retries: 25, # (Optional) Maximum number of retries. Periodically uses exponential backoff
+  instance_id: -> { cache_key_with_version }, # (Optional) Returns this instance's unique identifying key. Used for e.g. deferring jobs and marking them as erroring (TODO not implemented)
+
+
+```
+
+#### Update method return values
+
+Job method's return value or raised exception determines further executions of that specific instance job.
+
+```rb
+# As referred to by a previous `periodically` call
+def update_method
+  # Let's retrieve a normal value from the model instance
+  status = my_column_status
+
+  # No-op
+  #   Since we don't update `last_synced`, this method will get called again without much delay!
+  return if status == "pending"
+
+  # Log error and defer execution
+  #   This unique instance will be deferred for later execution (using exponential backoff) and the error logged
+  raise "something went wrong" if status == "error"
+
+  # Update checked delay
+  #   Updates the property we check against, thus making this instance not pass the Periodically condition
+  #   Note that this line is normal Rails code: Periodically conditions are database/anything-agnostic
+  update(last_synced: Time.now)
+end
+```
+
+The job method's return value can be used to defer further execution of the same model instance, even if it still passes the condition: `return Periodically::Defer(60.minutes)`
+
+## Dashboard
+
+(When implemented :D) Dashboard contains recently succeeded executions, failed executions (with stacktrace) and deferred executions.
+
+## Why not Sidekiq?
+
+With Sidekiq you can achieve something almost similar by combining a scheduled job that enqueues further unique jobs based on the condition.
+
+However, there are few advantages Periodically has for the specific usecase of per-instance non-critical jobs:
+
+- **Improved backpressure handling.** Due to knowing the conditions, we are able to track the number of pending jobs at all times. This enables early warnings for the developers in case of job buildup. (TODO)
+- **Better observability.** We know exactly how many items fulfill the condition and how many don't; therefore, we can visualize success rates and current status as a percentage of the total. (TODO)
+- **Cleaner per-instance retrying.** If we start executing a job, but suddenly want to defer execution by some time in Sidekiq, it is definitely doable with scheduled jobs. However, this may entrap you in a "scheduled unique job" hell: if some job keeps getting mistakenly deferred, it might be hard to find out about this behavior without some complex job tracking logic. In contrast, Periodically delivers this functionality for free due to more explicit control over job scheduling and rescheduling. (TODO)
+- **More clever polling.** Since we know the exact condition for new periodic jobs, we can deduce the next execution time and sleep accordingly. (TODO)
+- **Easier priority escalation.** Periodically selects jobs in order of the given condition and maintains no queue of its own; therefore it is trivial to prioritize certain jobs by adding a new query condition.
+
+Importantly, Sidekiq and Periodically aim to solve different problems. Nothing prevents one from using both at the same time.

data/lib/periodically.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "periodically/job"
+require "periodically/redis"
+require "periodically/model" if defined?(::Rails::Engine)
+
+module Periodically
+  @@registered = []
+
+  def self.logger
+    Rails.logger # TODO
+  end
+
+  REDIS_DEFAULTS = {
+    namespace: "periodically"
+  }
+
+  def self.execute_next
+    job, instance = @@registered.lazy.filter_map do |job|
+      instance = job.poll_next_instance
+      [job, instance] if instance
+    end.first
+
+    job.execute_instance(instance) if job && instance
+  end
+
+  def self.register(klass, method, opts)
+    @@registered.push(Periodically::Job.new(klass, method, opts))
+  end
+  
+  def self.redis_pool
+    @redis ||= Periodically::RedisConnection.create(REDIS_DEFAULTS)
+  end
+  
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      yield conn
+    end
+  end
+
+  def self.start
+    require 'rufus-scheduler'
+    scheduler = Rufus::Scheduler.new
+
+    Periodically.redis { |conn| conn.ping }
+    
+    scheduler.interval '10s' do
+      Periodically.execute_next
+    end
+  end
+end

data/lib/periodically/cli.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This file handles running Periodically in another process
+
+module Periodically
+  class CLI
+    def run
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+
+      Periodically.start
+    end
+  end
+end

data/lib/periodically/condition.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Periodically
+  module Condition
+    def self.class()
+
+    end
+    
+    def self.evaluate(func)
+      condition = func.call()
+      
+      simple = simple_evaluation(condition)
+  
+      # Filter condition to skip 
+      if evaluated.is_a?(ActiveRecord::Base)
+  
+      end
+    end
+
+    private
+
+    def self.simple_evaluation(condition)
+      row = evaluated.first()
+      row if !Periodically.redis {|conn| conn.exists("")}
+    end
+
+  end
+end

data/lib/periodically/job.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Periodically
+  DEFAULT_ERROR_DELAY = proc { |count| (count ** 4) + 15 + (rand(30)*(count+1)) }
+
+  class Job
+    def initialize(klass, method, opts)
+      @klass = klass
+      @method = method
+      @opts = opts
+    end
+
+    def poll_next_instance
+      ActiveRecord::Base.uncached do
+        where = @opts[:on].call()
+        where.to_a.find do |obj|
+          !instance_locked?(obj)
+        end
+      end
+    end
+
+    def execute_instance(instance)
+      return_value =
+        begin
+          instance.send(@method)
+        rescue => e
+          Periodically.logger.error("Job instance[#{instance}] execution raised an error: #{e}")
+
+          new_error_count = increase_instance_error_count(instance)
+          lock_instance(instance, DEFAULT_ERROR_DELAY.call(new_error_count))
+          return
+        end
+
+      clear_instance_error_count(instance)
+    end
+
+    private
+
+    def instance_key(instance)
+      instance.cache_key_with_version
+    end
+
+    def instance_locked?(instance)
+      Periodically.redis {|conn| conn.exists("locks:#{instance_key(instance)}")}
+    end
+
+    def increase_instance_error_count(instance)
+      error_count_key = "errors:#{instance_key(instance)}"
+      Periodically.redis {|conn| conn.incr(error_count_key)}
+    end
+
+    def clear_instance_error_count(instance)
+      error_count_key = "errors:#{instance_key(instance)}"
+      Periodically.redis {|conn| conn.del(error_count_key)}
+    end
+
+    def lock_instance(instance, seconds)
+      lock_key = "locks:#{instance_key(instance)}"
+
+      Periodically.redis do |conn|
+        conn.multi do |multi|
+          multi.set(lock_key, "1")
+          multi.expire(lock_key, seconds)
+        end
+      end
+    end
+  end
+end

data/lib/periodically/model.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Periodically
+  module Model
+    extend ActiveSupport::Concern
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def periodically(symbol, **opts)
+        Periodically.register(self, symbol, opts)
+      end
+    end
+  end
+end

data/lib/periodically/redis.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "connection_pool"
+require "redis"
+require "uri"
+
+module Periodically
+  class RedisConnection
+    class << self
+      def create(options = {})
+        options.keys.each do |key|
+          options[key.to_sym] = options.delete(key)
+        end
+
+        options[:id] = "Periodically-PID-#{::Process.pid}" unless options.key?(:id)
+        options[:url] ||= determine_redis_provider
+
+        size = if options[:size]
+          options[:size]
+        elsif ENV["RAILS_MAX_THREADS"]
+          Integer(ENV["RAILS_MAX_THREADS"])
+        else
+          5
+        end
+
+        pool_timeout = options[:pool_timeout] || 1
+
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
+          build_client(options)
+        end
+      end
+
+      private
+
+      def build_client(options)
+        namespace = options[:namespace]
+
+        client = Redis.new client_opts(options)
+        if namespace
+          begin
+            require "redis-namespace"
+            Redis::Namespace.new(namespace, redis: client)
+          rescue LoadError
+            Periodically.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                         "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+            exit(-127)
+          end
+        else
+          client
+        end
+      end
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Periodically::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+
+      def determine_redis_provider
+        # If you have this in your environment:
+        # MY_REDIS_URL=redis://hostname.example.com:1238/4
+        # then set:
+        # REDIS_PROVIDER=MY_REDIS_URL
+        # and Periodically will find your custom URL variable with no custom
+        # initialization code at all.
+        #
+        p = ENV["REDIS_PROVIDER"]
+        if p && p =~ /\:/
+          raise <<~EOM
+            REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
+            Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Periodically with REDIS_PROVIDER, e.g.:
+
+            REDISTOGO_URL=redis://somehost.example.com:6379/4
+            REDIS_PROVIDER=REDISTOGO_URL
+          EOM
+        end
+
+        ENV[
+          p || "REDIS_URL"
+        ]
+      end
+    end
+  end
+end

data/periodically.gemspec

@@ -0,0 +1,13 @@
+Gem::Specification.new do |gem|
+  gem.authors = ["wyozi"]
+  gem.summary = "Periodic background jobs for Ruby"
+
+  gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.name = "periodically"
+  gem.version = "0.0.1"
+  gem.required_ruby_version = ">= 2.5.0"
+
+  gem.add_dependency "redis", ">= 4.1.0"
+  gem.add_dependency "redis-namespace", ">= 1.7"
+  gem.add_dependency "connection_pool", ">= 2.2.2"
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: periodically
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- wyozi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- Gemfile
+- README.md
+- lib/periodically.rb
+- lib/periodically/cli.rb
+- lib/periodically/condition.rb
+- lib/periodically/job.rb
+- lib/periodically/model.rb
+- lib/periodically/redis.rb
+- periodically.gemspec
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Periodic background jobs for Ruby
+test_files: []