process_balancer 1.0.0

data/lib/process_balancer/lock/advisory_lock.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ProcessBalancer
+  module Lock
+    # This is lock implementation using advisory locks on the database via the with_advisory_lock gem
+    module AdvisoryLock
+      # class to wrap the lock handling and provide the "extend!" method contract
+      class DummyLock
+        def extend!; end
+      end
+
+      def worker_lock
+        key = "worker_lock_#{job_id}_#{worker_index}"
+        lock = DummyLock.new
+        ActiveRecord::Base.with_advisory_lock(key) do
+          yield lock
+        end
+      end
+    end
+  end
+end

data/lib/process_balancer/lock/simple_redis.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module ProcessBalancer
+  module Lock
+    # This is a simple implementation of a lock to ensure only one job runner is running for a worker
+    # This is only save for a single redis instance setup
+    # something more resilient should be used instead,
+    # e.g. and advisory lock in a DB or using RedLock ( https://github.com/leandromoreira/redlock-rb )
+    module SimpleRedis
+      def self.time_source
+        @time_source ||= if defined?(Process::CLOCK_MONOTONIC)
+                           proc { (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i }
+                         else
+                           proc { (Time.now.to_f * 1000).to_i }
+                         end
+      end
+
+      # class to wrap the lock handling and provide the "extend!" method contract
+      class LockHandler
+        def initialize(key, value, ttl)
+          @key      = key
+          @value    = value
+          @ttl      = ttl
+          @acquired = false
+        end
+
+        def acquire!
+          time_source = ProcessBalancer::Lock::SimpleRedis.time_source
+
+          timeout_ms = 5000
+          wait_time  = 0.02..0.1
+          start      = time_source.call
+
+          sleep(rand(wait_time)) while !(@acquired = try_lock) && (time_source.call - start) < timeout_ms
+        end
+
+        def release!
+          ProcessBalancer.redis do |c|
+            c.del(@key)
+          end
+        end
+
+        def acquired?
+          @acquired
+        end
+
+        def try_lock
+          ProcessBalancer.redis do |c|
+            c.set(@key, @value, nx: true, ex: @ttl)
+          end
+        end
+
+        def extend!
+          ProcessBalancer.redis do |c|
+            c.watch(@key)
+            if c.get(@key) == @value
+              c.multi do
+                c.set(@key, @value, ex: @ttl)
+              end
+            end
+          end
+        end
+      end
+
+      def worker_lock
+        lock = LockHandler.new("lock_#{job_id}_#{worker_index}", ProcessBalancer.identity, runtime_lock_timeout)
+        lock.acquire!
+
+        if lock.acquired?
+          begin
+            yield lock
+          ensure
+            lock.release!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/process_balancer/manager.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'json'
+
+require_relative 'util'
+require_relative 'watcher'
+
+require 'concurrent/atomic/atomic_fixnum'
+require 'concurrent/executor/thread_pool_executor'
+
+module ProcessBalancer
+  class Manager # :nodoc:
+    include Util
+
+    def initialize(options)
+      @options       = options
+      @done          = false
+      @process_index = Concurrent::AtomicFixnum.new(-1)
+      @process_count = Concurrent::AtomicFixnum.new(0)
+      @pool          = Concurrent::ThreadPoolExecutor.new(max_threads: @options[:max_threads], fallback_policy: :discard)
+
+      setup_job_watchers
+    end
+
+    def process_count
+      @process_count.value
+    end
+
+    def process_index
+      v = @process_index.value
+      v == -1 ? nil : v
+    end
+
+    def workers_for_job(job_id)
+      stopping? ? 0 : ProcessBalancer.scheduled_workers(job_id)
+    end
+
+    def run
+      @thread = start_thread('heartbeat', &method(:run_heartbeat))
+    end
+
+    def quiet
+      return if @done
+
+      @done = true
+
+      update_jobs
+    end
+
+    def stop
+      quiet
+
+      @pool.shutdown
+      @pool.wait_for_termination(ProcessBalancer.options[:shutdown_timeout])
+      @pool.kill
+
+      clear_heartbeat
+    end
+
+    def stopping?
+      @done
+    end
+
+    private
+
+    def process_count=(value)
+      @process_count.value = value
+    end
+
+    def process_index=(value)
+      v = value.nil? ? -1 : value
+
+      @process_index.value = v
+    end
+
+    def run_heartbeat
+      loop do
+        heartbeat
+        sleep 5
+      end
+      logger.info('Heartbeat stopping...')
+    end
+
+    def clear_heartbeat
+      redis do |c|
+        c.lrem(PROCESSES_KEY, 0, identity)
+      end
+    rescue StandardError
+      # ignore errors
+    end
+
+    def heartbeat
+      update_process_index
+
+      _exists, msg = update_state
+
+      if msg
+        ::Process.kill(msg, $PID)
+      else
+        update_jobs
+      end
+    rescue StandardError => e
+      logger.error("heartbeat: #{e.message} @ #{e.backtrace_locations&.first || ''}")
+    end
+
+    def update_state
+      exists, _, _, msg = redis do |c|
+        c.multi do
+          c.exists(identity)
+          c.hmset(identity, 'info', info_json, 'beat', Time.now.to_f, 'quiet', @done, 'worker', process_index)
+          c.expire(identity, 60)
+          c.rpop("#{identity}-signals")
+        end
+      end
+
+      [exists, msg]
+    end
+
+    def update_process_index
+      redis do |c|
+        c.watch(PROCESSES_KEY)
+
+        workers     = c.lrange(PROCESSES_KEY, 0, -1)
+        num_workers = workers.size
+        index       = workers.find_index(identity)
+
+        if index.nil?
+          new_length = c.multi do
+            c.rpush(PROCESSES_KEY, identity)
+          end
+          unless new_length.nil?
+            num_workers = new_length.first
+            index       = new_length.first - 1
+          end
+        else
+          c.unwatch
+        end
+        self.process_index = index
+        self.process_count = num_workers
+      end
+    end
+
+    def update_jobs
+      watcher_stats = {}
+      @watchers.each do |job_id, watcher|
+        watcher.update_worker_config(process_index, process_count, workers_for_job(job_id))
+        watcher_stats[job_id] = JSON.dump(watcher.stats)
+      end
+
+      workers_key = "#{identity}:workers"
+      redis do |c|
+        c.multi do
+          c.del(workers_key)
+          watcher_stats.each do |job_id, stats_data|
+            c.hset(workers_key, job_id, stats_data)
+          end
+          c.expire(workers_key, 60)
+        end
+      end
+    end
+
+    def setup_job_watchers
+      @watchers = {}
+      @options.fetch(:job_sets, []).each do |job_config|
+        job_id = job_config[:id]
+        logger.debug "Starting watcher for #{job_id}"
+        @watchers[job_id] = Watcher.new(@pool, job_config)
+      end
+    end
+
+    def info
+      @info ||= {
+          hostname:    hostname,
+          pid:         ::Process.pid,
+          identity:    identity,
+          max_threads: @options[:max_threads],
+      }
+    end
+
+    def info_json
+      @info_json ||= JSON.dump(info)
+    end
+  end
+end

data/lib/process_balancer/private/cancellation.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'concurrent/options'
+require 'concurrent/atomic/atomic_reference'
+require 'concurrent/promises'
+
+# This is a private copy of Concurrent::Cancellation from concurrent-ruby-edge so we do not depend on the edge gem
+
+module ProcessBalancer
+  module Private
+    Synchronization = Concurrent::Synchronization
+    Promises = Concurrent::Promises
+    CancelledOperationError = Concurrent::CancelledOperationError
+
+    # TODO (pitr-ch 27-Mar-2016): cooperation with mutex, condition, select etc?
+    # TODO (pitr-ch 10-Dec-2018): integrate with enumerator?
+    # token.cancelable(array.each_with_index).each do |v, i|
+    #   # stops iterating when cancelled
+    # end
+    # token.cancelable(array).each_with_index do |v, i|
+    #   # stops iterating when cancelled
+    # end
+
+    # The Cancellation abstraction provides cooperative cancellation.
+    #
+    # The standard methods `Thread#raise` of `Thread#kill` available in Ruby
+    # are very dangerous (see linked the blog posts bellow).
+    # Therefore concurrent-ruby provides an alternative.
+    # * <https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/>
+    # * <http://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/>
+    # * <http://blog.headius.com/2008/02/rubys-threadraise-threadkill-timeoutrb.html>
+    #
+    # It provides an object which represents a task which can be executed,
+    # the task has to get the reference to the object and periodically cooperatively check that it is not cancelled.
+    # Good practices to make tasks cancellable:
+    # * check cancellation every cycle of a loop which does significant work,
+    # * do all blocking actions in a loop with a timeout then on timeout check cancellation
+    #   and if ok block again with the timeout
+    #
+    # The idea was inspired by <https://msdn.microsoft.com/en-us/library/dd537607(v=vs.110).aspx>
+    # @!macro warn.edge
+    #
+    # {include:file:docs-source/cancellation.out.md}
+    class Cancellation < Synchronization::Object
+      safe_initialization!
+
+      # Create Cancellation which will cancel itself in given time
+      #
+      # @!macro promises.param.intended_time
+      # @return [Cancellation]
+      def self.timeout(intended_time)
+        new Concurrent::Promises.schedule(intended_time)
+      end
+
+      # Creates the cancellation object.
+      #
+      # @param [Promises::Future, Promises::Event] origin of the cancellation.
+      #   When it is resolved the cancellation is canceled.
+      # @example
+      #   cancellation, origin = Concurrent::Cancellation.new
+      # @see #to_ary
+      def initialize(origin = Promises.resolvable_event)
+        super()
+        @Origin = origin
+      end
+
+      # Allow to multi-assign the Cancellation object
+      # @return [Array(Cancellation, Promises::Future), Array(Cancellation, Promises::Event)]
+      # @example
+      #   cancellation         = Concurrent::Cancellation.new
+      #   cancellation, origin = Concurrent::Cancellation.new
+      def to_ary
+        [self, @Origin]
+      end
+
+      # The event or future which is the origin of the cancellation
+      # @return [Promises::Future, Promises::Event]
+      def origin
+        @Origin
+      end
+
+      # Is the cancellation cancelled?
+      # Respective, was the origin of the cancellation resolved.
+      # @return [true, false]
+      def canceled?
+        @Origin.resolved?
+      end
+
+      # Raise error when cancelled
+      # @param [#exception] error to be risen
+      # @raise the error
+      # @return [self]
+      def check!(error = CancelledOperationError)
+        raise error if canceled?
+        self
+      end
+
+      # Creates a new Cancellation which is cancelled when first
+      # of the supplied cancellations or self is cancelled.
+      #
+      # @param [Cancellation] cancellations to combine
+      # @return [Cancellation] new cancellation
+      def join(*cancellations)
+        Cancellation.new Promises.any_event(*[@Origin, *cancellations.map(&:origin)])
+      end
+
+      # Short string representation.
+      # @return [String]
+      def to_s
+        format '%s %s>', super[0..-2], canceled? ? 'canceled' : 'pending'
+      end
+
+      alias_method :inspect, :to_s
+    end
+  end
+end

data/lib/process_balancer/rails.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module ProcessBalancer
+  # Rails integration
+  class Rails < ::Rails::Engine
+    config.after_initialize do
+      ProcessBalancer.configure do |config|
+        if config.server?
+          config.options[:reloader] = ProcessBalancer::Rails::Reloader.new
+        end
+      end
+    end
+
+    # cleanup active record connections
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+        # ensure
+        # ActiveRecord::Base.clear_active_connections!
+      end
+    end
+  end
+end

data/lib/process_balancer/redis_connection.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'connection_pool'
+require 'redis'
+
+module ProcessBalancer
+  module RedisConnection # :nodoc:
+    def self.create(options = {})
+      options[:url] = determine_redis_provider
+      size          = options[:size] || 2
+      pool_timeout  = options[:pool_timeout] || 1
+      ConnectionPool.new(timeout: pool_timeout, size: size) do
+        build_client(options)
+      end
+    end
+
+    class << self
+      private
+
+      def determine_redis_provider
+        if ENV['REDIS_PROVIDER'] =~ /[^A-Za-z_]/
+          ProcessBalancer.logger.error 'REDIS_PROVIDER should be set to the name of the environment variable that contains the redis URL'
+        end
+        ENV[
+            ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+        ]
+      end
+
+      def client_opts(options)
+        opts = options.dup
+        opts.delete(:namespace)
+
+        opts[:driver]             ||= Redis::Connection.drivers.last || 'ruby'
+        opts[:reconnect_attempts] ||= 1
+        opts
+      end
+
+      def build_client(options)
+        namespace = options[:namespace]
+
+        client = Redis.new client_opts(options)
+        if namespace
+          begin
+            require 'redis/namespace'
+            Redis::Namespace.new(namespace, redis: client)
+          rescue LoadError
+            ProcessBalancer.logger.error "Your redis configuration uses namespace '#{namespace}' but redis-namespace gem is not in your Gemfile"
+            exit(-127)
+          end
+        else
+          client
+        end
+      end
+    end
+  end
+end