process_balancer 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9e2b7936e66ec20f0b2cf36e107dd9a2aae52bdd19e03617f44d6e36abd5c008
+  data.tar.gz: 5c7b264651a91513a59545e2fdcbbfca0bd13d079c4787e8fce3c90370e3d59c
+SHA512:
+  metadata.gz: 1c846bde2ec1126116b529471e338e3caec0bd87695f38e6fd61bbfca5e9bdf29c888dbb55b99a7b849dc3584c1807d95ddd198f05eec6fb0ba837d24c5a313d
+  data.tar.gz: 3aab7f4a78e10282065c1dfba9c7d39ea74e489d52de600c85f42e6608aab07864380aa36df49a8f13c2b3e4929230d010ed782561614648f98f81696ebd72d1

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+require 'pp' # workaround for FakeFS
+
+# Specify your gem's dependencies in process_balancer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Edward Rudd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.adoc

@@ -0,0 +1,90 @@
+= ProcessBalancer
+
+ProcessBalancer is a background job runner that is targeted toward the specific use-case of long running jobs.
+
+If you need a job runner that runs small background jobs, look to https://sidekiq.org/[Sidekiq].
+
+ProcessBalancer has built-in functionality to balance your jobs across multiple instances.
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'process_balancer'
+----
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install process_balancer
+
+== Usage
+
+Build a Job class that works through an iteration of your processing.
+The iteration does not have to be one record, it can be working through 1000 records.
+The iteration needs to be designed to lock its work atomically so that multiple concurrent workers could be running.
+The ProcessBalancer takes care of scaling out to run however many workers you want running across however many nodes you have running.
+Each instance of the Job will have a unique worker_id to ensure they do not trample on each other.
+
+[source,ruby]
+----
+class ProcessQueue < ProcessBalancer::Base
+  # set a worker locking algorithm
+  lock_driver :simple_redis
+
+  LOCK_SQL = <<~SQL
+    WITH T as (
+      SELECT ctid
+      FROM queue_table
+      WHERE status = #{QueueRecord::QUEUED} AND lock IS NULL
+      ORDER BY id
+      LIMIT 1000
+      FOR UPDATE SKIP LOCKED
+    )
+    UPDATE queue_records
+    SET lock = :lock, updated_at = :now
+    WHERE ctid = ANY(ARRAY(SELECT ctid FROM T))
+  SQL
+
+  def lock_records
+    # grab a # of records and lock them with the worker_id
+    sql = ActiveRecord::Base.sanitize_sql([LOCK_SQL, {lock: worker_index, now: Time.now}])
+    ActiveRecord::Base.connection.execute(sql)
+    # process those records
+    QueueRecord.where(lock: worker_index)
+  end
+
+  def process_record(entry)
+    # do processing
+    # mark record as processed and release the lock on that record
+    entry.update(lock: nil, status: QueueRecord::PROCESSED)
+  end
+
+  def unlock_records
+    # if any error occurs unlock any of our unprocessed records
+    QueueRecord.where(lock: worker_index).update_all(lock: nil)
+  end
+end
+----
+
+Configuration file
+
+[source,yaml]
+----
+jobs:
+  process_queue:
+    class: 'ProcessQueue'
+----
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/NetsoftHoldings/process_balancer.
+
+== License
+
+The gem is available as open source under the terms of the https://opensource.org/licenses/LGPL-3.0[LGPLv3 License].

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/TODO.adoc

@@ -0,0 +1 @@
+- [ ] add in simple error handling

data/exe/process_balancer

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/process_balancer/cli'
+
+begin
+  cli = ProcessBalancer::CLI.instance
+  cli.parse
+  cli.run
+rescue StandardError => e
+  raise e if $DEBUG
+
+  warn e.message
+  warn e.backtrace.join("\n")
+
+  exit 1
+end

data/lib/process_balancer.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'socket'
+require 'securerandom'
+
+require_relative 'process_balancer/version'
+require_relative 'process_balancer/redis_connection'
+require_relative 'process_balancer/base'
+
+module ProcessBalancer # :nodoc:
+  class Error < StandardError; end
+
+  PROCESSES_KEY    = 'processes'
+  WORKER_COUNT_KEY = 'worker_counts'
+
+  DEFAULTS = {
+      redis:            {},
+      job_sets:         [],
+      require:          '.',
+      max_threads:      10,
+      shutdown_timeout: 30,
+      reloader:         proc { |&block| block.call },
+  }.freeze
+
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+
+  def self.options=(opts)
+    @options = opts
+  end
+
+  ##
+  # Configuration for ProcessBalancer, use like:
+  #
+  #   ProcessBalancer.configure do |config|
+  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
+  #     if config.server?
+  #      # any configuration specific to server
+  #     end
+  #   end
+  def self.configure
+    yield self
+  end
+
+  def self.server?
+    defined?(ProcessBalancer::CLI)
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT, level: Logger::INFO)
+  end
+
+  def self.redis
+    raise ArgumentError, 'requires a block' unless block_given?
+
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => e
+        # if we are on a slave, disconnect and reopen to get back on the master
+        (conn.disconnect!; retryable = false; retry) if retryable && e.message =~ /READONLY/
+        raise
+      end
+    end
+  end
+
+  def self.redis_pool
+    @redis_pool ||= RedisConnection.create(options[:redis])
+  end
+
+  def self.redis=(hash)
+    @redis_pool = if hash.is_a?(ConnectionPool)
+                    hash
+                  else
+                    RedisConnection.create(hash)
+                  end
+  end
+
+  def self.reset
+    @redis_pool    = nil
+    @options       = nil
+    @logger        = nil
+    @process_nonce = nil
+    @identity      = nil
+  end
+
+  def self.hostname
+    ENV['DYNO'] || Socket.gethostname
+  end
+
+  def self.process_nonce
+    @process_nonce ||= SecureRandom.hex(6)
+  end
+
+  def self.identity
+    @identity ||= "#{hostname}:#{$PID}:#{process_nonce}"
+  end
+
+  def self.adjust_scheduled_workers(job_id, by: nil, to: nil)
+    if !to.nil?
+      redis { |c| c.hset(WORKER_COUNT_KEY, job_id.to_s, to) }
+    elsif !by.nil?
+      redis { |c| c.hincrby(WORKER_COUNT_KEY, job_id.to_s, by) }
+    else
+      raise ArgumentError, 'Must specify either by: (an increment/decrement) or to: (an exact value)'
+    end
+  end
+
+  def self.scheduled_workers(job_id)
+    value = redis { |c| c.hget(WORKER_COUNT_KEY, job_id.to_s) }&.to_i
+    value.nil? ? 1 : value
+  end
+
+  def self.running_workers(job_id)
+    count = 0
+
+    redis do |c|
+      workers = c.lrange(PROCESSES_KEY, 0, -1)
+
+      workers.each do |worker|
+        data = c.hget("#{worker}:workers", job_id)
+        if data
+          data = JSON.parse(data, symbolize_names: true)
+          count += (data.dig(:running)&.size || 0)
+        end
+      rescue JSON::ParserError
+        nil
+      end
+    end
+
+    count
+  end
+end
+
+require 'process_balancer/rails' if defined?(::Rails::Engine)

data/lib/process_balancer/base.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ProcessBalancer
+  class Base # :nodoc:
+    attr_reader :worker_index, :status, :options
+
+    def self.lock_driver(driver)
+      if driver.is_a?(Symbol)
+        file = "process_balancer/lock/#{driver}"
+        driver = driver.to_s
+        unless driver !~ /_/ && driver =~ /[A-Z]+.*/
+          driver = driver.split('_').map(&:capitalize).join
+        end
+        require file
+        klass = ProcessBalancer::Lock.const_get(driver)
+        include klass
+      else
+        raise ArgumentError, 'Please pass a symbol for the driver to use'
+      end
+    end
+
+    def initialize(worker_index, options = {})
+      @worker_index = worker_index
+      @options      = options
+    end
+
+    def perform
+      before_perform
+      worker_lock do |lock|
+        @status = nil
+        records = lock_records
+        lock.extend!
+        records&.each do |r|
+          process_record(r)
+          lock.extend!
+        end
+        @status
+      ensure
+        unlock_records
+        after_perform
+      end
+    end
+
+    def status_abort
+      @status = :abort
+    end
+
+    def status_sleep(duration)
+      @status = [:sleep, duration]
+    end
+
+    def runtime_lock_timeout
+      options[:runtime_lock_timeout] || 30
+    end
+
+    def job_id
+      options[:id]
+    end
+
+    def before_perform; end
+
+    def after_perform; end
+
+    def lock_records
+      raise NotImplementedError
+    end
+
+    def unlock_records
+      raise NotImplementedError
+    end
+
+    def process_record(record)
+      raise NotImplementedError
+    end
+
+    def worker_lock(&_block)
+      raise NotImplementedError, 'Specify a locking driver via lock_driver :driver'
+    end
+  end
+end

data/lib/process_balancer/cli.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+$stdout.sync = true
+
+require 'optparse'
+require 'yaml'
+require 'erb'
+
+require_relative '../process_balancer'
+require_relative 'manager'
+require_relative 'util'
+
+module ProcessBalancer
+  class CLI # :nodoc:
+    include Util
+
+    attr_reader :manager, :environment
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def parse(args = ARGV)
+      setup_options(args)
+      initialize_logger
+      validate!
+    end
+
+    def run
+      boot_system
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
+      Thread.current.name = 'main'
+
+      self_read, self_write = IO.pipe
+      signals               = %w[INT TERM TTIN TSTP USR1 USR2]
+
+      signals.each do |sig|
+        trap sig do
+          self_write.write("#{sig}\n")
+        end
+      rescue ArgumentError
+        logger.info "Signal #{sig} not supported"
+      end
+
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+
+      if options[:job_sets].empty?
+        logger.error 'No jobs configured! Configure your jobs in the configuration file.'
+      else
+        logger.info 'Configured jobs'
+        options[:job_sets].each do |config|
+          logger.info " - #{config[:id]}"
+        end
+      end
+
+      @manager = ProcessBalancer::Manager.new(options)
+
+      begin
+        @manager.run
+
+        while (readable_io = IO.select([self_read]))
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        logger.info 'Shutting down'
+        @manager.stop
+        logger.info 'Bye!'
+
+        # Explicitly exit so busy Processor threads wont block process shutdown.
+        exit(0)
+      end
+    end
+
+    private
+
+    # region initializing app
+    def boot_system
+      ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
+      if File.directory?(options[:require])
+        require 'rails'
+        if ::Rails::VERSION::MAJOR < 5
+          raise 'Only rails 5+ is supported'
+        else
+          require 'process_balancer/rails'
+          require File.expand_path("#{options[:require]}/config/environment.rb")
+        end
+      else
+        require options[:require]
+      end
+    end
+
+    # endregion
+
+    # region option and configuration handling
+    def parse_options(argv)
+      opts = {redis: {}}
+
+      @parser = OptionParser.new do |o|
+        o.on('-eENVIRONMENT', '--environment ENVIRONMENT', 'Specify the app environment') do |arg|
+          opts[:environment] = arg
+        end
+
+        o.on('-rREQUIRE', '--require REQUIRE', 'Specify rails app path or file to boot your app with jobs') do |arg|
+          opts[:require] = arg
+        end
+
+        o.on('-cFILE', '--config FILE', 'Specify a configuration file. Default is config/process_balancer.yml in the rails app') do |arg|
+          opts[:config_file] = arg
+        end
+
+        o.on('-v', '--[no-]verbose', 'Run verbosely') do |v|
+          opts[:verbose] = v
+        end
+      end
+
+      @parser.banner = 'process_balancer [options]'
+      @parser.on_tail '-h', '--help', 'Show help' do
+        logger.info parser.help
+        exit(1)
+      end
+
+      @parser.parse!(argv)
+
+      opts
+    end
+
+    def locate_config_file!(opts)
+      if opts[:config_file]
+        unless File.exist?(opts[:config_file])
+          raise ArgumentError, "Config file not found: #{opts[:config_file]}"
+        end
+      else
+        config_dir = if opts[:require] && File.directory?(opts[:require])
+                       File.join(opts[:require], 'config')
+                     else
+                       File.join(options[:require], 'config')
+                     end
+
+        %w[process_balancer.yml process_balancer.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          if File.exist?(path)
+            opts[:config_file] ||= path
+          end
+        end
+      end
+    end
+
+    def parse_config(path)
+      config = YAML.safe_load(ERB.new(File.read(path)).result, symbolize_names: true) || {}
+
+      opts = {}
+      # pull in global config
+      opts.merge!(config.dig(:global) || {})
+      # pull in ENV override config
+      opts.merge!(config.dig(:environments, environment.to_sym) || {})
+
+      opts[:job_sets] = parse_jobs(config)
+
+      opts
+    end
+
+    def parse_jobs(config)
+      (config[:jobs] || {}).map do |id, job|
+        {
+            id: id,
+            **job,
+        }
+      end
+    end
+
+    def setup_options(args)
+      opts = parse_options(args)
+
+      setup_environment opts[:environment]
+
+      locate_config_file!(opts)
+
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+
+      options.merge!(opts)
+    end
+
+    def validate!
+      if !File.exist?(options[:require]) ||
+          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+        logger.info 'Please point process balancer to a Rails application or a Ruby file'
+        logger.info 'to load your job classes with -r [DIR|FILE].'
+        logger.info @parser.help
+        exit(1)
+      end
+    end
+
+    def options
+      ProcessBalancer.options
+    end
+
+    def initialize_logger
+      logger.level = ::Logger::DEBUG if options[:verbose]
+    end
+
+    # endregion
+
+    # region signal handling
+    SIGNAL_HANDLERS = {
+        INT:  lambda { |_cli|
+          # Ctrl-C in terminal
+          raise Interrupt
+        },
+        TERM: lambda { |_cli|
+          # TERM is the signal that process must exit.
+          # Heroku sends TERM and then waits 30 seconds for process to exit.
+          raise Interrupt
+        },
+        USR1: lambda { |cli|
+          ProcessBalancer.logger.info 'Received USR1, no longer accepting new work'
+          cli.manager.quiet
+        },
+        TSTP: lambda { |cli|
+          ProcessBalancer.logger.info 'Received TSTP, no longer accepting new work'
+          cli.manager.quiet
+        },
+        TTIN: lambda { |_cli|
+          Thread.list.each do |thread|
+            ProcessBalancer.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+            if thread.backtrace
+              ProcessBalancer.logger.warn thread.backtrace.join("\n")
+            else
+              ProcessBalancer.logger.warn '<no backtrace available>'
+            end
+          end
+        },
+    }.freeze
+
+    def handle_signal(sig)
+      logger.debug "Got #{sig} signal"
+      handle = SIGNAL_HANDLERS[sig.to_sym]
+      if handle
+        handle.call(self)
+      else
+        logger.info("No signal handler for #{sig}")
+      end
+    end
+
+    # endregion
+
+    # region environment
+    def setup_environment(cli_env)
+      @environment = cli_env || ENV['APP_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
+    # endregion
+  end
+end