roundhouse-x 0.1.0

data/lib/roundhouse/exception_handler.rb

@@ -0,0 +1,30 @@
+require 'roundhouse'
+
+module Roundhouse
+  module ExceptionHandler
+
+    class Logger
+      def call(ex, ctxHash)
+        Roundhouse.logger.warn(ctxHash) if !ctxHash.empty?
+        Roundhouse.logger.warn "#{ex.class.name}: #{ex.message}"
+        Roundhouse.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+      end
+
+      # Set up default handler which just logs the error
+      Roundhouse.error_handlers << Roundhouse::ExceptionHandler::Logger.new
+    end
+
+    def handle_exception(ex, ctxHash={})
+      Roundhouse.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Roundhouse.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Roundhouse.logger.error ex
+          Roundhouse.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
+      end
+    end
+
+  end
+end

data/lib/roundhouse/fetch.rb

@@ -0,0 +1,154 @@
+require 'roundhouse'
+require 'roundhouse/monitor'
+require 'roundhouse/util'
+require 'roundhouse/actor'
+
+module Roundhouse
+  ##
+  # The Fetcher blocks on Redis, waiting for a message to process
+  # from the queues.  It gets the message and hands it to the Manager
+  # to assign to a ready Processor.
+  class Fetcher
+    include Util
+    include Actor
+
+    TIMEOUT = 1
+
+    attr_reader :down
+
+    def initialize(mgr, options)
+      @down = nil
+      @mgr = mgr
+      @strategy = Fetcher.strategy.new(options)
+    end
+
+    # Fetching is straightforward: the Manager makes a fetch
+    # request for each idle processor when Roundhouse starts and
+    # then issues a new fetch request every time a Processor
+    # finishes a message.
+    #
+    # Because we have to shut down cleanly, we can't block
+    # forever and we can't loop forever.  Instead we reschedule
+    # a new fetch if the current fetch turned up nothing.
+    def fetch
+      watchdog('Fetcher#fetch died') do
+        return if Roundhouse::Fetcher.done?
+
+        begin
+          work = @strategy.retrieve_work
+          ::Roundhouse.logger.info("Redis is online, #{Time.now - @down} sec downtime") if @down
+          @down = nil
+
+          if work
+            @mgr.async.assign(work)
+          else
+            after(0) { fetch }
+          end
+        rescue => ex
+          handle_fetch_exception(ex)
+        end
+
+      end
+    end
+
+    private
+
+    def pause
+      sleep(TIMEOUT)
+    end
+
+    def handle_fetch_exception(ex)
+      if !@down
+        logger.error("Error fetching message: #{ex}")
+        ex.backtrace.each do |bt|
+          logger.error(bt)
+        end
+      end
+      @down ||= Time.now
+      pause
+      after(0) { fetch }
+    rescue Celluloid::TaskTerminated
+      # If redis is down when we try to shut down, all the fetch backlog
+      # raises these errors.  Haven't been able to figure out what I'm doing wrong.
+    end
+
+    # Ugh.  Say hello to a bloody hack.
+    # Can't find a clean way to get the fetcher to just stop processing
+    # its mailbox when shutdown starts.
+    def self.done!
+      @done = true
+    end
+
+    def self.reset # testing only
+      @done = nil
+    end
+
+    def self.done?
+      defined?(@done) && @done
+    end
+
+    def self.strategy
+      Roundhouse.options[:fetch] || RoundRobinFetch
+    end
+  end
+
+  class RoundRobinFetch
+    def initialize(options = nil)
+    end
+
+    def retrieve_work
+      work = Roundhouse.redis { |conn| Roundhouse::Monitor.await_next_job(conn) }
+      UnitOfWork.new(*work) if work
+    end
+
+    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+    # an instance method will make it async to the Fetcher actor
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
+      Roundhouse.logger.debug { "Re-queueing terminated jobs" }
+      jobs_to_requeue = {}
+      inprogress.each do |unit_of_work|
+        jobs_to_requeue[unit_of_work.queue_id] ||= []
+        jobs_to_requeue[unit_of_work.queue_id] << unit_of_work.message
+      end
+
+      Roundhouse.redis do |conn|
+        conn.pipelined do
+          jobs_to_requeue.each do |queue_id, jobs|
+            Roundhouse::Monitor.requeue(conn, queue_id, jobs)
+          end
+        end
+        # REFACTOR NOTE: This has to happen outside the pipelining since
+        # we need to read. We can refactor to put this back
+        # after converting the Monitor operations as EVAL scripts
+        jobs_to_requeue.keys.each do |queue_id|
+          Roundhouse::Monitor.push(conn, queue_id)
+        end
+      end
+      Roundhouse.logger.info("Pushed #{inprogress.size} messages back to Redis")
+    rescue => ex
+      Roundhouse.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+    end
+
+    UnitOfWork = Struct.new(:full_queue_name, :message) do
+      QUEUE_REGEX = /.*#{Roundhouse::Monitor::QUEUE}:/.freeze
+
+      def acknowledge
+        # nothing to do
+      end
+
+      def queue_id
+        full_queue_name.gsub(QUEUE_REGEX, '')
+      end
+
+      def requeue
+        Roundhouse.redis do |conn|
+          Roundhouse::Monitor.requeue(conn, queue_id, message)
+          Roundhouse::Monitor.push(conn, queue_id)
+        end
+      end
+    end
+
+  end
+end

data/lib/roundhouse/launcher.rb

@@ -0,0 +1,98 @@
+require 'roundhouse/actor'
+require 'roundhouse/manager'
+require 'roundhouse/fetch'
+require 'roundhouse/scheduled'
+
+module Roundhouse
+  # The Launcher is a very simple Actor whose job is to
+  # start, monitor and stop the core Actors in Roundhouse.
+  # If any of these actors die, the Roundhouse process exits
+  # immediately.
+  class Launcher
+    include Actor
+    include Util
+
+    trap_exit :actor_died
+
+    attr_reader :manager, :poller, :fetcher
+
+    def initialize(options)
+      @condvar = Celluloid::Condition.new
+      @manager = Roundhouse::Manager.new_link(@condvar, options)
+      @poller = Roundhouse::Scheduled::Poller.new_link
+      @fetcher = Roundhouse::Fetcher.new_link(@manager, options)
+      @manager.fetcher = @fetcher
+      @done = false
+      @options = options
+    end
+
+    def actor_died(actor, reason)
+      # https://github.com/mperham/sidekiq/issues/2057#issuecomment-66485477
+      return if @done || !reason
+
+      Roundhouse.logger.warn("Roundhouse died due to the following error, cannot recover, process exiting")
+      handle_exception(reason)
+      exit(1)
+    end
+
+    def run
+      watchdog('Launcher#run') do
+        manager.async.start
+        poller.async.poll(true)
+
+        start_heartbeat
+      end
+    end
+
+    def stop
+      watchdog('Launcher#stop') do
+        @done = true
+        Roundhouse::Fetcher.done!
+        fetcher.terminate if fetcher.alive?
+        poller.terminate if poller.alive?
+
+        manager.async.stop(:shutdown => true, :timeout => @options[:timeout])
+        @condvar.wait
+        manager.terminate
+
+        # Requeue everything in case there was a worker who grabbed work while stopped
+        # This call is a no-op in Roundhouse but necessary for Roundhouse Pro.
+        Roundhouse::Fetcher.strategy.bulk_requeue([], @options)
+
+        stop_heartbeat
+      end
+    end
+
+    private
+
+    def start_heartbeat
+      key = identity
+      data = {
+        'hostname' => hostname,
+        'started_at' => Time.now.to_f,
+        'pid' => $$,
+        'tag' => @options[:tag] || '',
+        'concurrency' => @options[:concurrency],
+        'queues' => @options[:queues].uniq,
+        'labels' => Roundhouse.options[:labels],
+        'identity' => identity,
+      }
+      # this data doesn't change so dump it to a string
+      # now so we don't need to dump it every heartbeat.
+      json = Roundhouse.dump_json(data)
+      manager.heartbeat(key, data, json)
+    end
+
+    def stop_heartbeat
+      Roundhouse.redis do |conn|
+        conn.pipelined do
+          conn.srem('processes', identity)
+          conn.del("#{identity}:workers")
+        end
+      end
+    rescue
+      # best effort, ignore network errors
+    end
+
+  end
+end

data/lib/roundhouse/logging.rb

@@ -0,0 +1,104 @@
+require 'time'
+require 'logger'
+
+module Roundhouse
+  module Logging
+
+    class Pretty < Logger::Formatter
+      SPACE = " "
+
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:roundhouse_context]
+        " #{c.join(SPACE)}" if c && c.any?
+      end
+    end
+
+    class WithoutTimestamp < Pretty
+      def call(severity, time, program_name, message)
+        "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+    end
+
+    def self.with_context(msg)
+      Thread.current[:roundhouse_context] ||= []
+      Thread.current[:roundhouse_context] << msg
+      yield
+    ensure
+      Thread.current[:roundhouse_context].pop
+    end
+
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
+      oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
+    # This reopens ALL logfiles in the process that have been rotated
+    # using logrotate(8) (without copytruncate) or similar tools.
+    # A +File+ object is considered for reopening if it is:
+    #   1) opened with the O_APPEND and O_WRONLY flags
+    #   2) the current open file handle does not match its original open path
+    #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+    # Returns the number of files reopened
+    def self.reopen_logs
+      to_reopen = []
+      append_flags = File::WRONLY | File::APPEND
+
+      ObjectSpace.each_object(File) do |fp|
+        begin
+          if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+            to_reopen << fp
+          end
+        rescue IOError, Errno::EBADF
+        end
+      end
+
+      nr = 0
+      to_reopen.each do |fp|
+        orig_st = begin
+          fp.stat
+        rescue IOError, Errno::EBADF
+          next
+        end
+
+        begin
+          b = File.stat(fp.path)
+          next if orig_st.ino == b.ino && orig_st.dev == b.dev
+        rescue Errno::ENOENT
+        end
+
+        begin
+          File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+          fp.sync = true
+          nr += 1
+        rescue IOError, Errno::EBADF
+          # not much we can do...
+        end
+      end
+      nr
+    rescue RuntimeError => ex
+      # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
+      puts "Unable to reopen logs: #{ex.message}"
+    end
+
+    def logger
+      Roundhouse::Logging.logger
+    end
+  end
+end

data/lib/roundhouse/manager.rb

@@ -0,0 +1,236 @@
+# encoding: utf-8
+require 'roundhouse/util'
+require 'roundhouse/actor'
+require 'roundhouse/processor'
+require 'roundhouse/fetch'
+
+module Roundhouse
+
+  ##
+  # The main router in the system.  This
+  # manages the processor state and accepts messages
+  # from Redis to be dispatched to an idle processor.
+  #
+  class Manager
+    include Util
+    include Actor
+    trap_exit :processor_died
+
+    attr_reader :ready
+    attr_reader :busy
+    attr_accessor :fetcher
+
+    SPIN_TIME_FOR_GRACEFUL_SHUTDOWN = 1
+    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
+
+    def initialize(condvar, options={})
+      logger.debug { options.inspect }
+      @options = options
+      @count = options[:concurrency] || 25
+      raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
+      @done_callback = nil
+      @finished = condvar
+
+      @in_progress = {}
+      @threads = {}
+      @done = false
+      @busy = []
+      @ready = @count.times.map do
+        p = Processor.new_link(current_actor)
+        p.proxy_id = p.object_id
+        p
+      end
+    end
+
+    def stop(options={})
+      watchdog('Manager#stop died') do
+        should_shutdown = options[:shutdown]
+        timeout = options[:timeout]
+
+        @done = true
+
+        logger.info { "Terminating #{@ready.size} quiet workers" }
+        @ready.each { |x| x.terminate if x.alive? }
+        @ready.clear
+
+        return if clean_up_for_graceful_shutdown
+
+        hard_shutdown_in timeout if should_shutdown
+      end
+    end
+
+    def clean_up_for_graceful_shutdown
+      if @busy.empty?
+        shutdown
+        return true
+      end
+
+      after(SPIN_TIME_FOR_GRACEFUL_SHUTDOWN) { clean_up_for_graceful_shutdown }
+      false
+    end
+
+    def start
+      @ready.each { dispatch }
+    end
+
+    def when_done(&blk)
+      @done_callback = blk
+    end
+
+    def processor_done(processor)
+      watchdog('Manager#processor_done died') do
+        @done_callback.call(processor) if @done_callback
+        @in_progress.delete(processor.object_id)
+        @threads.delete(processor.object_id)
+        @busy.delete(processor)
+        if stopped?
+          processor.terminate if processor.alive?
+          shutdown if @busy.empty?
+        else
+          @ready << processor if processor.alive?
+        end
+        dispatch
+      end
+    end
+
+    def processor_died(processor, reason)
+      watchdog("Manager#processor_died died") do
+        @in_progress.delete(processor.object_id)
+        @threads.delete(processor.object_id)
+        @busy.delete(processor)
+
+        unless stopped?
+          p = Processor.new_link(current_actor)
+          p.proxy_id = p.object_id
+          @ready << p
+          dispatch
+        else
+          shutdown if @busy.empty?
+        end
+      end
+    end
+
+    def assign(work)
+      watchdog("Manager#assign died") do
+        if stopped?
+          # Race condition between Manager#stop if Fetcher
+          # is blocked on redis and gets a message after
+          # all the ready Processors have been stopped.
+          # Push the message back to redis.
+          work.requeue
+        else
+          processor = @ready.pop
+          @in_progress[processor.object_id] = work
+          @busy << processor
+          processor.async.process(work)
+        end
+      end
+    end
+
+    # A hack worthy of Rube Goldberg.  We need to be able
+    # to hard stop a working thread.  But there's no way for us to
+    # get handle to the underlying thread performing work for a processor
+    # so we have it call us and tell us.
+    def real_thread(proxy_id, thr)
+      @threads[proxy_id] = thr
+    end
+
+    PROCTITLES = [
+      proc { 'roundhouse'.freeze },
+      proc { Roundhouse::VERSION },
+      proc { |mgr, data| data['tag'] },
+      proc { |mgr, data| "[#{mgr.busy.size} of #{data['concurrency']} busy]" },
+      proc { |mgr, data| "stopping" if mgr.stopped? },
+    ]
+
+    def heartbeat(key, data, json)
+      results = PROCTITLES.map {|x| x.(self, data) }
+      results.compact!
+      $0 = results.join(' ')
+
+      ❤(key, json)
+      after(5) do
+        heartbeat(key, data, json)
+      end
+    end
+
+    def stopped?
+      @done
+    end
+
+    private
+
+    def ❤(key, json)
+      begin
+        _, _, _, msg = Roundhouse.redis do |conn|
+          conn.multi do
+            conn.sadd('processes', key)
+            conn.hmset(key, 'info', json, 'busy', @busy.size, 'beat', Time.now.to_f)
+            conn.expire(key, 60)
+            conn.rpop("#{key}-signals")
+          end
+        end
+
+        return unless msg
+
+        if JVM_RESERVED_SIGNALS.include?(msg)
+          Roundhouse::CLI.instance.handle_signal(msg)
+        else
+          ::Process.kill(msg, $$)
+        end
+      rescue => e
+        # ignore all redis/network issues
+        logger.error("heartbeat: #{e.message}")
+      end
+    end
+
+    def hard_shutdown_in(delay)
+      logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
+
+      after(delay) do
+        watchdog("Manager#hard_shutdown_in died") do
+          # We've reached the timeout and we still have busy workers.
+          # They must die but their messages shall live on.
+          logger.warn { "Terminating #{@busy.size} busy worker threads" }
+          logger.warn { "Work still in progress #{@in_progress.values.inspect}" }
+
+          requeue
+
+          @busy.each do |processor|
+            if processor.alive? && t = @threads.delete(processor.object_id)
+              t.raise Shutdown
+            end
+          end
+
+          @finished.signal
+        end
+      end
+    end
+
+    def dispatch
+      return if stopped?
+      # This is a safety check to ensure we haven't leaked
+      # processors somehow.
+      raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
+      raise "No ready processor!?" if @ready.empty?
+
+      @fetcher.async.fetch
+    end
+
+    def shutdown
+      requeue
+      @finished.signal
+    end
+
+    def requeue
+      # Re-enqueue terminated jobs
+      # NOTE: You may notice that we may push a job back to redis before
+      # the worker thread is terminated. This is ok because Roundhouse's
+      # contract says that jobs are run AT LEAST once. Process termination
+      # is delayed until we're certain the jobs are back in Redis because
+      # it is worse to lose a job than to run it twice.
+      Roundhouse::Fetcher.strategy.bulk_requeue(@in_progress.values, @options)
+      @in_progress.clear
+    end
+  end
+end