roundhouse-x 0.1.0

data/lib/roundhouse/processor.rb

@@ -0,0 +1,159 @@
+require 'roundhouse/util'
+require 'roundhouse/actor'
+require 'roundhouse/monitor'
+
+require 'roundhouse/middleware/server/retry_jobs'
+require 'roundhouse/middleware/server/logging'
+
+module Roundhouse
+  ##
+  # The Processor receives a message from the Manager and actually
+  # processes it.  It instantiates the worker, runs the middleware
+  # chain and then calls Roundhouse::Worker#perform.
+  class Processor
+    # To prevent a memory leak, ensure that stats expire. However, they should take up a minimal amount of storage
+    # so keep them around for a long time
+    STATS_TIMEOUT = 24 * 60 * 60 * 365 * 5
+
+    include Util
+    include Actor
+
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Server::Logging
+        m.add Middleware::Server::RetryJobs
+        if defined?(::ActiveRecord::Base)
+          require 'roundhouse/middleware/server/active_record'
+          m.add Roundhouse::Middleware::Server::ActiveRecord
+        end
+      end
+    end
+
+    attr_accessor :proxy_id
+
+    def initialize(boss)
+      @boss = boss
+    end
+
+    def process(work)
+      msgstr = work.message
+      queue = work.queue_id
+
+      @boss.async.real_thread(proxy_id, Thread.current)
+
+      ack = true
+      begin
+        msg = Roundhouse.load_json(msgstr)
+        klass  = msg['class'].constantize
+        worker = klass.new
+        worker.jid = msg['jid']
+
+        stats(worker, msg, queue) do
+          Roundhouse.server_middleware.invoke(worker, msg, queue) do
+            execute_job(worker, cloned(msg['args']))
+          end
+        end
+
+        # Put the queue back into rotation
+        Roundhouse.redis { |conn| Roundhouse::Monitor.push(conn, queue) }
+      rescue Roundhouse::Shutdown
+        # Had to force kill this job because it didn't finish
+        # within the timeout.  Don't acknowledge the work since
+        # we didn't properly finish it.
+        ack = false
+      rescue Exception => ex
+        handle_exception(ex, msg || { :message => msgstr })
+        raise
+      ensure
+        work.acknowledge if ack
+      end
+
+      @boss.async.processor_done(current_actor)
+    end
+
+    def inspect
+      "<Processor##{object_id.to_s(16)}>"
+    end
+
+    def execute_job(worker, cloned_args)
+      worker.perform(*cloned_args)
+    end
+
+    private
+
+    def thread_identity
+      @str ||= Thread.current.object_id.to_s(36)
+    end
+
+    def stats(worker, msg, queue)
+      # Do not conflate errors from the job with errors caused by updating
+      # stats so calling code can react appropriately
+      retry_and_suppress_exceptions do
+        hash = Roundhouse.dump_json({:queue => queue, :payload => msg, :run_at => Time.now.to_i })
+        Roundhouse.redis do |conn|
+          conn.multi do
+            conn.hmset("#{identity}:workers", thread_identity, hash)
+            conn.expire("#{identity}:workers", 60*60*4)
+          end
+        end
+      end
+
+      begin
+        yield
+      rescue Exception
+        retry_and_suppress_exceptions do
+          failed = "stat:failed:#{Time.now.utc.to_date}"
+          Roundhouse.redis do |conn|
+            conn.multi do
+              conn.incrby("stat:failed", 1)
+              conn.incrby(failed, 1)
+              conn.expire(failed, STATS_TIMEOUT)
+            end
+          end
+        end
+        raise
+      ensure
+        retry_and_suppress_exceptions do
+          processed = "stat:processed:#{Time.now.utc.to_date}"
+          Roundhouse.redis do |conn|
+            conn.multi do
+              conn.hdel("#{identity}:workers", thread_identity)
+              conn.incrby("stat:processed", 1)
+              conn.incrby(processed, 1)
+              conn.expire(processed, STATS_TIMEOUT)
+            end
+          end
+        end
+      end
+    end
+
+    # Deep clone the arguments passed to the worker so that if
+    # the message fails, what is pushed back onto Redis hasn't
+    # been mutated by the worker.
+    def cloned(ary)
+      Marshal.load(Marshal.dump(ary))
+    end
+
+    # If an exception occurs in the block passed to this method, that block will be retried up to max_retries times.
+    # All exceptions will be swallowed and logged.
+    def retry_and_suppress_exceptions(max_retries = 5)
+      retry_count = 0
+      begin
+        yield
+      rescue => e
+        retry_count += 1
+        if retry_count <= max_retries
+          Roundhouse.logger.debug {"Suppressing and retrying error: #{e.inspect}"}
+          pause_for_recovery(retry_count)
+          retry
+        else
+          handle_exception(e, { :message => "Exhausted #{max_retries} retries"})
+        end
+      end
+    end
+
+    def pause_for_recovery(retry_count)
+      sleep(retry_count)
+    end
+  end
+end

data/lib/roundhouse/rails.rb

@@ -0,0 +1,24 @@
+module Roundhouse
+  def self.hook_rails!
+    # no op
+    # This may be completely removed in the future
+    # Those extensions make sense for Sidekiq, but do
+    # not make sense for Roundhouse
+  end
+
+  # Removes the generic aliases which MAY clash with names of already
+  #  created methods by other applications. The methods `roundhouse_delay`,
+  #  `roundhouse_delay_for` and `roundhouse_delay_until` can be used instead.
+  def self.remove_delay!
+    # no op
+    # This may be completely removed in the future
+    # Those extensions make sense for Sidekiq, but do
+    # not make sense for Roundhouse
+  end
+
+  class Rails < ::Rails::Engine
+    initializer 'roundhouse' do
+      Roundhouse.hook_rails!
+    end
+  end if defined?(::Rails)
+end

data/lib/roundhouse/redis_connection.rb

@@ -0,0 +1,77 @@
+require 'connection_pool'
+require 'redis'
+require 'uri'
+
+module Roundhouse
+  class RedisConnection
+    class << self
+
+      def create(options={})
+        options[:url] ||= determine_redis_provider
+
+        # need a connection for Fetcher and Retry
+        size = options[:size] || (Roundhouse.server? ? (Roundhouse.options[:concurrency] + 2) : 5)
+        pool_timeout = options[:pool_timeout] || 1
+
+        log_info(options)
+
+        ConnectionPool.new(:timeout => pool_timeout, :size => size) do
+          build_client(options)
+        end
+      end
+
+      private
+
+      def build_client(options)
+        namespace = options[:namespace]
+
+        client = Redis.new client_opts(options)
+        if namespace
+          require 'redis/namespace'
+          Redis::Namespace.new(namespace, :redis => client)
+        else
+          client
+        end
+      end
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] = opts[:driver] || 'ruby'
+
+        opts
+      end
+
+      def log_info(options)
+        # Don't log Redis AUTH password
+        redacted = "REDACTED"
+        scrubbed_options = options.dup
+        if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
+          uri.password = redacted
+          scrubbed_options[:url] = uri.to_s
+        end
+        if scrubbed_options[:password]
+          scrubbed_options[:password] = redacted
+        end
+        if Roundhouse.server?
+          Roundhouse.logger.info("Booting Roundhouse #{Roundhouse::VERSION} with redis options #{scrubbed_options}")
+        else
+          Roundhouse.logger.debug("#{Roundhouse::NAME} client with redis options #{scrubbed_options}")
+        end
+      end
+
+      def determine_redis_provider
+        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+      end
+
+    end
+  end
+end

data/lib/roundhouse/scheduled.rb

@@ -0,0 +1,115 @@
+require 'roundhouse'
+require 'roundhouse/util'
+require 'roundhouse/actor'
+require 'roundhouse/api'
+
+module Roundhouse
+  module Scheduled
+    SETS = %w(retry schedule)
+
+    class Enq
+      def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
+        # A job's "score" in Redis is the time at which it should be processed.
+        # Just check Redis for the set of jobs with a timestamp before now.
+        Roundhouse.redis do |conn|
+          sorted_sets.each do |sorted_set|
+            # Get the next item in the queue if it's score (time to execute) is <= now.
+            # We need to go through the list one at a time to reduce the risk of something
+            # going wrong between the time jobs are popped from the scheduled queue and when
+            # they are pushed onto a work queue and losing the jobs.
+            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
+
+              # Pop item off the queue and add it to the work queue. If the job can't be popped from
+              # the queue, it's because another process already popped it so we can move on to the
+              # next one.
+              if conn.zrem(sorted_set, job)
+                Roundhouse::Client.push(Roundhouse.load_json(job))
+                Roundhouse::Logging.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              end
+            end
+          end
+        end
+      end
+    end
+
+    ##
+    # The Poller checks Redis every N seconds for jobs in the retry or scheduled
+    # set have passed their timestamp and should be enqueued.  If so, it
+    # just pops the job back onto its original queue so the
+    # workers can pick it up like any other job.
+    class Poller
+      include Util
+      include Actor
+
+      INITIAL_WAIT = 10
+
+      def initialize
+        @enq = (Roundhouse.options[:scheduled_enq] || Roundhouse::Scheduled::Enq).new
+      end
+
+      def poll(first_time=false)
+        watchdog('scheduling poller thread died!') do
+          initial_wait if first_time
+
+          begin
+            @enq.enqueue_jobs
+          rescue => ex
+            # Most likely a problem with redis networking.
+            # Punt and try again at the next interval
+            logger.error ex.message
+            logger.error ex.backtrace.first
+          end
+
+          after(random_poll_interval) { poll }
+        end
+      end
+
+      private
+
+      # Calculates a random interval that is ±50% the desired average.
+      def random_poll_interval
+        poll_interval_average * rand + poll_interval_average.to_f / 2
+      end
+
+      # We do our best to tune the poll interval to the size of the active Roundhouse
+      # cluster.  If you have 30 processes and poll every 15 seconds, that means one
+      # Roundhouse is checking Redis every 0.5 seconds - way too often for most people
+      # and really bad if the retry or scheduled sets are large.
+      #
+      # Instead try to avoid polling more than once every 15 seconds.  If you have
+      # 30 Roundhouse processes, we'll poll every 30 * 15 or 450 seconds.
+      # To keep things statistically random, we'll sleep a random amount between
+      # 225 and 675 seconds for each poll or 450 seconds on average.  Otherwise restarting
+      # all your Roundhouse processes at the same time will lead to them all polling at
+      # the same time: the thundering herd problem.
+      #
+      # We only do this if poll_interval is unset (the default).
+      def poll_interval_average
+        Roundhouse.options[:poll_interval_average] ||= scaled_poll_interval
+      end
+
+      # Calculates an average poll interval based on the number of known Roundhouse processes.
+      # This minimizes a single point of failure by dispersing check-ins but without taxing
+      # Redis if you run many Roundhouse processes.
+      def scaled_poll_interval
+        pcount = Roundhouse::ProcessSet.new.size
+        pcount = 1 if pcount == 0
+        pcount * Roundhouse.options[:average_scheduled_poll_interval]
+      end
+
+      def initial_wait
+        begin
+          # Have all processes sleep between 5-15 seconds.  10 seconds
+          # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+          # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
+          sleep(INITIAL_WAIT) unless Roundhouse.options[:poll_interval_average]
+          sleep(5 * rand)
+        rescue Celluloid::TaskTerminated
+          # Hit Ctrl-C when Roundhouse is finished booting and we have a chance
+          # to get here.
+        end
+      end
+
+    end
+  end
+end

data/lib/roundhouse/testing/inline.rb

@@ -0,0 +1,28 @@
+require 'roundhouse/testing'
+
+##
+# The Roundhouse inline infrastructure overrides perform_async so that it
+# actually calls perform instead. This allows workers to be run inline in a
+# testing environment.
+#
+# This is similar to `Resque.inline = true` functionality.
+#
+# Example:
+#
+#   require 'roundhouse/testing/inline'
+#
+#   $external_variable = 0
+#
+#   class ExternalWorker
+#     include Roundhouse::Worker
+#
+#     def perform
+#       $external_variable = 1
+#     end
+#   end
+#
+#   assert_equal 0, $external_variable
+#   ExternalWorker.perform_async
+#   assert_equal 1, $external_variable
+#
+Roundhouse::Testing.inline!

data/lib/roundhouse/testing.rb

@@ -0,0 +1,193 @@
+require 'securerandom'
+require 'roundhouse'
+
+module Roundhouse
+
+  class Testing
+    class << self
+      attr_accessor :__test_mode
+
+      def __set_test_mode(mode)
+        if block_given?
+          current_mode = self.__test_mode
+          begin
+            self.__test_mode = mode
+            yield
+          ensure
+            self.__test_mode = current_mode
+          end
+        else
+          self.__test_mode = mode
+        end
+      end
+
+      def disable!(&block)
+        __set_test_mode(:disable, &block)
+      end
+
+      def fake!(&block)
+        __set_test_mode(:fake, &block)
+      end
+
+      def inline!(&block)
+        __set_test_mode(:inline, &block)
+      end
+
+      def enabled?
+        self.__test_mode != :disable
+      end
+
+      def disabled?
+        self.__test_mode == :disable
+      end
+
+      def fake?
+        self.__test_mode == :fake
+      end
+
+      def inline?
+        self.__test_mode == :inline
+      end
+    end
+  end
+
+  # Default to fake testing to keep old behavior
+  Roundhouse::Testing.fake!
+
+  class EmptyQueueError < RuntimeError; end
+
+  class Client
+    alias_method :raw_push_real, :raw_push
+
+    def raw_push(payloads)
+      if Roundhouse::Testing.fake?
+        payloads.each do |job|
+          job['class'].constantize.jobs << Roundhouse.load_json(Roundhouse.dump_json(job))
+        end
+        true
+      elsif Roundhouse::Testing.inline?
+        payloads.each do |job|
+          job['jid'] ||= SecureRandom.hex(12)
+          klass = job['class'].constantize
+          klass.jobs.unshift Roundhouse.load_json(Roundhouse.dump_json(job))
+          klass.perform_one
+        end
+        true
+      else
+        raw_push_real(payloads)
+      end
+    end
+  end
+
+  module Worker
+    ##
+    # The Roundhouse testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'roundhouse/testing'
+    #
+    #   assert_equal 0, HardWorker.jobs.size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, HardWorker.jobs.size
+    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #
+    #   assert_equal 0, Roundhouse::Extensions::DelayedMailer.jobs.size
+    #   MyMailer.delay.send_welcome_email('foo@example.com')
+    #   assert_equal 1, Roundhouse::Extensions::DelayedMailer.jobs.size
+    #
+    # You can also clear and drain all workers' jobs:
+    #
+    #   assert_equal 0, Roundhouse::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 0, Roundhouse::Extensions::DelayedModel.jobs.size
+    #
+    #   MyMailer.delay.send_welcome_email('foo@example.com')
+    #   MyModel.delay.do_something_hard
+    #
+    #   assert_equal 1, Roundhouse::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 1, Roundhouse::Extensions::DelayedModel.jobs.size
+    #
+    #   Roundhouse::Worker.clear_all # or .drain_all
+    #
+    #   assert_equal 0, Roundhouse::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 0, Roundhouse::Extensions::DelayedModel.jobs.size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Roundhouse::Worker.clear_all
+    #     end
+    #   end
+    #
+    # or for acceptance testing, i.e. with cucumber:
+    #
+    #   AfterStep do
+    #     Roundhouse::Worker.drain_all
+    #   end
+    #
+    #   When I sign up as "foo@example.com"
+    #   Then I should receive a welcome email to "foo@example.com"
+    #
+    module ClassMethods
+
+      # Jobs queued for this worker
+      def jobs
+        Worker.jobs[self]
+      end
+
+      # Clear all jobs for this worker
+      def clear
+        jobs.clear
+      end
+
+      # Drain and run all jobs for this worker
+      def drain
+        while job = jobs.shift do
+          worker = new
+          worker.jid = job['jid']
+          worker.bid = job['bid'] if worker.respond_to?(:bid=)
+          execute_job(worker, job['args'])
+        end
+      end
+
+      # Pop out a single job and perform it
+      def perform_one
+        raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
+        job = jobs.shift
+        worker = new
+        worker.jid = job['jid']
+        worker.bid = job['bid'] if worker.respond_to?(:bid=)
+        execute_job(worker, job['args'])
+      end
+
+      def execute_job(worker, args)
+        worker.perform(*args)
+      end
+    end
+
+    class << self
+      def jobs # :nodoc:
+        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      # Clear all queued jobs across all workers
+      def clear_all
+        jobs.clear
+      end
+
+      # Drain all queued jobs across all workers
+      def drain_all
+        until jobs.values.all?(&:empty?) do
+          jobs.keys.each(&:drain)
+        end
+      end
+    end
+  end
+end

data/lib/roundhouse/util.rb

@@ -0,0 +1,68 @@
+require 'socket'
+require 'securerandom'
+require 'roundhouse/exception_handler'
+require 'roundhouse/core_ext'
+
+module Roundhouse
+  ##
+  # This module is part of Roundhouse core and not intended for extensions.
+  #
+  module Util
+    include ExceptionHandler
+
+    EXPIRY = 60 * 60 * 24
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, { context: last_words })
+      raise ex
+    end
+
+    def logger
+      Roundhouse.logger
+    end
+
+    def redis(&block)
+      Roundhouse.redis(&block)
+    end
+
+    def hostname
+      ENV['DYNO'] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
+    end
+
+    def fire_event(event, reverse=false)
+      arr = Roundhouse.options[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { event: event })
+        end
+      end
+    end
+
+    def want_a_hertz_donut?
+      # what's a hertz donut?
+      # punch!  Hurts, don't it?
+      info = Roundhouse.redis {|c| c.info }
+      if info['connected_clients'].to_i > 1000 && info['hz'].to_i >= 10
+        Roundhouse.logger.warn { "Your Redis `hz` setting is too high at #{info['hz']}.  See mperham/sidekiq#2431.  Set it to 3 in #{info[:config_file]}" }
+        true
+      else
+        Roundhouse.logger.debug { "Redis hz: #{info['hz']}.  Client count: #{info['connected_clients']}" }
+        false
+      end
+    end
+
+  end
+end

data/lib/roundhouse/version.rb

@@ -0,0 +1,3 @@
+module Roundhouse
+  VERSION = "0.1.0"
+end