roundhouse-x 0.1.0

data/lib/roundhouse/middleware/chain.rb

@@ -0,0 +1,149 @@
+module Roundhouse
+  # Middleware is code configured to run before/after
+  # a message is processed.  It is patterned after Rack
+  # middleware. Middleware exists for the client side
+  # (pushing jobs onto the queue) as well as the server
+  # side (when jobs are actually processed).
+  #
+  # To add middleware for the client:
+  #
+  # Roundhouse.configure_client do |config|
+  #   config.client_middleware do |chain|
+  #     chain.add MyClientHook
+  #   end
+  # end
+  #
+  # To modify middleware for the server, just call
+  # with another block:
+  #
+  # Roundhouse.configure_server do |config|
+  #   config.server_middleware do |chain|
+  #     chain.add MyServerHook
+  #     chain.remove ActiveRecord
+  #   end
+  # end
+  #
+  # To insert immediately preceding another entry:
+  #
+  # Roundhouse.configure_client do |config|
+  #   config.client_middleware do |chain|
+  #     chain.insert_before ActiveRecord, MyClientHook
+  #   end
+  # end
+  #
+  # To insert immediately after another entry:
+  #
+  # Roundhouse.configure_client do |config|
+  #   config.client_middleware do |chain|
+  #     chain.insert_after ActiveRecord, MyClientHook
+  #   end
+  # end
+  #
+  # This is an example of a minimal server middleware:
+  #
+  # class MyServerHook
+  #   def call(worker_instance, msg, queue)
+  #     puts "Before work"
+  #     yield
+  #     puts "After work"
+  #   end
+  # end
+  #
+  # This is an example of a minimal client middleware, note
+  # the method must return the result or the job will not push
+  # to Redis:
+  #
+  # class MyClientHook
+  #   def call(worker_class, msg, queue, redis_pool)
+  #     puts "Before push"
+  #     result = yield
+  #     puts "After push"
+  #     result
+  #   end
+  # end
+  #
+  module Middleware
+    class Chain
+      include Enumerable
+      attr_reader :entries
+
+      def initialize_copy(copy)
+        copy.instance_variable_set(:@entries, entries.dup)
+      end
+
+      def each(&block)
+        entries.each(&block)
+      end
+
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *args)
+        remove(klass) if exists?(klass)
+        entries << Entry.new(klass, *args)
+      end
+
+      def prepend(klass, *args)
+        remove(klass) if exists?(klass)
+        entries.insert(0, Entry.new(klass, *args))
+      end
+
+      def insert_before(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || 0
+        entries.insert(i, new_entry)
+      end
+
+      def insert_after(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
+        entries.insert(i+1, new_entry)
+      end
+
+      def exists?(klass)
+        any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        map(&:make_new)
+      end
+
+      def clear
+        entries.clear
+      end
+
+      def invoke(*args)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            yield
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/roundhouse/middleware/i18n.rb

@@ -0,0 +1,41 @@
+#
+# Simple middleware to save the current locale and restore it when the job executes.
+# Use it by requiring it in your initializer:
+#
+#     require 'roundhouse/middleware/i18n'
+#
+module Roundhouse::Middleware::I18n
+  # Get the current locale and store it in the message
+  # to be sent to Roundhouse.
+  class Client
+    def call(worker_class, msg, queue, redis_pool)
+      msg['locale'] ||= I18n.locale
+      yield
+    end
+  end
+
+  # Pull the msg locale out and set the current thread to use it.
+  class Server
+    def call(worker, msg, queue)
+      I18n.locale = msg['locale'] || I18n.default_locale
+      yield
+    ensure
+      I18n.locale = I18n.default_locale
+    end
+  end
+end
+
+Roundhouse.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Roundhouse::Middleware::I18n::Client
+  end
+end
+
+Roundhouse.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Roundhouse::Middleware::I18n::Client
+  end
+  config.server_middleware do |chain|
+    chain.add Roundhouse::Middleware::I18n::Server
+  end
+end

data/lib/roundhouse/middleware/server/active_record.rb

@@ -0,0 +1,13 @@
+module Roundhouse
+  module Middleware
+    module Server
+      class ActiveRecord
+        def call(*args)
+          yield
+        ensure
+          ::ActiveRecord::Base.clear_active_connections!
+        end
+      end
+    end
+  end
+end

data/lib/roundhouse/middleware/server/logging.rb

@@ -0,0 +1,40 @@
+module Roundhouse
+  module Middleware
+    module Server
+      class Logging
+
+        def call(worker, item, queue)
+          Roundhouse::Logging.with_context(log_context(worker, item)) do
+            begin
+              start = Time.now
+              logger.info { "start" }
+              yield
+              logger.info { "done: #{elapsed(start)} sec" }
+            rescue Exception
+              logger.info { "fail: #{elapsed(start)} sec" }
+              raise
+            end
+          end
+        end
+
+        private
+
+        # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+        # attribute to expose the underlying thing.
+        def log_context(worker, item)
+          klass = item['wrapped'.freeze] || worker.class.to_s
+          "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+        end
+
+        def elapsed(start)
+          (Time.now - start).round(3)
+        end
+
+        def logger
+          Roundhouse.logger
+        end
+      end
+    end
+  end
+end
+

data/lib/roundhouse/middleware/server/retry_jobs.rb

@@ -0,0 +1,206 @@
+require 'roundhouse/scheduled'
+require 'roundhouse/api'
+
+module Roundhouse
+  module Middleware
+    module Server
+      ##
+      # Automatically retry jobs that fail in Roundhouse.
+      # Roundhouse's retry support assumes a typical development lifecycle:
+      #
+      #   0. push some code changes with a bug in it
+      #   1. bug causes job processing to fail, roundhouse's middleware captures
+      #      the job and pushes it onto a retry queue
+      #   2. roundhouse retries jobs in the retry queue multiple times with
+      #      an exponential delay, the job continues to fail
+      #   3. after a few days, a developer deploys a fix.  the job is
+      #      reprocessed successfully.
+      #   4. once retries are exhausted, roundhouse will give up and move the
+      #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+      #      manually in the Web UI.
+      #   5. After 6 months on the DJQ, Roundhouse will discard the job.
+      #
+      # A job looks like:
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+      #
+      # The 'retry' option also accepts a number (in place of 'true'):
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+      #
+      # The job will be retried this number of times before giving up. (If simply
+      # 'true', Roundhouse retries 25 times)
+      #
+      # We'll add a bit more data to the job to support retries:
+      #
+      #  * 'queue' - the queue to use
+      #  * 'retry_count' - number of times we've retried so far.
+      #  * 'error_message' - the message from the exception
+      #  * 'error_class' - the exception class
+      #  * 'failed_at' - the first time it failed
+      #  * 'retried_at' - the last time it was retried
+      #  * 'backtrace' - the number of lines of error backtrace to store
+      #
+      # We don't store the backtrace by default as that can add a lot of overhead
+      # to the job and everyone is using an error service, right?
+      #
+      # The default number of retry attempts is 25 which works out to about 3 weeks
+      # of retries. You can pass a value for the max number of retry attempts when
+      # adding the middleware using the options hash:
+      #
+      #   Roundhouse.configure_server do |config|
+      #     config.server_middleware do |chain|
+      #       chain.add Roundhouse::Middleware::Server::RetryJobs, :max_retries => 7
+      #     end
+      #   end
+      #
+      # or limit the number of retries for a particular worker with:
+      #
+      #    class MyWorker
+      #      include Roundhouse::Worker
+      #      roundhouse_options :retry => 10
+      #    end
+      #
+      class RetryJobs
+        include Roundhouse::Util
+
+        DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+        def initialize(options = {})
+          @max_retries = options.fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+        end
+
+        def call(worker, msg, queue)
+          yield
+        rescue Roundhouse::Shutdown
+          # ignore, will be pushed back onto queue during hard_shutdown
+          raise
+        rescue Exception => e
+          # ignore, will be pushed back onto queue during hard_shutdown
+          raise Roundhouse::Shutdown if exception_caused_by_shutdown?(e)
+
+          raise e unless msg['retry']
+          attempt_retry(worker, msg, queue, e)
+        end
+
+        private
+
+        def attempt_retry(worker, msg, queue, exception)
+          max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+
+          msg['queue'] = if msg['retry_queue']
+            msg['retry_queue']
+          else
+            queue
+          end
+
+          # App code can stuff all sorts of crazy binary data into the error message
+          # that won't convert to JSON.
+          m = exception.message[0..10_000]
+          if m.respond_to?(:scrub!)
+            m.force_encoding("utf-8")
+            m.scrub!
+          end
+
+          msg['error_message'] = m
+          msg['error_class'] = exception.class.name
+          count = if msg['retry_count']
+            msg['retried_at'] = Time.now.to_f
+            msg['retry_count'] += 1
+          else
+            msg['failed_at'] = Time.now.to_f
+            msg['retry_count'] = 0
+          end
+
+          if msg['backtrace'] == true
+            msg['error_backtrace'] = exception.backtrace
+          elsif !msg['backtrace']
+            # do nothing
+          elsif msg['backtrace'].to_i != 0
+            msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+          end
+
+          if count < max_retry_attempts
+            delay = delay_for(worker, count)
+            logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+            retry_at = Time.now.to_f + delay
+            payload = Roundhouse.dump_json(msg)
+            Roundhouse.redis do |conn|
+              conn.zadd('retry', retry_at.to_s, payload)
+            end
+          else
+            # Goodbye dear message, you (re)tried your best I'm sure.
+            retries_exhausted(worker, msg)
+          end
+
+          raise exception
+        end
+
+        def retries_exhausted(worker, msg)
+          logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+          begin
+            if worker.roundhouse_retries_exhausted_block?
+              worker.roundhouse_retries_exhausted_block.call(msg)
+            end
+          rescue => e
+            handle_exception(e, { context: "Error calling retries_exhausted for #{worker.class}", job: msg })
+          end
+
+          send_to_morgue(msg) unless msg['dead'] == false
+        end
+
+        def send_to_morgue(msg)
+          Roundhouse.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+          payload = Roundhouse.dump_json(msg)
+          now = Time.now.to_f
+          Roundhouse.redis do |conn|
+            conn.multi do
+              conn.zadd('dead', now, payload)
+              conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+              conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+            end
+          end
+        end
+
+        def retry_attempts_from(msg_retry, default)
+          if msg_retry.is_a?(Fixnum)
+            msg_retry
+          else
+            default
+          end
+        end
+
+        def delay_for(worker, count)
+          worker.roundhouse_retry_in_block? && retry_in(worker, count) || seconds_to_delay(count)
+        end
+
+        # delayed_job uses the same basic formula
+        def seconds_to_delay(count)
+          (count ** 4) + 15 + (rand(30)*(count+1))
+        end
+
+        def retry_in(worker, count)
+          begin
+            worker.roundhouse_retry_in_block.call(count)
+          rescue Exception => e
+            handle_exception(e, { context: "Failure scheduling retry using the defined `roundhouse_retry_in` in #{worker.class.name}, falling back to default" })
+            nil
+          end
+        end
+
+        def exception_caused_by_shutdown?(e, checked_causes = [])
+          # In Ruby 2.1.0 only, check if exception is a result of shutdown.
+          return false unless defined?(e.cause)
+
+          # Handle circular causes
+          checked_causes << e.object_id
+          return false if checked_causes.include?(e.cause.object_id)
+
+          e.cause.instance_of?(Roundhouse::Shutdown) ||
+            exception_caused_by_shutdown?(e.cause, checked_causes)
+        end
+
+      end
+    end
+  end
+end

data/lib/roundhouse/monitor.rb

@@ -0,0 +1,124 @@
+module Roundhouse
+  # This class implements two things:
+  #   1. A queueing semaphore - the fetcher can pop the next available
+  #      exclusive right to something (such as API request with a given
+  #      auth token)
+  #   2. Track which access right is temporarily suspended
+  class Monitor
+    ACTIVE    = 1
+    EMPTY     = 0
+    SUSPENDED = -1
+
+    # This helps catch problems with key names at runtime
+    SEMAPHORE = 'semaphore'.freeze
+    BUCKETS   = 'buckets'.freeze
+    QUEUE     = 'queue'.freeze
+    SCHEDULE  = 'schedule'.freeze
+    STATUS    = 'status'.freeze
+
+    class << self
+      # Find the first active queue
+      # If nothing is in the rotation, then block
+      def pop(conn)
+        loop do
+          _, q_id = conn.brpop(SEMAPHORE)
+          return q_id if queue_status(conn, q_id) == ACTIVE
+        end
+      end
+
+      def push(conn, q_id)
+        return unless queue_status(conn, q_id) == ACTIVE
+        conn.lpush(SEMAPHORE, q_id)
+      end
+
+      # Bulk requeue (push from right). Usually done
+      # via Client, when Roundhouse is terminating
+      def requeue(conn, q_id, jobs)
+        conn.rpush("#{QUEUE}:#{q_id}", jobs)
+      end
+
+      def await_next_job(conn)
+        loop do
+          queue_id = pop(conn)
+          job = pop_job(conn, queue_id)
+          return queue_id, job if job
+          Roundhouse::Monitor.set_queue_is_empty(conn, queue_id)
+        end
+      end
+
+      def pop_job(conn, q_id)
+        conn.rpop("#{QUEUE}:#{q_id}")
+      end
+
+      def push_job(conn, payloads)
+        return schedule(conn, payloads) if payloads.first['at']
+
+        q_id = payloads.first['queue_id']
+        now = Time.now.to_f
+        to_push = payloads.map do |entry|
+          entry['enqueued_at'.freeze] = now
+          Roundhouse.dump_json(entry)
+        end
+        conn.lpush("#{QUEUE}:#{q_id}", to_push)
+
+        maybe_add_to_rotation(conn, q_id)
+      end
+
+      def set_queue_is_empty(conn, q_id)
+        set_queue_status(conn, q_id, EMPTY)
+      end
+
+      def activate(conn, q_id)
+        set_queue_status(conn, q_id, ACTIVE)
+      end
+
+      def suspend(conn, q_id)
+        set_queue_status(conn, q_id, SUSPENDED)
+      end
+
+      def resume(conn, q_id)
+        return unless queue_status(conn, q_id) == SUSPENDED
+        set_queue_status(conn, q_id, ACTIVE)
+        conn.lpush(SEMAPHORE, q_id)
+      end
+
+      def queue_status(conn, q_id)
+        conn.hget(status_bucket(q_id), q_id).to_i || EMPTY
+      end
+
+      def maybe_add_to_rotation(conn, q_id)
+        # NOTE: this really should be written in LUA to make
+        # sure this is set to ACTIVE after pushing it into the
+        # queuing semaphore. Otherwise, race conditions might
+        # creep in giving this queue an unfair advantage.
+        # See: https://github.com/resque/redis-namespace/blob/master/lib/redis/namespace.rb#L403-L413
+        # See: https://www.redisgreen.net/blog/intro-to-lua-for-redis-programmers/
+        return false unless queue_status(conn, q_id) == EMPTY
+        activate(conn, q_id)
+        conn.lpush(SEMAPHORE, q_id)
+      end
+
+      def status_bucket(q_id)
+        "#{STATUS}:#{bucket_num(q_id)}"
+      end
+
+      def bucket_num(q_id)
+        q_id.to_i / 1000
+      end
+
+      private
+
+      def schedule(conn, payloads)
+        conn.zadd(SCHEDULE.freeze, payloads.map do |hash|
+              at = hash.delete('at'.freeze).to_s
+              [at, Roundhouse.dump_json(hash)]
+          end )
+      end
+
+      def set_queue_status(conn, q_id, status)
+        conn.sadd(BUCKETS, bucket_num(q_id))
+        conn.hset(status_bucket(q_id), q_id, status)
+      end
+    end
+  end
+end

data/lib/roundhouse/paginator.rb

@@ -0,0 +1,42 @@
+module Roundhouse
+  module Paginator
+
+    def page(key, pageidx=1, page_size=25, opts=nil)
+      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      total_size = 0
+      items = []
+      starting = pageidx * page_size
+      ending = starting + page_size - 1
+
+      Roundhouse.redis do |conn|
+        type = conn.type(key)
+
+        case type
+        when 'zset'
+          rev = opts && opts[:reverse]
+          total_size, items = conn.multi do
+            conn.zcard(key)
+            if rev
+              conn.zrevrange(key, starting, ending, :with_scores => true)
+            else
+              conn.zrange(key, starting, ending, :with_scores => true)
+            end
+          end
+          [current_page, total_size, items]
+        when 'list'
+          total_size, items = conn.multi do
+            conn.llen(key)
+            conn.lrange(key, starting, ending)
+          end
+          [current_page, total_size, items]
+        when 'none'
+          [1, 0, []]
+        else
+          raise "can't page a #{type}"
+        end
+      end
+    end
+
+  end
+end