roundhouse-x 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58561b1a622c1151e562d2b5663f76028f59499f
-  data.tar.gz: 4ae16eb8c3c77399fb530560d12a1fbc39d0e695
+  metadata.gz: e419459121a0e984bdeb39afef3442ff962a37a9
+  data.tar.gz: 8781d7c6f13f23bdc40c2a30311ac73d26476b07
 SHA512:
-  metadata.gz: 734f9b44dc5d7198264ea151163c0eb066d4ea151d88380b7060fd7a2d315251deb913a829de8cd496f95257f14f975d07fbaa62d31fece29aac2ab8a2bd6b1f
-  data.tar.gz: a8de13c124fed036d51df2d58b02a60c07885d1849ef9bbd93d3b1d2fcc9582945fc7fa571cdeec00578354c41dd9b93b9b38d46c990057fe5f7ce4c6e8d0a4f
+  metadata.gz: 5648e6e5e86ab1e55b5c5e1a6e6f3eb2f660ed7013dc67b9e6b6ebf9910ebf9e2f2b2a5f375b94528c44c9b54d61d4b61e2a3c57510211e1d7aba38e2054ac1b
+  data.tar.gz: 0b8591db28a22914c1eb5e1c0ca65fc90988e6f3b91ac623f07c1c2cca86453f03396112696bce5b0ff64fb1d13592e061266407d30909114aa98258a210f63f

data/README.md

@@ -1,23 +1,30 @@
 Roundhouse
 ==============
 
-Roundhouse is based on Sidekiq 3.5.0 (HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d)
+Roundhouse is based on Sidekiq 3.5.0 (`HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d`). It
+dispatches jobs in a very specialized way for a very narrow set of problems. For example,
+if you don't need to manage a large number of rate-limited access to an external resource,
+you probably want to use Sidekiq and not Roundhouse.
 
-It uses the basic framework of a multi-threaded workers and managers to solve a different
-kind of problem. This problem resembles that of a load balancing problem: only one worker
-should be working on a work for a given user. This requires:
+The problem Roundhouse solves resembles that of a load balancing problem: only one worker
+should be working on a work for a given user. Any other worker can work on any other user's
+work, as only as no other worker is working on it. We also want to do this fairly so that every
+user has a chance at getting work done steadily. This requires:
 
-1. A queue for each user
-2. A circular buffer to cycle through all user queues
-3. A way to track how many pieces of jobs to process per user, before moving to the next user's queue.
+1. A queue for each user. (Which often means a lot of queues)
+2. A turntable semaphore to control access to queues in a round-robin fashion
+3. A way to track the status of each queue (active, empty, suspended) and when to push a queue back into the turntable
 
 Other things Roundhouse should be able to do:
 
-1. It should be able to run side-by side with Sidekiq client
+1. It should be able to run side-by side with Sidekiq client.
 2. Be able to suspend, resume, and defer work
 3. Metrics to measure lead times and give estimates on when work might be completed
 4. Coordinate across a number of worker processes, across machines
 
+Roundhouse is in pre-alpha right now, designed for a specialized need we have. You probably don't want to
+try this in production.
+
 Install
 ----------------
 
@@ -40,6 +47,30 @@ end
 APIWorker.perform_async(api_token.id, item.id)
 ```
 
+Why the name 'Roundhouse'?
+--------------------------
+
+The name 'Sidekiq' is obviously a play on the word, 'sidekick', both to mean a companion that
+helps a hero out in the background, and the name of a martial arts kicking technique. It
+is a great name for a background processing manager that works well with Rails apps.
+
+When Roundhouse was first concieved, it was meant as a play on the word 'sidekick'. Since Roundhouse
+controls access to a queue in a round-robin fashion, I thought of a 'roundhouse kick'. Later, while
+implementing the code, I found out that a 'roundhouse' is an old term in the railway industry.
+In the old days, you needed a turntable to move trains around, or to reverse their direction.
+Although in modern times, 'Roundhouse' has come to mean the general facilities to maintain
+trains, this idea of a turntable at the center of a roundhouse is a powerful metaphor for the way
+Roundhouse dispatches work to background workers.
+
+The structure that actually controls access is neither a counting or a binary semaphore. I
+had originally thought to call it a "queing semaphore' but that confuses a lot of people. The
+term 'turntable semaphore' better describes this.
+
+Unfortunately, someone had written a Ruby gem several years ago called RoundhousE. I have no
+idea what that project does, other than that it has something to do with Ruby and .NET. So for
+now, this gem is available as `roundhouse-x`. (Yes, that is quite a creative name).
+
+
 Sidekiq License
 -----------------
 

data/lib/roundhouse.rb

@@ -14,7 +14,6 @@ module Roundhouse
   LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
 
   DEFAULTS = {
-    queues: [],
     labels: [],
     concurrency: 25,
     require: '.',
@@ -33,8 +32,7 @@ module Roundhouse
   }
 
   DEFAULT_WORKER_OPTIONS = {
-    'retry' => true,
-    'queue' => 'default'
+    'retry' => true
   }
 
   def self.❨╯°□°❩╯︵┻━┻
@@ -93,6 +91,16 @@ module Roundhouse
     end
   end
 
+  # Suspends a queue
+  def self.suspend_queue(queue_id)
+    self.redis { |conn| Roundhouse::Monitor.suspend(conn, queue_id) }
+  end
+
+  # Resumes a queue
+  def self.resume_queue(queue_id)
+    self.redis { |conn| Roundhouse::Monitor.resume(conn, queue_id) }
+  end
+
   def self.client_middleware
     @client_chain ||= Middleware::Chain.new
     yield @client_chain if block_given?

data/lib/roundhouse/api.rb

@@ -62,7 +62,7 @@ module Roundhouse
           conn.zcard('retry'.freeze)
           conn.zcard('dead'.freeze)
           conn.scard('processes'.freeze)
-          conn.llen('semaphore'.freeze)
+          conn.llen(Roundhouse::Monitor::TURNTABLE.freeze)
           conn.smembers('processes'.freeze)
           conn.smembers(Roundhouse::Monitor::BUCKETS)
         end

data/lib/roundhouse/cli.rb

@@ -64,7 +64,6 @@ module Roundhouse
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Roundhouse::LICENSE
-      logger.info "Upgrade to Roundhouse Pro for more features and support: http://roundhouse.org" unless defined?(::Roundhouse::Pro)
 
       fire_event(:startup)
 
@@ -102,18 +101,16 @@ module Roundhouse
     end
 
     def self.banner
-%q{         s
-          ss
-     sss  sss         ss
-     s  sss s   ssss sss   ____  _     _      _    _
-     s     sssss ssss     / ___|(_) __| | ___| | _(_) __ _
-    s         sss         \___ \| |/ _` |/ _ \ |/ / |/ _` |
-    s sssss  s             ___) | | (_| |  __/   <| | (_| |
-    ss    s  s            |____/|_|\__,_|\___|_|\_\_|\__, |
-    s     s s                                           |_|
-          s s
-         sss
-         sss }
+%q{
+______                      _ _                          
+| ___ \                    | | |                         
+| |_/ /___  _   _ _ __   __| | |__   ___  _   _ ___  ___ 
+|    // _ \| | | | '_ \ / _` | '_ \ / _ \| | | / __|/ _ \
+| |\ \ (_) | |_| | | | | (_| | | | | (_) | |_| \__ \  __/
+\_| \_\___/ \__,_|_| |_|\__,_|_| |_|\___/ \__,_|___/\___|
+                                                         
+                                                         
+}
     end
 
     def handle_signal(sig)
@@ -262,8 +259,6 @@ module Roundhouse
     end
 
     def validate!
-      options[:queues] << 'default' if options[:queues].empty?
-
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
@@ -303,11 +298,6 @@ module Roundhouse
           opts[:index] = Integer(arg.match(/\d+/)[0])
         end
 
-        o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
-          queue, weight = arg.split(",")
-          parse_queue opts, queue, weight
-        end
-
         o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
           opts[:require] = arg
         end
@@ -368,7 +358,6 @@ module Roundhouse
       if File.exist?(cfile)
         opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
         opts = opts.merge(opts.delete(environment) || {})
-        parse_queues(opts, opts.delete(:queues) || [])
       else
         # allow a non-existent config file so Roundhouse
         # can be deployed by cap with just the defaults.
@@ -381,16 +370,5 @@ module Roundhouse
       end
       opts
     end
-
-    def parse_queues(opts, queues_and_weights)
-      queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
-    end
-
-    def parse_queue(opts, q, weight=nil)
-      [weight.to_i, 1].max.times do
-       (opts[:queues] ||= []) << q
-      end
-      opts[:strict] = false if weight.to_i > 0
-    end
   end
 end

data/lib/roundhouse/fetch.rb

@@ -32,7 +32,7 @@ module Roundhouse
     # a new fetch if the current fetch turned up nothing.
     def fetch
       watchdog('Fetcher#fetch died') do
-        return if Roundhouse::Fetcher.done?
+        logger.debug 'Fetcher terminating in #fetch' and return if Roundhouse::Fetcher.done?
 
         begin
           work = @strategy.retrieve_work
@@ -77,6 +77,7 @@ module Roundhouse
     # its mailbox when shutdown starts.
     def self.done!
       @done = true
+      Roundhouse.logger.debug 'Fetcher: setting done'
     end
 
     def self.reset # testing only
@@ -94,10 +95,11 @@ module Roundhouse
 
   class RoundRobinFetch
     def initialize(options = nil)
+      # ignore options
     end
 
     def retrieve_work
-      work = Roundhouse.redis { |conn| Roundhouse::Monitor.await_next_job(conn) }
+      work = Roundhouse.redis { |conn| Roundhouse::Monitor.maybe_next_job(conn) }
       UnitOfWork.new(*work) if work
     end
 

data/lib/roundhouse/launcher.rb

@@ -46,6 +46,7 @@ module Roundhouse
 
     def stop
       watchdog('Launcher#stop') do
+        logger.debug 'Stopping launcher'
         @done = true
         Roundhouse::Fetcher.done!
         fetcher.terminate if fetcher.alive?
@@ -73,7 +74,6 @@ module Roundhouse
         'pid' => $$,
         'tag' => @options[:tag] || '',
         'concurrency' => @options[:concurrency],
-        'queues' => @options[:queues].uniq,
         'labels' => Roundhouse.options[:labels],
         'identity' => identity,
       }

data/lib/roundhouse/monitor.rb

@@ -1,6 +1,8 @@
+require 'roundhouse/script'
+
 module Roundhouse
   # This class implements two things:
-  #   1. A queueing semaphore - the fetcher can pop the next available
+  #   1. A turntable semaphore - the fetcher can pop the next available
   #      exclusive right to something (such as API request with a given
   #      auth token)
   #   2. Track which access right is temporarily suspended
@@ -10,25 +12,32 @@ module Roundhouse
     SUSPENDED = -1
 
     # This helps catch problems with key names at runtime
-    SEMAPHORE = 'semaphore'.freeze
-    BUCKETS   = 'buckets'.freeze
-    QUEUE     = 'queue'.freeze
-    SCHEDULE  = 'schedule'.freeze
-    STATUS    = 'status'.freeze
+    TURNTABLE   = 'turntable'.freeze
+    IN_ROTATION = 'inrotation'.freeze
+    BUCKETS     = 'buckets'.freeze
+    QUEUE       = 'queue'.freeze
+    SCHEDULE    = 'schedule'.freeze
+    STATUS      = 'status'.freeze
+
+    # Number of seconds to block on turntable
+    TURNTABLE_TIMEOUT = 5
 
     class << self
       # Find the first active queue
-      # If nothing is in the rotation, then block
+      # Return nil if nothing is there. The fetcher is responsible
+      # for polling.
       def pop(conn)
-        loop do
-          _, q_id = conn.brpop(SEMAPHORE)
-          return q_id if queue_status(conn, q_id) == ACTIVE
-        end
+        _, q_id = conn.brpop(TURNTABLE, TURNTABLE_TIMEOUT)
+        conn.srem(IN_ROTATION, q_id)
+        return q_id if queue_status(conn, q_id) == ACTIVE
+        return nil
       end
 
+      # Atomic push
+      # Push queue into turntable if and only if queue status is active
       def push(conn, q_id)
-        return unless queue_status(conn, q_id) == ACTIVE
-        conn.lpush(SEMAPHORE, q_id)
+        # NOTE: this version of redis-namespace has a bug when you do keys: argv: params
+        TURNTABLE_PUSH.call conn, [status_bucket(q_id), TURNTABLE, IN_ROTATION], [q_id]
       end
 
       # Bulk requeue (push from right). Usually done
@@ -37,17 +46,19 @@ module Roundhouse
         conn.rpush("#{QUEUE}:#{q_id}", jobs)
       end
 
-      def await_next_job(conn)
-        loop do
-          queue_id = pop(conn)
-          job = pop_job(conn, queue_id)
-          return queue_id, job if job
-          Roundhouse::Monitor.set_queue_is_empty(conn, queue_id)
-        end
+      def maybe_next_job(conn)
+        queue_id = pop(conn)
+        return nil unless queue_id
+
+        job = pop_job(conn, queue_id)
+        return queue_id, job if job
+        return nil
       end
 
+      # Atomically pop job
+      # If nothing is in the queue, set queue status to empty
       def pop_job(conn, q_id)
-        conn.rpop("#{QUEUE}:#{q_id}")
+        POP_JOB.call conn, ["#{QUEUE}:#{q_id}", status_bucket(q_id)], [q_id]
       end
 
       def push_job(conn, payloads)
@@ -77,25 +88,20 @@ module Roundhouse
       end
 
       def resume(conn, q_id)
-        return unless queue_status(conn, q_id) == SUSPENDED
-        set_queue_status(conn, q_id, ACTIVE)
-        conn.lpush(SEMAPHORE, q_id)
+        RESUME_QUEUE.call conn, [status_bucket(q_id), TURNTABLE, IN_ROTATION], [q_id]
       end
 
       def queue_status(conn, q_id)
         conn.hget(status_bucket(q_id), q_id).to_i || EMPTY
       end
 
+      # Only push onto turntable is status is empty
+      # If empty, push into turntable and set to active
+      # Subsequent job pushes would see the active queue and
+      # not push into turntable
+      # Used after pushing a job, and conditionally putting it into rotation
       def maybe_add_to_rotation(conn, q_id)
-        # NOTE: this really should be written in LUA to make
-        # sure this is set to ACTIVE after pushing it into the
-        # queuing semaphore. Otherwise, race conditions might
-        # creep in giving this queue an unfair advantage.
-        # See: https://github.com/resque/redis-namespace/blob/master/lib/redis/namespace.rb#L403-L413
-        # See: https://www.redisgreen.net/blog/intro-to-lua-for-redis-programmers/
-        return false unless queue_status(conn, q_id) == EMPTY
-        activate(conn, q_id)
-        conn.lpush(SEMAPHORE, q_id)
+        MAYBE_TURNTABLE_PUSH.call conn, [status_bucket(q_id), TURNTABLE, BUCKETS, IN_ROTATION], [q_id, bucket_num(q_id)]
       end
 
       def status_bucket(q_id)
@@ -106,6 +112,51 @@ module Roundhouse
         q_id.to_i / 1000
       end
 
+      # TODO: This needs to be in a lua script and atomic
+      # Need to test how well this scales too. Should be testing about
+      # 10,000 queues
+      def rebuild_turntable!
+        Roundhouse.redis do |conn|
+          buckets = conn.smembers(BUCKETS)
+          queues = conn.pipelined do
+            buckets.each { |bucket| conn.hgetall("#{STATUS}:#{bucket}") }
+          end
+
+          all_queue_ids = queues.map(&:keys).flatten
+          queue_len_res = conn.pipelined do
+            all_queue_ids.each { |queue| conn.llen("#{QUEUE}:#{queue}") }
+          end
+
+          queue_len = all_queue_ids.zip(queue_len_res)
+          status = queues.inject({}) { |a,x| a.merge(x) }
+
+          conn.multi do
+            conn.del(TURNTABLE)
+            queue_len.each do |(q_id, len)|
+              s = status[q_id].to_i
+
+              case s
+              when SUSPENDED then next
+              when ACTIVE then
+                if len > 0
+                  conn.lpush(TURNTABLE, q_id)
+                  conn.sadd(IN_ROTATION, q_id)
+                else
+                  set_queue_status(conn, q_id, EMPTY, false)
+                end
+              when EMPTY then
+                next if len <= 0
+                conn.lpush(TURNTABLE, q_id)
+                conn.sadd(IN_ROTATION, q_id)
+                set_queue_status(conn, q_id, ACTIVE, false)
+              else
+                set_queue_status(conn, q_id, SUSPENDED, false)
+              end
+            end
+          end
+        end
+      end
+
       private
 
       def schedule(conn, payloads)
@@ -115,10 +166,60 @@ module Roundhouse
           end )
       end
 
-      def set_queue_status(conn, q_id, status)
-        conn.sadd(BUCKETS, bucket_num(q_id))
+      def set_queue_status(conn, q_id, status, add_bucket = true)
+        conn.sadd(BUCKETS, bucket_num(q_id)) if add_bucket
         conn.hset(status_bucket(q_id), q_id, status)
       end
-    end
+    end # class << self
+
+    WOLVERINE_CONFIG = Roundhouse::Script::Configuration.new
+
+    LUA_TURNTABLE_PUSH = <<END
+local status = redis.call("HGET", KEYS[1], ARGV[1])
+local in_rotation = redis.call("SISMEMBER", KEYS[3], ARGV[1])
+if status == "#{ACTIVE}" and in_rotation == 0 then
+  redis.call("SADD", KEYS[3], ARGV[1])
+  return redis.call("LPUSH", KEYS[2], ARGV[1])
+end
+return nil
+END
+    LUA_MAYBE_TURNTABLE_PUSH = <<END
+local status = redis.call("HGET", KEYS[1], ARGV[1])
+if status == "#{EMPTY}" or status == nil or status == false then
+  redis.call('SADD', KEYS[3], ARGV[2])
+  redis.call("HSET", KEYS[1], ARGV[1], #{ACTIVE})
+  local in_rotation = redis.call("SISMEMBER", KEYS[4], ARGV[1])
+  if in_rotation == 0 then
+    redis.call('SADD', KEYS[4], ARGV[1])
+    return redis.call("LPUSH", KEYS[2], ARGV[1])
   end
 end
+return nil
+END
+    LUA_POP_JOB = <<END
+local job = redis.call("RPOP", KEYS[1])
+if type(job) == "string" then
+  return job
+end
+redis.call("HSET", KEYS[2], ARGV[1], #{EMPTY})
+return nil
+END
+    LUA_RESUME_QUEUE = <<END
+local status = redis.call("HGET", KEYS[1], ARGV[1])
+if status == "#{SUSPENDED}" then
+  redis.call("HSET", KEYS[1], ARGV[1], #{ACTIVE})
+  local in_rotation = redis.call("SISMEMBER", KEYS[3], ARGV[1])
+  if in_rotation == 0 then
+    redis.call('SADD', KEYS[4], ARGV[1])
+    return redis.call("LPUSH", KEYS[2], ARGV[1])
+  end
+end
+return nil
+END
+    TURNTABLE_PUSH       = Roundhouse::Script.new(LUA_TURNTABLE_PUSH.freeze, name: :turntable_push, config: WOLVERINE_CONFIG)
+    MAYBE_TURNTABLE_PUSH = Roundhouse::Script.new(LUA_MAYBE_TURNTABLE_PUSH.freeze, name: :maybe_turntable_push, config: WOLVERINE_CONFIG)
+    POP_JOB              = Roundhouse::Script.new(LUA_POP_JOB.freeze, name: :pop_job, config: WOLVERINE_CONFIG)
+    RESUME_QUEUE         = Roundhouse::Script.new(LUA_RESUME_QUEUE.freeze, name: :resume_queue, config: WOLVERINE_CONFIG)
+
+  end # Monitor
+end