fairway 0.0.1

data/.rbenv-gemsets

@@ -0,0 +1 @@
+fairway

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,9 @@
+source :rubygems
+
+# Specify your gem's dependencies in fairway.gemspec
+gemspec
+
+group :test do
+  gem "sidekiq"
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    fairway (0.0.1)
+      activesupport
+      hiredis
+      redis
+      redis-namespace
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.2.11)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    celluloid (0.12.4)
+      facter (>= 1.6.12)
+      timers (>= 1.0.0)
+    connection_pool (1.0.0)
+    diff-lcs (1.1.3)
+    facter (1.6.17)
+    hiredis (0.4.5)
+    i18n (0.6.1)
+    multi_json (1.5.0)
+    redis (3.0.2)
+    redis-namespace (1.2.1)
+      redis (~> 3.0.0)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.2)
+    sidekiq (2.7.2)
+      celluloid (~> 0.12.0)
+      connection_pool (~> 1.0)
+      multi_json (~> 1)
+      redis (~> 3)
+      redis-namespace
+    timers (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fairway!
+  rspec
+  sidekiq

data/README.markdown

@@ -0,0 +1,123 @@
+# Fairway (redis backbone)
+
+## Responsibilities
+
+* Allow consumption of messages from various sources
+* Publish messages for services who are listening
+* adds messages to queues which services have registered
+* keeps track of which facets have messages queued
+* allows messages to be pulled off of queues in round-robin by facet
+
+### Queue structure
+
+Once a service registers a queue, with a regex for what messages should be
+added to the queue, the backbone will begin pushing matching messages onto the queue.
+
+In some cases, queuing systems can have problems in multi-user systems. If one user
+queues up a ton of messages, messages for other users may be delayed.
+
+Driver solves that by allowing "facets" for each queue, which can be set to a facet
+for each user.
+
+Each facet is added to a list if it has messages waiting to be processed. This list
+is used to enforce a round-robin stategy for pulling messages off of the queue. This
+means we'll process one message for every facet which has messages queued, before
+looping back and processing additional messages.
+
+### Redis LUA scripting
+
+Driver uses [LUA scripts](http://redis.io/commands/eval) inside of redis heavily. This is for a few reasons:
+
+* There is complex logic that can't be expressed in normal redis commands.
+* Each script contains many redis commands and it's important that these
+commands are processed atomically.  A LUA script does that.
+* Since the script is run inside of redis, once the script has started,
+there's very low latency for each redis command.  So, the script executes
+much faster than if we made each call independantly over the network.
+
+This means your Redis version must be `>= 2.6.0`
+
+### Usage
+
+ Add driver to your Gemfile
+
+    gem 'driver', git: 'git@github.com:customerio/driver.git'
+
+Make sure to `bundle install`.
+
+##### Configure driver
+
+    Driver.configure do |config|
+      config.redis     = { host: "yourserver.com", port: 6379 }
+      config.namespace = "letsdrive"
+
+      config.facet do |message|
+        message[:user_id]
+      end
+
+      config.topic do |message|
+        "#{message[:user_id]}:#{message[:type]}"
+      end
+    end
+
+If you don't configure, it'll default to:
+
+    Driver.configure do |config|
+      config.redis     = { host: "localhost", port: 6379 }
+      config.namespace = nil
+
+      config.facet do |message|
+        message[:facet]
+      end
+
+      config.topic do |message|
+        message[:topic]
+      end
+    end
+
+##### Create an instance of driver
+
+    driver = Driver::Client.new
+
+##### Send messages
+
+    driver.deliver(facet: 1, type: :page, name: "http://customer.io/blog", referrer: "http://customer.io")
+
+You can pass any hash of data you'd like. Using the default configuration, this message will have a topic
+of `page`, which can useful if you'd like to listen for, or process, messages.
+
+##### Listen for messages
+
+If a message is sent in the middle of the forest, and no one is listening, was it ever really sent?
+
+You can listen for messages that are delivered by driver by subscribing to message topics:
+
+    driver.redis.psubscribe("page:*google*") do |on|
+      on.pmessage do |pattern, channel, message|
+        puts "[#{channel}] #{message}"
+      end
+    end
+
+If you've configured your topic to be `"#{message[:type]}:#{message[:name]}`, this will listen for any page events with google in the name.
+
+Now, if you deliver a message, it'll be printed out on the console.
+
+*Note:* redis psubscribe is blocking. So, you'll need multiple console windows open.
+One to deliver the message, and one to listen for them.
+
+##### Create a queue
+
+Ok, so now you can listen to messages, but what if your listener dies and you miss all your important messages?
+
+Not to worry, you can tell driver to queue up any messages you want to know about.
+
+    driver.register_queue("myqueue", ".*:page:.*google.*")
+
+Now driver will deliver all page events with google in the name to the queue named `myqueue`. To retrieve messages
+from your queue:
+
+    message = driver.pull("myqueue")
+
+This will return a message or nil (if no messages are queued).
+
+*Note:* `pull` is facet aware, and will rotate through all facets with queued messages.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/boot.rb

@@ -0,0 +1,7 @@
+require "rubygems"
+
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.join(File.dirname(__FILE__), "Gemfile")
+
+require "bundler/setup"
+Bundler.require(:default)

data/fairway.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fairway/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "fairway"
+  gem.version       = Fairway::VERSION
+  gem.authors       = ["John Allison"]
+  gem.email         = ["john@customer.io"]
+  gem.description   = "A fair way to queue work in multi-user systems."
+  gem.summary       = "A fair way to queue work in multi-user systems."
+  gem.homepage      = "https://github.com/customerio/fairway"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency("activesupport")
+  gem.add_dependency("redis")
+  gem.add_dependency("hiredis")
+  gem.add_dependency("redis-namespace")
+end

data/lib/fairway/channeled_connection.rb

@@ -0,0 +1,17 @@
+module Fairway
+  class ChanneledConnection
+    def initialize(connection, &block)
+      @connection = connection
+      @block = block
+    end
+
+    def deliver(message)
+      channel = @block.call(message)
+      @connection.deliver(message, channel)
+    end
+
+    def scripts
+      @connection.scripts
+    end
+  end
+end

data/lib/fairway/config.rb

@@ -0,0 +1,57 @@
+module Fairway
+  class Config
+    attr_accessor :namespace
+    attr_reader :queues
+
+    DEFAULT_FACET = "default"
+
+    QueueDefinition = Struct.new(:name, :channel)
+
+    def initialize
+      @redis_options = {}
+      @namespace = nil
+      @facet = lambda { |message| DEFAULT_FACET }
+      @queues = []
+      yield self if block_given?
+    end
+
+    def facet(&block)
+      if block_given?
+        @facet = block
+      else
+        @facet
+      end
+    end
+
+    def register_queue(name, channel = Connection::DEFAULT_CHANNEL)
+      @queues << QueueDefinition.new(name, channel)
+    end
+
+    def redis=(options)
+      @redis_options = options
+    end
+
+    def redis
+      @redis ||= Redis::Namespace.new(@namespace, redis: raw_redis)
+    end
+
+    def scripts
+      @scripts ||= Scripts.new(raw_redis, scripts_namespace)
+    end
+
+  private
+
+    def scripts_namespace
+      if @namespace.blank?
+        ""
+      else
+        "#{@namespace}:"
+      end
+    end
+
+    def raw_redis
+      @raw_redis ||= Redis.new(@redis_options.merge(hiredis: true))
+    end
+
+  end
+end

data/lib/fairway/connection.rb

@@ -0,0 +1,28 @@
+require "fairway/scripts"
+
+module Fairway
+  class Connection
+    DEFAULT_CHANNEL = "default"
+
+    def initialize(config = Fairway.config)
+      @config = config
+
+      @config.queues.each do |queue|
+        scripts.fairway_register_queue(queue.name, queue.channel)
+      end
+    end
+
+    def deliver(message, channel = DEFAULT_CHANNEL)
+      scripts.fairway_deliver(
+        channel,
+        @config.facet.call(message),
+        message.to_json
+      )
+    end
+
+    def scripts
+      @config.scripts
+    end
+
+  end
+end

data/lib/fairway/queue_reader.rb

@@ -0,0 +1,12 @@
+module Fairway
+  class QueueReader
+    def initialize(connection, *queue_names)
+      @connection = connection
+      @queue_names = [queue_names].flatten!
+    end
+
+    def pull
+      @connection.scripts.fairway_pull(@queue_names)
+    end
+  end
+end

data/lib/fairway/scripts.rb

@@ -0,0 +1,43 @@
+require "pathname"
+require "digest/sha1"
+
+module Fairway
+  class Scripts
+    def self.script_shas
+      @script_shas ||= {}
+    end
+
+    def initialize(redis, namespace)
+      @redis = redis
+      @namespace = namespace
+    end
+
+    def method_missing(method_name, *args)
+      loaded = false
+      @redis.evalsha(script_sha(method_name), [@namespace], args)
+    rescue Redis::CommandError => ex
+      if ex.message.include?("NOSCRIPT") && !loaded
+        @redis.script(:load, script_source(method_name))
+        loaded = true
+        retry
+      else
+        raise
+      end
+    end
+
+  private
+
+    def script_sha(name)
+      self.class.script_shas[name] ||= Digest::SHA1.hexdigest(script_source(name))
+    end
+
+    def script_source(name)
+      script_path(name).read
+    end
+
+    def script_path(name)
+      Pathname.new(__FILE__).dirname.join("../../redis/#{name}.lua")
+    end
+
+  end
+end

data/lib/fairway/sidekiq/composite_fetch.rb

@@ -0,0 +1,37 @@
+module Fairway
+  module Sidekiq
+    class CompositeFetch
+      attr_reader :fetches
+
+      def initialize(fetches)
+        @fetches = []
+
+        fetches.each do |fetch, weight|
+          [weight.to_i, 1].max.times do
+            @fetches << fetch
+          end
+        end
+      end
+
+      def fetch_order
+        fetches.shuffle.uniq
+      end
+
+      def retrieve_work
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work"
+
+        fetch_order.each do |fetch|
+          work = fetch.retrieve_work
+
+          if work
+            ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
+            return work
+          end
+        end
+
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+        return nil
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq/fetcher.rb

@@ -0,0 +1,37 @@
+module Fairway
+  module Sidekiq
+    class Fetcher < ::Sidekiq::Fetcher
+      attr_reader :mgr, :strategy
+
+      def initialize(mgr, fetch)
+        @mgr = mgr
+        @strategy = fetch
+      end
+
+      def done!
+        @done = true
+      end
+
+      def fetch
+        watchdog('Fetcher#fetch died') do
+          return if @done
+
+          begin
+            work = @strategy.retrieve_work
+
+            if work
+              @mgr.async.assign(work)
+            else
+              after(TIMEOUT) { fetch }
+            end
+          rescue => ex
+            logger.error("Error fetching message: #{ex}")
+            logger.error(ex.backtrace.first)
+            sleep(TIMEOUT)
+            after(0) { fetch }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq/fetcher_factory.rb

@@ -0,0 +1,23 @@
+module Fairway
+  module Sidekiq
+    class FetcherFactory
+      def initialize(fetch)
+        @fetch = fetch
+      end
+
+      def done!
+        @fetcher.done!
+      end
+
+      def strategy
+        # This is only used for ::Sidekiq::BasicFetch.bulk_requeue
+        # which is the same for us.
+        ::Sidekiq::BasicFetch
+      end
+
+      def new(mgr, options)
+        @fetcher = Fetcher.new(mgr, @fetch)
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq/non_blocking_fetch.rb

@@ -0,0 +1,64 @@
+module Fairway
+  module Sidekiq
+    class NonBlockingFetch < ::Sidekiq::BasicFetch
+      attr_reader :queues
+
+      def initialize(options)
+        @queues = options[:queues].map { |q| "queue:#{q}" }
+      end
+
+      def retrieve_work
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work #{queues_cmd}"
+
+        work = ::Sidekiq.redis do |conn|
+          script = <<-SCRIPT
+            -- take advantage of non-blocking scripts
+            for i = 1, #KEYS do
+              local work = redis.call('rpop', KEYS[i]);
+
+              if work then
+                return {KEYS[i], work};
+              end
+            end
+
+            return nil;
+          SCRIPT
+
+          conn.eval(script, queues_cmd.map{|q| "#{namespace}#{q}"})
+        end
+
+        if (work)
+          ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
+          work = UnitOfWork.new(*work)
+        else
+          ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+        end
+
+        work
+      end
+
+      def queues_cmd
+        @queues.shuffle.uniq
+      end
+
+      def namespace
+        @namespace ||= begin
+          namespaces = []
+
+          ::Sidekiq.redis do |conn|
+            wrapper = conn
+
+            while defined?(wrapper.redis)
+              namespaces.unshift(wrapper.namespace)
+              wrapper = wrapper.redis
+            end
+
+            namespace = namespaces.join(":")
+            namespace += ":" unless namespace.blank?
+            namespace
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq/queue_fetch.rb

@@ -0,0 +1,33 @@
+require "sidekiq/fetch"
+
+module Fairway
+  module Sidekiq
+    class QueueFetch < ::Sidekiq::BasicFetch
+      def initialize(queue_reader, &block)
+        @queue_reader = queue_reader
+        @message_to_job = block if block_given?
+      end
+
+      def retrieve_work
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work"
+        unit_of_work = nil
+
+        fairway_queue, work = @queue_reader.pull
+
+        if work
+          decoded_work = JSON.parse(work)
+          work = @message_to_job.call(fairway_queue, decoded_work).to_json if @message_to_job
+          unit_of_work = UnitOfWork.new(decoded_work["queue"], work)
+        end
+
+        if unit_of_work
+          ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
+        else
+          ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+        end
+
+        unit_of_work
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq.rb

@@ -0,0 +1,20 @@
+require "sidekiq"
+require "sidekiq/manager"
+
+require "fairway/sidekiq/composite_fetch"
+require "fairway/sidekiq/fetcher"
+require "fairway/sidekiq/fetcher_factory"
+require "fairway/sidekiq/non_blocking_fetch"
+require "fairway/sidekiq/queue_fetch"
+
+# conn         = Fairway::Connection.new
+# queue_reader = Fairway::QueueReader.new(conn, "fairway")
+#
+# queue_fetch  = Fairway::Sidekiq::QueueFetch.new(queue_reader) do |message|
+#   # Transform message into a sidekiq job
+#   message
+# end
+# 
+# non_blocking_fetch = Fairway::Sidekiq::NonBlockingFetch.new(Sidekiq.options)
+# fetch              = Fairway::Sidekiq::CompositeFetch.new(queue_fetch => 1, non_blocking_fetch => 1)
+# Sidekiq.options[:fetcher] = Fairway::Sidekiq::FetcherFactory.new(fetch)

data/lib/fairway/version.rb

@@ -0,0 +1,3 @@
+module Fairway
+  VERSION = "0.0.1"
+end

data/lib/fairway.rb

@@ -0,0 +1,22 @@
+require "fairway/version"
+
+require "active_support/core_ext"
+require "redis"
+require "hiredis"
+require "redis-namespace"
+
+require "fairway/config"
+require "fairway/scripts"
+require "fairway/channeled_connection"
+require "fairway/connection"
+require "fairway/queue_reader"
+
+module Fairway
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure
+    yield(config) if block_given?
+  end
+end

data/redis/fairway_deliver.lua

@@ -0,0 +1,25 @@
+local namespace = KEYS[1];
+local topic     = ARGV[1];
+local facet     = ARGV[2];
+local message   = ARGV[3];
+
+local registered_queues_key = namespace .. 'registered_queues';
+local registered_queues     = redis.call('hgetall', registered_queues_key);
+
+for i = 1, #registered_queues, 2 do
+  local queue_name    = registered_queues[i];
+  local queue_message = registered_queues[i+1];
+
+  if string.find(topic, queue_message) then
+    local active_facets = namespace .. queue_name .. ':active_facets';
+    local facet_queue   = namespace .. queue_name .. ':facet_queue';
+
+    redis.call('lpush', namespace .. queue_name .. ':' .. facet, message)
+
+    if redis.call('sadd', active_facets, facet) == 1 then
+      redis.call('lpush', facet_queue, facet);
+    end
+  end
+end
+
+redis.call('publish', namespace .. topic, message);

data/redis/fairway_pull.lua

@@ -0,0 +1,21 @@
+local namespace = KEYS[1];
+
+for index, queue_name in ipairs(ARGV) do
+  local active_facets = namespace .. queue_name .. ':active_facets';
+  local facet_queue   = namespace .. queue_name .. ':facet_queue';
+
+  local facet = redis.call('rpop', facet_queue);
+
+  if facet then
+    local message_queue = namespace .. queue_name .. ':' .. facet;
+    local message = redis.call('rpop', message_queue);
+
+    if redis.call('llen', message_queue) == 0 then
+      redis.call('srem', active_facets, facet);
+    else
+      redis.call('lpush', facet_queue, facet);
+    end
+
+    return {queue_name, message};
+  end
+end

data/redis/fairway_register_queue.lua

@@ -0,0 +1,5 @@
+local hash          = KEYS[1] .. 'registered_queues';
+local queue_name    = ARGV[1];
+local queue_message = ARGV[2];
+
+redis.call('hset', hash, queue_name, queue_message);