queue-bus 0.5.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.DS_Store

data/.rbenv-version

@@ -0,0 +1 @@
+1.9.3-p194

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use 1.9.3-p194@queue-bus --install --create
+export PATH=./bin:$PATH

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem "debugger"
+gem "rake"

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Brian Leonard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown

@@ -0,0 +1,264 @@
+## Resque Bus
+
+This gem uses Redis and Resque to allow simple asynchronous communication between apps.
+
+### Install
+
+To install, include the 'resque-bus' gem and add the following to your Rakefile:
+
+```ruby
+require "queue_bus/tasks"
+```
+
+### Example
+
+Application A can publish an event
+
+```ruby
+# config
+Resque.redis = "192.168.1.1:6379"
+
+# business logic
+QueueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+
+# or do it later
+QueueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+```
+
+Application B is subscribed to events
+
+```ruby
+# config
+Resque.redis = "192.168.1.1:6379"
+
+# initializer
+QueueBus.dispatch("app_b") do
+  # processes event on app_b_default queue
+  # subscribe is short-hand to subscribe to your 'default' queue and this block with process events with the name "user_created"
+  subscribe "user_created" do |attributes|
+    NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+  end
+
+  # processes event on app_b_critical queue
+  # critical is short-hand to subscribe to your 'critical' queue and this block with process events with the name "user_paid"
+  critical "user_paid" do |attributes|
+    CreditCard.charge!(attributes)
+  end
+
+  # you can pass any queue name you would like to process from as well IE: `banana "peeled" do |attributes|`
+
+  # and regexes work as well. note that with the above configuration along with this regex,
+  # the following as well as the corresponding block above would both be executed
+  subscribe /^user_/ do |attributes|
+    Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+  end
+
+  # the above all filter on just the event_type, but you can filter on anything
+  # this would be _any_ event that has a user_id and the page value of homepage regardless of bus_event_type
+  subscribe "my_key", { "user_id" => :present, "page" => "homepage"} do
+    Mixpanel.homepage_action!(attributes["action"])
+  end
+end
+```
+
+Applications can also subscribe within classes using the provided `Subscriber` module.
+
+```ruby
+class SimpleSubscriber
+  include QueueBus::Subscriber
+  subscribe :my_method
+
+  def my_method(attributes)
+    # heavy lifting
+  end
+end
+```
+
+The following is equivalent to the original initializer and shows more options:
+
+```ruby
+class OtherSubscriber
+  include QueueBus::Subscriber
+  application :app_b
+
+  subscribe :user_created
+  subscribe_queue :app_b_critical, :user_paid
+  subscribe_queue :app_b_default, :user_action, :bus_event_type => /^user_/
+  subscribe :homepage_method, :user_id => :present, :page => "homepage"
+
+  def user_created(attributes)
+    NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+  end
+
+  def user_paid(attributes)
+    CreditCard.charge!(attributes)
+  end
+
+  def user_action(attributes)
+    Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+  end
+
+  def homepage_method
+    Mixpanel.homepage_action!(attributes["action"])
+  end
+end
+```
+
+Note: This subscribes when this class is loaded, so it needs to be in your load or otherwise referenced/required during app initialization to work properly.
+
+### Commands
+
+Each app needs to tell Redis about its subscriptions:
+
+    $ rake resquebus:subscribe
+
+The subscription block is run inside a Resque worker which needs to be started for each app.
+
+    $ rake resquebus:setup resque:work
+
+The incoming queue also needs to be processed on a dedicated or all the app servers.
+
+    $ rake resquebus:driver resque:work
+
+If you want retry to work for subscribing apps, you should run resque-scheduler
+
+    $ rake resque:scheduler
+
+### Adapters
+
+QueueBus now supports multiple adapters! By default QueueBus uses Resque but you can now configure your application to use Sidekiq to drive and subscribe the bus.
+
+First be sure to configure QueueBus to use Sidekiq early in your applications' initialization cycle:
+```
+QueueBus.adapter = 'Sidekiq'
+```
+You will be responsible for setting up the queues for your Sidekiq clients however you can get the appropriate queue names with the following tasks:
+For driving applications:
+```
+$ rake resquebus:driver:sidekiq
+```
+For subscribing applications:
+```
+$ rake resquebus:setup:sidekiq
+```
+These tasks will provide the queue_names and some minimal suggestions for starting the client.
+
+Your subscribing applications will still need to also use the appropriate rake task:
+```
+$ rake resquebus:subscribe:sidekiq
+```
+
+At the moment you are expected to include the Sidekiq gem in your own applications.
+
+And yes we are planning on renaming and restructuring the project! Please contact the maintainer if you would like to add a different adapter.
+
+### Heartbeat
+
+We've found it useful to have the bus act like `cron`, triggering timed jobs throughout the system. Resque Bus calls this a heartbeat.
+It uses resque-scheduler to trigger the events. You can enable it in your Rakefile.
+
+```ruby
+# resque.rake
+namespace :resque do
+  task :setup => [:environment] do
+    QueueBus.heartbeat!
+  end
+end
+```
+
+Or add it to your `schedule.yml` directly
+
+```yaml
+resquebus_heartbeat:
+  cron: "* * * * *"
+  class: "::QueueBus::Heartbeat"
+  queue: bus_incoming
+  description: "I publish a heartbeat_minutes event every minute"
+```
+
+It is the equivalent of doing this every minute
+
+```ruby
+seconds = minutes * (60)
+hours   = minutes / (60)
+days    = minutes / (60*24)
+
+now     = Time.at(seconds)
+
+attributes = {}
+
+now = Time.now
+seconds = now.to_i
+QueueBus.publish("hearbeat_minutes", {
+  "epoch_seconds" => seconds,
+  "epoch_minutes" => seconds / 1.minute,
+  "epoch_hours"   => seconds / 1.hour,
+  "epoch_days"    => seconds / 1.day,
+  "minute"        => now.min
+  "hour"          => now.hour
+  "day"           => now.day
+  "month"         => now.month
+  "year"          => now.year
+  "yday"          => now.yday
+  "wday"          => now.wday
+})
+```
+
+This allows you do something like this:
+
+```ruby
+QueueBus.dispatch("app_c") do
+  # runs at 10:20, 11:20, etc
+  subscribe "once_an_hour", 'bus_event_type' => 'heartbeat_minutes', 'minute' => 20 do |attributes|
+    Sitemap.generate!
+  end
+
+  # runs every five minutes
+  subscribe "every_five_minutes", 'bus_event_type' => 'heartbeat_minutes' do |attributes|
+    next unless attributes["epoch_minutes"] % 5 == 0
+    HealthCheck.run!
+  end
+
+  # runs at 8am on the first of every month
+  subscribe "new_month_morning", 'bus_event_type' => 'heartbeat_minutes', 'day' => 1, hour' => 8, 'minute' => 0,  do |attributes|
+    next unless attributes["epoch_minutes"] % 5 == 0
+    Token.old.expire!
+  end
+end
+```
+
+### Local Mode
+
+For development, a local mode is provided and is specified in the configuration.
+
+```ruby
+# config
+QueueBus.local_mode = :standalone
+or
+QueueBus.local_mode = :inline
+```
+
+Standalone mode does not require a separate resquebus:driver task to be running to process the
+incoming queue. Simply publishing to the bus will distribute the incoming events
+to the appropriate application specific queue.  A separate resquebus:work task does
+still need to be run to process these events
+
+Inline mode skips queue processing entirely and directly dispatches the
+event to the appropriate code block.
+
+You can also say `QueueBus.local_mode = :suppress` to turn off publishing altogether.
+This can be helpful inside some sort of migration, for example.
+
+### TODO
+
+* Sidekiq adapter
+* Refactor rake tasks for resque/sidekiq
+* Refactor to a storage adapter for Redis, so we can store subscription info in MySQL or something else
+* Replace local modes with adapters
+* There are a few spots in the code with TODO notes
+* Make this not freak out in development without Redis or when Redis is down
+* We might not actually need to publish in tests
+* Add some rspec helpers for the apps to use: should_ post an event_publish or something along those lines
+* Allow calling resquebus:setup and resquebus:driver together (append to ENV['QUEUES'], don't replace it)
+
+Copyright (c) 2011 Brian Leonard, released under the MIT license

data/Rakefile

@@ -0,0 +1 @@
+$LOAD_PATH.unshift 'lib'

data/lib/queue-bus.rb

@@ -0,0 +1,62 @@
+require "queue_bus/version"
+
+module QueueBus
+
+  autoload :Application,      'queue_bus/application'
+  autoload :Config,           'queue_bus/config'
+  autoload :Dispatch,         'queue_bus/dispatch'
+  autoload :Dispatchers,      'queue_bus/dispatchers'
+  autoload :Driver,           'queue_bus/driver'
+  autoload :Heartbeat,        'queue_bus/heartbeat'
+  autoload :Local,            'queue_bus/local'
+  autoload :Matcher,          'queue_bus/matcher'
+  autoload :Publishing,       'queue_bus/publishing'
+  autoload :Publisher,        'queue_bus/publisher'
+  autoload :Rider,            'queue_bus/rider'
+  autoload :Subscriber,       'queue_bus/subscriber'
+  autoload :Subscription,     'queue_bus/subscription'
+  autoload :SubscriptionList, 'queue_bus/subscription_list'
+  autoload :TaskManager,      'queue_bus/task_manager'
+  autoload :Util,             'queue_bus/util'
+  autoload :Worker,           'queue_bus/worker'
+
+  module Adapters
+    autoload :Base,           'queue_bus/adapters/base'
+    autoload :Data,           'queue_bus/adapters/data'
+  end
+
+  class << self
+
+    include Publishing
+    extend Forwardable
+
+    def_delegators :config, :default_app_key=, :default_app_key,
+                            :default_queue=, :default_queue,
+                            :local_mode=, :local_mode,
+                            :before_publish=, :before_publish_callback,
+                            :logger=, :logger, :log_application, :log_worker,
+                            :hostname=, :hostname,
+                            :adapter=, :adapter,
+                            :incoming_queue=, :incoming_queue,
+                            :redis
+
+    def_delegators :_dispatchers, :dispatch, :dispatchers, :dispatcher_by_key, :dispatcher_execute
+
+    protected
+
+    def reset
+      # used by tests
+      @config = nil
+      @_dispatchers = nil
+    end
+
+    def config
+      @config ||= ::QueueBus::Config.new
+    end
+
+    def _dispatchers
+      @_dispatchers ||= ::QueueBus::Dispatchers.new
+    end
+  end
+
+end

data/lib/queue_bus/adapters/base.rb

@@ -0,0 +1,41 @@
+module QueueBus
+  module Adapters
+    class Base
+      # adapters need to define the NonImplemented methods in this class
+
+      def initialize
+        enabled!
+      end
+
+      def enabled!
+        # called the first time we know we are using this adapter
+        # it would be a good spot to require the libraries you're using
+        raise NotImplementedError
+      end
+
+      def redis
+        # for now, we're always using redis as a storage mechanism so give us one
+        raise NotImplementedError
+      end
+
+      def enqueue(queue_name, klass, hash)
+        # enqueue the given class (Driver/Rider) in your queue
+        raise NotImplementedError
+      end
+
+      def enqueue_at(epoch_seconds, queue_name, klass, hash)
+        # enqueue the given class (Publisher) in your queue to run at given time
+        raise NotImplementedError
+      end
+
+      def setup_heartbeat!
+        # if possible, tell a recurring job system to publish every minute
+        raise NotImplementedError
+      end
+
+      def worker_included(base)
+        # optional method for including more modules in classes that work in the queue
+      end
+    end
+  end
+end

data/lib/queue_bus/adapters/data.rb

@@ -0,0 +1,65 @@
+# a base adapter just for publishing and redis connection
+module QueueBus
+  module Adapters
+    class Data < QueueBus::Adapters::Base
+      def enabled!
+        # nothing to do
+      end
+
+      def redis=(client)
+        @redis = client
+      end
+
+      def redis(&block)
+        raise "no redis instance set" unless @redis
+        block.call(@redis)
+      end
+
+      def enqueue(queue_name, klass, hash)
+        push(queue_name, :class => klass.to_s, :args => [hash])
+      end
+
+      def enqueue_at(epoch_seconds, queue_name, klass, hash)
+        item = delayed_job_to_hash_with_queue(queue_name, klass, [hash])
+        delayed_push(epoch_seconds, item)
+      end
+
+      def setup_heartbeat!(queue_name)
+        raise NotImplementedError
+      end
+
+      protected
+
+      def push(queue, item)
+        watch_queue(queue)
+        self.redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
+      end
+
+      # Used internally to keep track of which queues we've created.
+      # Don't call this directly.
+      def watch_queue(queue)
+        self.redis { |redis| redis.sadd(:queues, queue.to_s) }
+      end
+
+      # Used internally to stuff the item into the schedule sorted list.
+      # +timestamp+ can be either in seconds or a datetime object
+      # Insertion if O(log(n)).
+      # Returns true if it's the first job to be scheduled at that time, else false
+      def delayed_push(timestamp, item)
+        self.redis do |redis|
+          # First add this item to the list for this timestamp
+          redis.rpush("delayed:#{timestamp.to_i}", ::QueueBus::Util.encode(item))
+
+          # Now, add this timestamp to the zsets.  The score and the value are
+          # the same since we'll be querying by timestamp, and we don't have
+          # anything else to store.
+          redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
+        end
+      end
+      
+      def delayed_job_to_hash_with_queue(queue, klass, args)
+        {:class => klass.to_s, :args => args, :queue => queue}
+      end
+    end
+  end
+end

data/lib/queue_bus/application.rb

@@ -0,0 +1,121 @@
+module QueueBus
+  class Application
+
+    class << self
+
+      def all
+        # note the names arent the same as we started with
+        ::QueueBus.redis { |redis| redis.smembers(app_list_key).collect{ |val| new(val) } }
+      end
+    end
+
+    attr_reader :app_key, :redis_key
+
+
+    def initialize(app_key)
+      @app_key = self.class.normalize(app_key)
+      @redis_key = "#{self.class.app_single_key}:#{@app_key}"
+      # raise error if only other chars
+      raise "Invalid application name" if @app_key.gsub("_", "").size == 0
+    end
+
+    def subscribe(subscription_list, log = false)
+      @subscriptions = nil
+
+      if subscription_list == nil || subscription_list.size == 0
+        unsubscribe
+        return true
+      end
+
+      temp_key = "temp_#{redis_key}:#{rand(999999999)}"
+
+      ::QueueBus.redis do |redis|
+        redis_hash = subscription_list.to_redis
+        redis_hash.each do |key, hash|
+          redis.hset(temp_key, key, QueueBus::Util.encode(hash))
+        end
+
+        # make it the real one
+        redis.rename(temp_key, redis_key)
+        redis.sadd(self.class.app_list_key, app_key)
+
+        if log
+          redis.hgetall(redis_key).inspect
+        end
+      end
+
+      true
+    end
+
+    def unsubscribe
+      # TODO: clean up known queues?
+      ::QueueBus.redis do |redis|
+        redis.srem(self.class.app_list_key, app_key)
+        redis.del(redis_key)
+      end
+    end
+
+    def no_connect_queue_names_for(subscriptions)
+      out = []
+      subscriptions.all.each do |sub|
+        queue = "#{app_key}_#{sub.queue_name}"
+        out << queue
+      end
+      out << "#{app_key}_default"
+      out.uniq
+    end
+
+    def subscription_matches(attributes)
+      out = subscriptions.matches(attributes)
+      out.each do |sub|
+        sub.app_key = self.app_key
+      end
+      out
+    end
+
+    def event_display_tuples
+      out = []
+      subscriptions.all.each do |sub|
+        out << [sub.class_name, sub.queue_name, sub.matcher.filters]
+      end
+      out
+    end
+
+    protected
+
+    def self.normalize(val)
+      val.to_s.gsub(/\W/, "_").downcase
+    end
+
+    def self.app_list_key
+      "bus_apps"
+    end
+
+    def self.app_single_key
+      "bus_app"
+    end
+
+    def event_queues
+      ::QueueBus.redis { |redis| redis.hgetall(redis_key) }
+    end
+
+    def subscriptions
+      @subscriptions ||= SubscriptionList.from_redis(read_redis_hash)
+    end
+
+    def read_redis_hash
+      out = {}
+      ::QueueBus.redis do |redis|
+        redis.hgetall(redis_key).each do |key, val|
+          begin
+            out[key] = ::QueueBus::Util.decode(val)
+          rescue ::QueueBus::Util::DecodeException
+            out[key] = val
+          end
+        end
+      end
+      out
+    end
+
+  end
+end

data/lib/queue_bus/config.rb

@@ -0,0 +1,98 @@
+module QueueBus
+  class Config
+    def adapter=val
+      raise "Adapter already set to #{@adapter_instance.class.name}" if @adapter_instance
+      if val.is_a?(Class)
+        @adapter_instance = name_or_klass.new
+      elsif val.is_a?(::QueueBus::Adapters::Base)
+        @adapter_instance = val
+      else
+        class_name = ::QueueBus::Util.classify(val)
+        @adapter_instance = ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
+      end
+      @adapter_instance
+    end
+
+    def adapter
+      return @adapter_instance if @adapter_instance
+      raise "no adapter has been set"
+    end
+
+    def redis(&block)
+      # TODO: could allow setting for non-redis adapters
+      adapter.redis(&block)
+    end
+
+    def default_app_key=val
+      @default_app_key = Application.normalize(val)
+    end
+
+    def default_app_key
+      @default_app_key
+    end
+
+    def default_queue=val
+      @default_queue = val
+    end
+
+    def default_queue
+      @default_queue
+    end
+
+    def local_mode=value
+      @local_mode = value
+    end
+
+    def local_mode
+      @local_mode
+    end
+
+    def incoming_queue=val
+      @incoming_queue = val
+    end
+
+    def incoming_queue
+      @incoming_queue ||= "bus_incoming"
+    end
+
+    def hostname
+      @hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
+    end
+
+    def hostname=val
+      @hostname = val
+    end
+
+    def before_publish=(proc)
+      @before_publish_callback = proc
+    end
+
+    def before_publish_callback(attributes)
+      if @before_publish_callback
+        @before_publish_callback.call(attributes)
+      end
+    end
+
+    def logger
+      @logger
+    end
+
+    def logger=val
+      @logger = val
+    end
+
+    def log_application(message)
+      if logger
+        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+        logger.info("** [#{time}] #$$: QueueBus #{message}")
+      end
+    end
+
+    def log_worker(message)
+      if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
+        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+        puts "** [#{time}] #$$: #{message}"
+      end
+    end
+  end
+end