resque-bus 0.2.3

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rbenv-version

@@ -0,0 +1 @@
+1.9.3-p194

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use 1.9.3-p194@resque-bus --install --create
+export PATH=./bin:$PATH

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem "debugger"
+gem "rake"

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Brian Leonard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown

@@ -0,0 +1,152 @@
+## Resque Bus
+
+This gem uses Redis and Resque to allow simple asynchronous communication between apps.
+
+### Example
+
+Application A can publish an event
+
+    # config
+    ResqueBus.redis = "192.168.1.1:6379"
+
+    # business logic
+    ResqueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+    
+    # or do it later
+    ResqueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+
+Application B is subscribed to events
+
+    # config
+    ResqueBus.redis = "192.168.1.1:6379"
+    
+    # initializer
+    ResqueBus.dispatch("app_b") do
+      # processes event on app_b_default queue
+      # subscribe is short-hand to subscribe to your 'default' queue and this block with process events with the name "user_created"
+      subscribe "user_created" do |attributes|
+        NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+      end
+      
+      # processes event on app_b_critical queue
+      # critical is short-hand to subscribe to your 'critical' queue and this block with process events with the name "user_paid"
+      critical "user_paid" do |attributes|
+        CreditCard.charge!(attributes)
+      end
+
+      # you can pass any queue name you would like to process from as well IE: `banana "peeled" do |attributes|`
+      
+      # and regexes work as well. note that with the above configuration along with this regex,
+      # the following as well as the corresponding block above would both be executed
+      subscribe /^user_/ do |attributes|
+        Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+      end
+      
+      # the above all filter on just the event_type, but you can filter on anything
+      # this would be _any_ event that has a user_id and the page value of homepage regardless of bus_event_type
+      subscribe "my_key", { "user_id" => :present, "page" => "homepage"} do
+        Mixpanel.homepage_action!(attributes["action"])
+      end
+    end
+
+Applications can also subscribe within classes using the provided `Subscriber` module. 
+
+    class SimpleSubscriber
+      include ResqueBus::Subscriber
+      subscribe :my_method
+  
+      def my_method(attributes)
+        # heavy lifting
+      end
+    end
+
+The following is equivalent to the original initializer and shows more options:
+
+    class OtherSubscriber
+      include ResqueBus::Subscriber
+      application :app_b
+
+      subscribe :user_created
+      subscribe_queue :app_b_critical, :user_paid
+      subscribe_queue :app_b_default, :user_action, :bus_event_type => /^user_/
+      subscribe :homepage_method, :user_id => :present, :page => "homepage"
+
+      def user_created(attributes)
+        NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+      end
+
+      def user_paid(attributes)
+        CreditCard.charge!(attributes)
+      end
+
+      def user_action(attributes)
+        Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+      end
+
+      def homepage_method
+        Mixpanel.homepage_action!(attributes["action"])
+      end
+    end
+
+
+The subscription block is run inside a Resque worker which needs to be started for each app.
+
+    $ rake resquebus:setup resque:work
+    
+The incoming queue also needs to be processed on a dedicated or all the app servers.
+
+    $ rake resquebus:driver resque:work
+    
+If you want retry to work for subscribing apps, you should run resque-scheduler
+    
+    $ rake resquebus:driver resque:scheduler
+
+### Compatibility
+
+ResqueBus can live along side another instance of Resque that points at a different Redis server.
+    
+    # config
+    Resque.redis = "192.168.1.0:6379"
+    ResqueBus.redis = "192.168.1.1:6379"
+
+If no Redis instance is given specifically, ResqueBus will use the Resque one.
+
+    # config
+    Resque.redis = "192.168.1.0:6379"
+
+That will use the default (resque) namespace which can be helpful for using the tooling. Conflict with queue names are unlikely. You can change the namespace if you like though.
+
+    # config
+    Resque.redis = "192.168.1.0:6379"
+    ResqusBus.redis.namespace = :get_on_the_bus
+
+
+### Local Mode
+
+For development, a local mode is also provided and is specified in the
+configuration.
+
+    # config
+    ResqueBus.local_mode = :standalone
+    or 
+    ResqueBus.local_mode = :inline
+
+Standalone mode does not require a separate resquebus:driver task to be running to process the
+incoming queue. Simply publishing to the bus will distribute the incoming events 
+to the appropriate application specific queue.  A separate resquebus:work task does 
+still need to be run to process these events
+
+Inline mode skips queue processing entirely and directly dispatches the
+event to the appropriate code block. 
+
+
+### TODO
+
+* There are a few spots in the code with TODO notes
+* Make this not freak out in development without Redis or when Redis is down
+* We might not actually need to publish in tests
+* Add some rspec helpers for the apps to use: should_ post an event_publish or something along those lines
+* A synchronous version will be needed for several use cases. Make it so that an event can go in real-time to one subscriber and still be async to the rest.
+* Should this use resque-retry or should they jsut go into the failure queue?
+    
+Copyright (c) 2011 Brian Leonard, released under the MIT license

data/Rakefile

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift 'lib'
+require 'resque/tasks'
+require 'resque_bus/tasks'

data/lib/resque-bus.rb

@@ -0,0 +1,250 @@
+require 'redis/namespace'
+require 'resque'
+
+require "resque_bus/version"
+require 'resque_bus/util'
+require 'resque_bus/matcher'
+require 'resque_bus/subscription'
+require 'resque_bus/subscription_list'
+require 'resque_bus/subscriber'
+require 'resque_bus/application'
+require 'resque_bus/publisher'
+require 'resque_bus/driver'
+require 'resque_bus/local'
+require 'resque_bus/rider'
+require 'resque_bus/dispatch'
+require 'resque_bus/task_manager'
+
+module ResqueBus
+  extend self
+  
+  def default_app_key=val
+    @default_app_key = Application.normalize(val)
+  end
+  
+  def default_app_key
+    @default_app_key
+  end
+  
+  def default_queue=val
+    @default_queue = val
+  end
+  
+  def default_queue
+    @default_queue
+  end
+
+  def hostname
+    @hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
+  end
+  
+  def dispatch(app_key=nil, &block)
+    dispatcher = dispatcher_by_key(app_key)
+    dispatcher.instance_eval(&block)
+    dispatcher
+  end
+  
+  def dispatchers
+    @dispatchers ||= {}
+    @dispatchers.values
+  end
+  
+  def dispatcher_by_key(app_key)
+    app_key = Application.normalize(app_key || default_app_key)
+    @dispatchers ||= {}
+    @dispatchers[app_key] ||= Dispatch.new(app_key)
+  end
+  
+  def dispatcher_execute(app_key, key, attributes)
+    @dispatchers ||= {}
+    dispatcher = @dispatchers[app_key]
+    dispatcher.execute(key, attributes) if dispatcher
+  end
+
+  def local_mode=value
+    @local_mode = value
+  end
+
+  def local_mode
+    @local_mode
+  end
+
+  # Accepts:
+  #   1. A 'hostname:port' String
+  #   2. A 'hostname:port:db' String (to select the Redis db)
+  #   3. A 'hostname:port/namespace' String (to set the Redis namespace)
+  #   4. A Redis URL String 'redis://host:port'
+  #   5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
+  #      or `Redis::Namespace`.
+  def redis=(server)
+    case server
+    when String
+      if server =~ /redis\:\/\//
+        redis = Redis.connect(:url => server, :thread_safe => true)
+      else
+        server, namespace = server.split('/', 2)
+        host, port, db = server.split(':')
+        redis = Redis.new(:host => host, :port => port,
+          :thread_safe => true, :db => db)
+      end
+      namespace ||= default_namespace
+
+      @redis = Redis::Namespace.new(namespace, :redis => redis)
+    when Redis::Namespace
+      @redis = server
+    else
+      @redis = Redis::Namespace.new(default_namespace, :redis => server)
+    end
+  end
+
+  # Returns the current Redis connection. If none has been created, will
+  # create a new one from the Reqsue one (with a different namespace)
+  def redis
+    return @redis if @redis
+    copy = Resque.redis.clone
+    copy.namespace = default_namespace
+    self.redis = copy
+    self.redis
+  end
+  
+  def original_redis=(server)
+    @original_redis = server
+  end
+  def original_redis
+    @original_redis
+  end
+  
+  def publish_metadata(event_type, attributes={})
+    # TODO: "bus_app_key" => application.app_key ?
+    bus_attr = {"bus_published_at" => Time.now.to_i, "created_at" => Time.now.to_i, "bus_event_type" => event_type}
+    bus_attr["bus_id"] ||= "#{Time.now.to_i}-#{generate_uuid}"
+    bus_attr["bus_app_hostname"] = hostname
+    bus_attr.merge(attributes || {})
+  end
+  
+  def generate_uuid
+    require 'securerandom' unless defined?(SecureRandom)
+    return SecureRandom.uuid
+    
+    rescue Exception => e
+      # secure random not there
+      # big random number a few times
+      n_bytes = [42].pack('i').size
+      n_bits = n_bytes * 8
+      max = 2 ** (n_bits - 2) - 1
+      return "#{rand(max)}-#{rand(max)}-#{rand(max)}"
+  end
+  
+  def publish(event_type, attributes = {})
+    to_publish = publish_metadata(event_type, attributes)
+    ResqueBus.log_application("Event published: #{event_type} #{to_publish.inspect}")
+    if local_mode
+      ResqueBus::Local.perform(to_publish)
+    else
+      enqueue_to(incoming_queue, Driver, to_publish)
+    end
+  end
+  
+  def publish_at(timestamp_or_epoch, event_type, attributes = {})
+    to_publish = publish_metadata(event_type, attributes)
+    to_publish["bus_delayed_until"] ||= timestamp_or_epoch.to_i
+    to_publish.delete("bus_published_at") unless attributes["bus_published_at"] # will be put on when it actually does it
+    
+    ResqueBus.log_application("Event published:#{event_type} #{to_publish.inspect} publish_at: #{timestamp_or_epoch.to_i}")
+    item = delayed_job_to_hash_with_queue(incoming_queue, Publisher, [event_type, to_publish])
+    delayed_push(timestamp_or_epoch, item)
+  end
+  
+  def enqueue_to(queue, klass, *args)
+    push(queue, :class => klass.to_s, :args => args)
+  end
+  
+  def logger
+    @logger
+  end
+  
+  def logger=val
+    @logger = val
+  end
+  
+  def log_application(message)
+    if logger
+      time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+      logger.info("** [#{time}] #$$: ResqueBus #{message}")
+    end
+  end
+  
+  def log_worker(message)
+    if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
+      time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+      puts "** [#{time}] #$$: #{message}"
+    end
+  end
+  
+  protected
+  
+  def reset
+    # used by tests
+    @redis = nil # clear instance of redis
+    @dispatcher = nil
+    @default_app_key = nil
+    @default_queue = nil
+  end
+  
+  def incoming_queue
+    "resquebus_incoming"
+  end
+
+  def default_namespace
+    # It might play better on the same server, but overall life is more complicated
+    :resque
+  end
+  
+  ## From Resque, but using a (possibly) different instance of Redis
+  
+  # Pushes a job onto a queue. Queue name should be a string and the
+  # item should be any JSON-able Ruby object.
+  #
+  # Resque works generally expect the `item` to be a hash with the following
+  # keys:
+  #
+  #   class - The String name of the job to run.
+  #    args - An Array of arguments to pass the job. Usually passed
+  #           via `class.to_class.perform(*args)`.
+  #
+  # Example
+  #
+  #   Resque.push('archive', :class => 'Archive', :args => [ 35, 'tar' ])
+  #
+  # Returns nothing
+  def push(queue, item)
+    watch_queue(queue)
+    redis.rpush "queue:#{queue}", Resque.encode(item)
+  end
+  
+  # Used internally to keep track of which queues we've created.
+  # Don't call this directly.
+  def watch_queue(queue)
+    redis.sadd(:queues, queue.to_s)
+  end
+  
+  ### From Resque Scheduler
+  # Used internally to stuff the item into the schedule sorted list.
+  # +timestamp+ can be either in seconds or a datetime object
+  # Insertion if O(log(n)).
+  # Returns true if it's the first job to be scheduled at that time, else false
+  def delayed_push(timestamp, item)
+    # First add this item to the list for this timestamp
+    redis.rpush("delayed:#{timestamp.to_i}", Resque.encode(item))
+
+    # Now, add this timestamp to the zsets.  The score and the value are
+    # the same since we'll be querying by timestamp, and we don't have
+    # anything else to store.
+    redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
+  end
+  
+  def delayed_job_to_hash_with_queue(queue, klass, args)
+    {:class => klass.to_s, :args => args, :queue => queue}
+  end
+  
+end

data/lib/resque_bus/application.rb

@@ -0,0 +1,110 @@
+module ResqueBus
+  class Application
+    attr_reader :app_key, :redis_key
+    
+    def self.all
+      # note the names arent the same as we started with
+      ResqueBus.redis.smembers(app_list_key).collect{ |val| new(val) }
+    end
+    
+    def initialize(app_key)
+      @app_key = self.class.normalize(app_key)
+      @redis_key = "#{self.class.app_single_key}:#{@app_key}"
+      # raise error if only other chars
+      raise "Invalid application name" if @app_key.gsub("_", "").size == 0
+    end
+    
+    def subscribe(subscription_list, log = false)
+      @subscriptions = nil
+      
+      if subscription_list == nil || subscription_list.size == 0
+        unsubscribe
+        return true
+      end
+      
+      temp_key = "temp_#{redis_key}:#{rand(999999999)}"
+      
+      redis_hash = subscription_list.to_redis
+      redis_hash.each do |key, hash|
+        ResqueBus.redis.hset(temp_key, key, Resque.encode(hash))
+      end
+      
+      # make it the real one
+      ResqueBus.redis.rename(temp_key, redis_key)
+      ResqueBus.redis.sadd(self.class.app_list_key, app_key)
+      
+      if log
+        puts ResqueBus.redis.hgetall(redis_key).inspect
+      end
+      
+      true
+    end
+        
+    def unsubscribe
+      # TODO: clean up known queues?
+      ResqueBus.redis.srem(self.class.app_list_key, app_key)
+      ResqueBus.redis.del(redis_key)
+    end
+    
+    def no_connect_queue_names_for(subscriptions)
+      out = []
+      subscriptions.all.each do |sub|
+        queue = "#{app_key}_#{sub.queue_name}"
+        out << queue
+      end
+      out << "#{app_key}_default"
+      out.uniq
+    end
+    
+    def subscription_matches(attributes)
+      out = subscriptions.matches(attributes)
+      out.each do |sub|
+        sub.app_key = self.app_key
+      end
+      out
+    end
+    
+    def event_display_tuples
+      out = []
+      subscriptions.all.each do |sub|
+        out << [sub.event_name, sub.queue_name]
+      end
+      out
+    end
+    
+    protected
+
+    def self.normalize(val)
+      val.to_s.gsub(/\W/, "_").downcase
+    end
+    
+    def self.app_list_key
+      "resquebus_apps"
+    end
+    
+    def self.app_single_key
+      "resquebus_app"
+    end
+    
+    def event_queues
+      ResqueBus.redis.hgetall(redis_key)
+    end
+    
+    def subscriptions
+      @subscriptions ||= SubscriptionList.from_redis(read_redis_hash)
+    end
+    
+    def read_redis_hash
+      out = {}
+      ResqueBus.redis.hgetall(redis_key).each do |key, val|
+        begin
+          out[key] = Resque.decode(val)
+        rescue Resque::Helpers::DecodeException
+          out[key] = val
+        end
+      end
+      out
+    end
+   
+  end
+end

data/lib/resque_bus/dispatch.rb

@@ -0,0 +1,59 @@
+# Creates a DSL for apps to define their blocks to run for event_types
+
+module ResqueBus
+  class Dispatch
+    attr_reader :app_key, :subscriptions
+    def initialize(app_key)
+      @app_key = Application.normalize(app_key)
+      @subscriptions = SubscriptionList.new
+    end
+    
+    def size
+      @subscriptions.size
+    end
+    
+    def subscribe(key, matcher_hash = nil, &block)
+      dispatch_event("default", key, matcher_hash, block)
+    end
+    
+    # allows definitions of other queues
+    def method_missing(method_name, *args, &block)
+      if args.size == 1 and block
+        dispatch_event(method_name, args[0], nil, block)
+      elsif args.size == 2 and block
+        dispatch_event(method_name, args[0], args[1], block)
+      else
+        super
+      end
+    end
+    
+    def execute(key, attributes)
+      sub = subscriptions.key(key)
+      if sub
+        sub.execute!(attributes)
+      else
+        # TODO: log that it's not there
+      end
+    end
+
+    def subscription_matches(attributes)
+      out = subscriptions.matches(attributes)
+      out.each do |sub|
+        sub.app_key = self.app_key
+      end
+      out
+    end
+    
+    def dispatch_event(queue, key, matcher_hash, block)
+      # if not matcher_hash, assume key is a event_type regex
+      matcher_hash ||= { "bus_event_type" => key }
+      add_subscription("#{app_key}_#{queue}", key, "::ResqueBus::Rider", matcher_hash, block)
+    end
+    
+    def add_subscription(queue_name, key, class_name, matcher_hash = nil, block)
+      sub = Subscription.register(queue_name, key, class_name, matcher_hash, block)
+      subscriptions.add(sub)
+      sub
+    end
+  end
+end

data/lib/resque_bus/driver.rb

@@ -0,0 +1,28 @@
+module ResqueBus
+  # fans out an event to multiple queues
+  class Driver
+
+    def self.subscription_matches(attributes)
+      out = []
+      Application.all.each do |app|
+        subs = app.subscription_matches(attributes)
+        out.concat(subs)
+      end
+      out
+    end
+
+    def self.perform(attributes={})
+      raise "No attribiutes passed" if attributes.empty?
+
+      ResqueBus.log_worker("Driver running: #{attributes.inspect}")
+
+      subscription_matches(attributes).each do |sub|
+        ResqueBus.log_worker("  ...sending to #{sub.queue_name} queue with class #{sub.class_name} for app #{sub.app_key} because of subscription: #{sub.key}")
+        
+        bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
+        ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
+      end
+    end
+
+  end
+end

data/lib/resque_bus/local.rb

@@ -0,0 +1,32 @@
+module ResqueBus
+  # only process local queues
+  class Local
+
+    def self.perform(attributes = {})
+      ResqueBus.log_worker("Local running: #{attributes.inspect}")
+
+      # looking for subscriptions, not queues
+      subscription_matches(attributes).each do |sub|
+         bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
+        to_publish = bus_attr.merge(attributes || {})
+        if ResqueBus.local_mode == :standalone
+          ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
+        # defaults to inline mode
+        else ResqueBus.local_mode == :inline
+          sub.execute!(to_publish)
+        end
+      end
+    end
+
+    # looking directly at subscriptions loaded into dispatcher
+    # so we don't need redis server up
+    def self.subscription_matches(attributes)
+      out = []
+      ResqueBus.dispatchers.each do |dispatcher|
+        out.concat(dispatcher.subscription_matches(attributes))
+      end
+      out
+    end
+
+  end
+end