queue-bus 0.5.0

data/lib/queue_bus/task_manager.rb

@@ -0,0 +1,52 @@
+module QueueBus
+  class TaskManager
+
+    attr_reader :logging
+    
+    def initialize(logging)
+      @logging = logging
+    end
+    
+    def subscribe!
+      count = 0
+      ::QueueBus.dispatchers.each do |dispatcher|
+        subscriptions = dispatcher.subscriptions
+        if subscriptions.size > 0
+          count += subscriptions.size
+          log "Subscribing #{dispatcher.app_key} to #{subscriptions.size} subscriptions"
+          app = ::QueueBus::Application.new(dispatcher.app_key)
+          app.subscribe(subscriptions, logging)
+          log "  ...done"
+        end
+      end
+      count
+    end
+    
+    def unsubscribe!
+      count = 0
+      ::QueueBus.dispatchers.each do |dispatcher|
+        log "Unsubcribing from #{dispatcher.app_key}"
+        app = ::QueueBus::Application.new(dispatcher.app_key)
+        app.unsubscribe
+        count += 1
+        log "  ...done"
+      end
+    end
+    
+    def queue_names
+      # let's not talk to redis in here. Seems to screw things up
+      queues = []
+      ::QueueBus.dispatchers.each do |dispatcher|
+        dispatcher.subscriptions.all.each do |sub|
+          queues << sub.queue_name
+        end
+      end
+      
+      queues.uniq
+    end
+    
+    def log(message)
+      puts(message) if logging
+    end
+  end
+end

data/lib/queue_bus/util.rb

@@ -0,0 +1,87 @@
+require 'multi_json'
+
+module QueueBus
+  module Util
+    extend self
+
+    class DecodeException < StandardError; end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.dump object
+      else
+        MultiJson.encode object
+      end
+    end
+
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      return unless object
+
+      begin
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.load object
+        else
+          MultiJson.decode object
+        end
+      rescue ::MultiJson::DecodeError => e
+        raise DecodeException, e.message, e.backtrace
+      end
+    end
+    
+    def underscore(camel_cased_word)
+      word = camel_cased_word.to_s.dup
+      word.gsub!('::', '/')
+      # word.gsub!(/(?:([A-Za-z\d])|^)(#{inflections.acronym_regex})(?=\b|[^a-z])/) { "#{$1}#{$1 && '_'}#{$2.downcase}" }
+      word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
+      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      word.tr!("-", "_")
+      word.downcase!
+      word
+    end
+
+    def classify(table_name)
+      # strip out any leading schema name
+      # camelize(singularize(table_name.to_s.sub(/.*\./, '')))
+      camelize(table_name.to_s.sub(/.*\./, ''))
+    end
+
+    def camelize(term)
+      string = term.to_s
+      # string = string.sub(/^[a-z\d]*/) { inflections.acronyms[$&] || $&.capitalize }
+      string = string.sub(/^[a-z\d]*/) { $&.capitalize }
+      # string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{inflections.acronyms[$2] || $2.capitalize}" }
+      string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
+      string.gsub!(/\//, '::')
+      string
+    end
+    
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        if constant == Object
+          constant.const_get(name)
+        else
+          candidate = constant.const_get(name)
+          next candidate if constant.const_defined?(name, false)
+          next candidate unless Object.const_defined?(name)
+
+          # Go down the ancestors to check it it's owned
+          # directly before we reach Object or the end of ancestors.
+          constant = constant.ancestors.inject do |const, ancestor|
+            break const    if ancestor == Object
+            break ancestor if ancestor.const_defined?(name, false)
+            const
+          end
+
+          # owner is in Object, so raise
+          constant.const_get(name, false)
+        end
+      end
+    end
+  end
+end

data/lib/queue_bus/version.rb

@@ -0,0 +1,3 @@
+module QueueBus
+  VERSION = "0.5.0"
+end

data/lib/queue_bus/worker.rb

@@ -0,0 +1,14 @@
+module QueueBus
+  module Worker
+
+    def self.included(base)
+      ::QueueBus.adapter.worker_included(base)
+    end
+
+    # all our workers include this one
+    def perform(*args)
+      # instance method level support for sidekiq
+      self.class.perform(*args)
+    end
+  end
+end

data/lib/tasks/resquebus.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'queue_bus/tasks'

data/queue-bus.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "queue_bus/version"
+
+Gem::Specification.new do |s|
+  s.name        = "queue-bus"
+  s.version     = QueueBus::VERSION
+  s.authors     = ["Brian Leonard"]
+  s.email       = ["brian@bleonard.com"]
+  s.homepage    = ""
+  s.summary     = %q{A simple event bus on top of background queues}
+  s.description = %q{A simple event bus on top of common background queues. Publish and subscribe to events as they occur using what you already have.}
+
+  s.rubyforge_project = "queue-bus"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("multi_json")
+  s.add_dependency("redis")
+
+  # if using resque
+  # s.add_development_dependency('resque', ['>= 1.10.0', '< 2.0'])
+  # s.add_development_dependency('resque-scheduler', '>= 2.0.1')
+  # s.add_development_dependency('resque-retry')
+
+  s.add_development_dependency("rspec")
+  s.add_development_dependency("timecop")
+  s.add_development_dependency("json_pure")
+end

data/spec/adapter/publish_at_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe "Publishing an event in the future" do
+
+  before(:each) do
+    Timecop.freeze(now)
+    QueueBus.stub(:generate_uuid).and_return("idfhlkj")
+  end
+  after(:each) do
+    Timecop.return
+  end
+  let(:delayed_attrs) { {"bus_delayed_until" => future.to_i,
+                     "bus_id" => "#{now.to_i}-idfhlkj",
+                     "bus_app_hostname" =>  `hostname 2>&1`.strip.sub(/.local/,'')} }
+
+  let(:bus_attrs) { delayed_attrs.merge({"bus_published_at" => worktime.to_i})}
+  let(:now)    { Time.parse("01/01/2013 5:00")}
+  let(:future) { Time.at(now.to_i + 60) }
+  let(:worktime) {Time.at(future.to_i + 1)}
+
+  it "should add it to Redis then to the real queue" do
+    hash = {:one => 1, "two" => "here", "id" => 12 }
+    event_name = "event_name"
+    QueueBus.publish_at(future, event_name, hash)
+
+    schedule = QueueBus.redis { |redis| redis.zrange("delayed_queue_schedule", 0, 1) }
+    schedule.should == [future.to_i.to_s]
+
+    val = QueueBus.redis { |redis| redis.lpop("delayed:#{future.to_i}") }
+    hash = JSON.parse(val)
+
+    hash["class"].should == "QueueBus::Publisher"
+    hash["args"].should == [ {"bus_event_type"=>"event_name", "two"=>"here", "one"=>1, "id" => 12}.merge(delayed_attrs) ]
+    hash["queue"].should == "bus_incoming"
+
+    val = QueueBus.redis { |redis| redis.lpop("queue:bus_incoming") }
+    val.should == nil # nothing really added
+
+    Timecop.freeze(worktime)
+    QueueBus::Publisher.perform(*hash["args"])
+
+    val = QueueBus.redis { |redis| redis.lpop("queue:bus_incoming") }
+    hash = JSON.parse(val)
+    hash["class"].should == "QueueBus::Driver"
+    hash["args"].should == [ {"bus_event_type"=>"event_name", "two"=>"here", "one"=>1, "id" => 12}.merge(bus_attrs) ]
+  end
+
+end

data/spec/adapter/support.rb

@@ -0,0 +1,15 @@
+require 'redis'
+
+def reset_test_adapter
+  QueueBus.send(:reset)
+  QueueBus.adapter = :data
+  QueueBus.adapter.redis = Redis.new
+end
+
+def adapter_under_test_class
+  QueueBus::Adapters::Data
+end
+
+def adapter_under_test_symbol
+  :data
+end

data/spec/adapter_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe "adapter is set" do
+  it "should call it's enabled! method on init" do
+    QueueBus.send(:reset)
+    adapter_under_test_class.any_instance.should_receive(:enabled!)
+    instance = adapter_under_test_class.new
+    QueueBus.send(:reset)
+  end
+
+  it "should be defaulting to Data from spec_helper" do
+    QueueBus.adapter.is_a?(adapter_under_test_class).should == true
+  end
+end

data/spec/application_spec.rb

@@ -0,0 +1,152 @@
+require 'spec_helper'
+
+module QueueBus
+  describe Application do
+    describe ".all" do
+      it "should return empty array when none" do
+        Application.all.should == []
+      end
+      it "should return registered applications when there are some" do
+        Application.new("One").subscribe(test_list(test_sub("fdksjh")))
+        Application.new("Two").subscribe(test_list(test_sub("fdklhf")))
+        Application.new("Three").subscribe(test_list(test_sub("fkld")))
+
+        Application.all.collect(&:app_key).should =~ ["one", "two", "three"]
+
+        Application.new("two").unsubscribe
+        Application.all.collect(&:app_key).should =~ ["one", "three"]
+      end
+    end
+
+    describe ".new" do
+      it "should have a key" do
+        Application.new("something").app_key.should == "something"
+
+        Application.new("some thing").app_key.should == "some_thing"
+        Application.new("some-thing").app_key.should == "some_thing"
+        Application.new("some_thing").app_key.should == "some_thing"
+        Application.new("Some Thing").app_key.should == "some_thing"
+      end
+
+      it "should raise an error if not valid" do
+        lambda {
+          Application.new("")
+        }.should raise_error
+
+        lambda {
+          Application.new(nil)
+        }.should raise_error
+
+        lambda {
+          Application.new("/")
+        }.should raise_error
+      end
+    end
+
+    describe "#read_redis_hash" do
+      it "should handle old and new values" do
+
+        QueueBus.redis { |redis| redis.hset("bus_app:myapp", "new_one", QueueBus::Util.encode("queue_name" => "x", "bus_event_type" => "event_name") ) }
+        QueueBus.redis { |redis| redis.hset("bus_app:myapp", "old_one", "oldqueue_name") }
+        app = Application.new("myapp")
+        val = app.send(:read_redis_hash)
+        val.should == {"new_one" => {"queue_name" => "x", "bus_event_type" => "event_name"}, "old_one" => "oldqueue_name"}
+      end
+    end
+
+    describe "#subscribe" do
+      let(:sub1) { test_sub("event_one", "default") }
+      let(:sub2) { test_sub("event_two", "default") }
+      let(:sub3) { test_sub("event_three", "other") }
+      it "should add array to redis" do
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+        Application.new("myapp").subscribe(test_list(sub1, sub2))
+
+        QueueBus.redis { |redis| redis.hgetall("bus_app:myapp") }.should == {"event_two"=>"{\"queue_name\":\"default\",\"key\":\"event_two\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_two\"}}",
+                                                                  "event_one"=>"{\"queue_name\":\"default\",\"key\":\"event_one\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_one\"}}"}
+        QueueBus.redis { |redis| redis.hkeys("bus_app:myapp") }.should =~ ["event_one", "event_two"]
+        QueueBus.redis { |redis| redis.smembers("bus_apps") }.should =~ ["myapp"]
+      end
+      it "should add string to redis" do
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+        Application.new("myapp").subscribe(test_list(sub1))
+
+        QueueBus.redis { |redis| redis.hgetall("bus_app:myapp") }.should == {"event_one"=>"{\"queue_name\":\"default\",\"key\":\"event_one\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_one\"}}"}
+        QueueBus.redis { |redis| redis.hkeys("bus_app:myapp") }.should =~ ["event_one"]
+        QueueBus.redis { |redis| redis.smembers("bus_apps") }.should =~ ["myapp"]
+      end
+      it "should multiple queues to redis" do
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+        Application.new("myapp").subscribe(test_list(sub1, sub2, sub3))
+        QueueBus.redis { |redis| redis.hgetall("bus_app:myapp") }.should == {"event_two"=>"{\"queue_name\":\"default\",\"key\":\"event_two\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_two\"}}", "event_one"=>"{\"queue_name\":\"default\",\"key\":\"event_one\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_one\"}}",
+                                                                  "event_three"=>"{\"queue_name\":\"other\",\"key\":\"event_three\",\"class\":\"::QueueBus::Rider\",\"matcher\":{\"bus_event_type\":\"event_three\"}}"}
+        QueueBus.redis { |redis| redis.hkeys("bus_app:myapp") }.should =~ ["event_three", "event_two", "event_one"]
+        QueueBus.redis { |redis| redis.smembers("bus_apps") }.should =~ ["myapp"]
+      end
+
+      it "should do nothing if nil or empty" do
+
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+
+        Application.new("myapp").subscribe(nil)
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+
+        Application.new("myapp").subscribe([])
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+      end
+
+      it "should call unsubscribe" do
+        app = Application.new("myapp")
+        app.should_receive(:unsubscribe)
+        app.subscribe([])
+      end
+    end
+
+    describe "#unsubscribe" do
+      it "should remove items" do
+        QueueBus.redis { |redis| redis.sadd("bus_apps", "myapp") }
+        QueueBus.redis { |redis| redis.sadd("bus_apps", "other") }
+        QueueBus.redis { |redis| redis.hset("bus_app:myapp", "event_one", "myapp_default") }
+
+        Application.new("myapp").unsubscribe
+
+        QueueBus.redis { |redis| redis.smembers("bus_apps") }.should == ["other"]
+        QueueBus.redis { |redis| redis.get("bus_app:myapp") }.should be_nil
+      end
+    end
+
+    describe "#subscription_matches" do
+      it "should return if it is there" do
+        Application.new("myapp").subscription_matches("bus_event_type"=>"three").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should == []
+
+        subs = test_list(test_sub("one_x"), test_sub("one_y"), test_sub("one"), test_sub("two"))
+        Application.new("myapp").subscribe(subs)
+        Application.new("myapp").subscription_matches("bus_event_type"=>"three").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should == []
+
+        Application.new("myapp").subscription_matches("bus_event_type"=>"two").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "two", "default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"one").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "one", "default", "::QueueBus::Rider"]]
+      end
+
+      it "should handle * wildcards" do
+        subs = test_list(test_sub("one.+"), test_sub("one"), test_sub("one_.*"), test_sub("two"))
+        Application.new("myapp").subscribe(subs)
+        Application.new("myapp").subscription_matches("bus_event_type"=>"three").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should == []
+
+        Application.new("myapp").subscription_matches("bus_event_type"=>"onex").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "one.+", "default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"one").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "one", "default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"one_x").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "one.+","default", "::QueueBus::Rider"], ["myapp", "one_.*", "default", "::QueueBus::Rider"]]
+      end
+
+      it "should handle actual regular expressions" do
+        subs = test_list(test_sub(/one.+/), test_sub("one"), test_sub(/one_.*/), test_sub("two"))
+        Application.new("myapp").subscribe(subs)
+        Application.new("myapp").subscription_matches("bus_event_type"=>"three").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should == []
+
+        Application.new("myapp").subscription_matches("bus_event_type"=>"onex").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "(?-mix:one.+)", "default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"donex").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "(?-mix:one.+)", "default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"one").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "one" ,"default", "::QueueBus::Rider"]]
+        Application.new("myapp").subscription_matches("bus_event_type"=>"one_x").collect{|s| [s.app_key, s.key, s.queue_name, s.class_name]}.should =~ [["myapp", "(?-mix:one.+)", "default", "::QueueBus::Rider"], ["myapp", "(?-mix:one_.*)", "default", "::QueueBus::Rider"]]
+      end
+    end
+  end
+end

data/spec/config_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+module QueueBus
+  module Adapters
+    class TestOne
+
+    end
+  end
+end
+
+describe "QueueBus config" do
+  it "should set the default app key" do
+    QueueBus.default_app_key.should == nil
+
+    QueueBus.default_app_key = "my_app"
+    QueueBus.default_app_key.should == "my_app"
+
+    QueueBus.default_app_key = "something here"
+    QueueBus.default_app_key.should == "something_here"
+  end
+
+  it "should set the default queue" do
+    QueueBus.default_queue.should == nil
+
+    QueueBus.default_queue = "my_queue"
+    QueueBus.default_queue.should == "my_queue"
+  end
+
+  it "should set the local mode" do
+    QueueBus.local_mode.should == nil
+    QueueBus.local_mode = :standalone
+    QueueBus.local_mode.should == :standalone
+  end
+
+  it "should set the hostname" do
+    QueueBus.hostname.should_not == nil
+    QueueBus.hostname = "whatever"
+    QueueBus.hostname.should == "whatever"
+  end
+
+  it "should set before_publish callback" do
+    QueueBus.before_publish = lambda {|attributes| 42 }
+    QueueBus.before_publish_callback({}).should == 42
+  end
+
+
+  it "should use the default Redis connection" do
+    QueueBus.redis { |redis| redis }.should_not eq(nil)
+  end
+
+  it "should default to given adapter" do
+    QueueBus.adapter.is_a?(adapter_under_test_class).should == true
+
+    # and should raise if already set
+    lambda {
+      QueueBus.adapter = :data
+    }.should raise_error
+  end
+
+  context "with a fresh load" do
+    before(:each) do
+      QueueBus.send(:reset)
+    end
+
+    it "should be able to be set to resque" do
+      QueueBus.adapter = adapter_under_test_symbol
+      QueueBus.adapter.is_a?(adapter_under_test_class).should == true
+
+      # and should raise if already set
+      lambda {
+        QueueBus.adapter = :data
+      }.should raise_error
+    end
+
+    it "should be able to be set to something else" do
+
+      QueueBus.adapter = :test_one
+      QueueBus.adapter.is_a?(QueueBus::Adapters::TestOne).should == true
+    end
+  end
+
+
+end

data/spec/dispatch_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper' 
+
+module QueueBus
+  describe Dispatch do
+    it "should not start with any applications" do
+      Dispatch.new("d").subscriptions.size.should == 0
+    end
+    
+    it "should register code to run and execute it" do
+      dispatch = Dispatch.new("d")
+      dispatch.subscribe("my_event") do |attrs|
+        Runner1.run(attrs)
+      end
+      sub = dispatch.subscriptions.key("my_event")
+      sub.send(:executor).is_a?(Proc).should == true
+
+      Runner.value.should == 0
+      dispatch.execute("my_event", {"bus_event_type" => "my_event", "ok" => true})
+      Runner1.value.should == 1
+      Runner1.attributes.should == {"bus_event_type" => "my_event", "ok" => true}
+      
+    end
+    
+    it "should not crash if not there" do
+      lambda {
+        Dispatch.new("d").execute("fdkjh", "bus_event_type" => "fdkjh")
+      }.should_not raise_error
+    end
+    
+    describe "Top Level" do
+      before(:each) do
+         QueueBus.dispatch("testit") do
+           subscribe "event1" do |attributes|
+             Runner2.run(attributes)
+           end
+
+           subscribe "event2" do
+             Runner2.run({})
+           end
+
+           high "event3" do
+             Runner2.run({})
+           end
+           
+           low /^patt.+ern/ do
+             Runner.run({})
+           end
+         end
+       end
+       
+      it "should register and run" do
+        Runner2.value.should == 0
+        QueueBus.dispatcher_execute("testit", "event2", "bus_event_type" => "event2")
+        Runner2.value.should == 1
+        QueueBus.dispatcher_execute("testit", "event1", "bus_event_type" => "event1")
+        Runner2.value.should == 2
+        QueueBus.dispatcher_execute("testit", "event1", "bus_event_type" => "event1")
+        Runner2.value.should == 3
+      end
+      
+      it "should return the subscriptions" do
+        dispatcher = QueueBus.dispatcher_by_key("testit")
+        subs = dispatcher.subscriptions.all
+        tuples = subs.collect{ |sub| [sub.key, sub.queue_name]}
+        tuples.should =~ [  ["event1", "testit_default"],
+                            ["event2", "testit_default"],
+                            ["event3", "testit_high"],
+                            [ "(?-mix:^patt.+ern)", "testit_low"]
+                         ]
+      end
+    
+    end
+  end
+
+end
+