queue-bus 0.5.0

data/lib/queue_bus/dispatch.rb

@@ -0,0 +1,61 @@
+# Creates a DSL for apps to define their blocks to run for event_types
+
+module QueueBus
+  class Dispatch
+
+    attr_reader :app_key, :subscriptions
+    
+    def initialize(app_key)
+      @app_key = Application.normalize(app_key)
+      @subscriptions = SubscriptionList.new
+    end
+    
+    def size
+      @subscriptions.size
+    end
+    
+    def subscribe(key, matcher_hash = nil, &block)
+      dispatch_event("default", key, matcher_hash, block)
+    end
+    
+    # allows definitions of other queues
+    def method_missing(method_name, *args, &block)
+      if args.size == 1 && block
+        dispatch_event(method_name, args[0], nil, block)
+      elsif args.size == 2 && block
+        dispatch_event(method_name, args[0], args[1], block)
+      else
+        super
+      end
+    end
+    
+    def execute(key, attributes)
+      sub = subscriptions.key(key)
+      if sub
+        sub.execute!(attributes)
+      else
+        # TODO: log that it's not there
+      end
+    end
+
+    def subscription_matches(attributes)
+      out = subscriptions.matches(attributes)
+      out.each do |sub|
+        sub.app_key = self.app_key
+      end
+      out
+    end
+    
+    def dispatch_event(queue, key, matcher_hash, block)
+      # if not matcher_hash, assume key is a event_type regex
+      matcher_hash ||= { "bus_event_type" => key }
+      add_subscription("#{app_key}_#{queue}", key, "::QueueBus::Rider", matcher_hash, block)
+    end
+    
+    def add_subscription(queue_name, key, class_name, matcher_hash = nil, block)
+      sub = Subscription.register(queue_name, key, class_name, matcher_hash, block)
+      subscriptions.add(sub)
+      sub
+    end
+  end
+end

data/lib/queue_bus/dispatchers.rb

@@ -0,0 +1,26 @@
+module QueueBus
+  class Dispatchers
+    def dispatch(app_key=nil, &block)
+      dispatcher = dispatcher_by_key(app_key)
+      dispatcher.instance_eval(&block)
+      dispatcher
+    end
+    
+    def dispatchers
+      @dispatchers ||= {}
+      @dispatchers.values
+    end
+    
+    def dispatcher_by_key(app_key)
+      app_key = Application.normalize(app_key || ::QueueBus.default_app_key)
+      @dispatchers ||= {}
+      @dispatchers[app_key] ||= Dispatch.new(app_key)
+    end
+    
+    def dispatcher_execute(app_key, key, attributes)
+      @dispatchers ||= {}
+      dispatcher = @dispatchers[app_key]
+      dispatcher.execute(key, attributes) if dispatcher
+    end
+  end
+end

data/lib/queue_bus/driver.rb

@@ -0,0 +1,31 @@
+module QueueBus
+  # fans out an event to multiple queues
+  class Driver
+    include ::QueueBus::Worker
+
+    class << self
+      def subscription_matches(attributes)
+        out = []
+        Application.all.each do |app|
+          subs = app.subscription_matches(attributes)
+          out.concat(subs)
+        end
+        out
+      end
+
+      def perform(attributes={})
+        raise "No attributes passed" if attributes.empty?
+
+        ::QueueBus.log_worker("Driver running: #{attributes.inspect}")
+
+        subscription_matches(attributes).each do |sub|
+          ::QueueBus.log_worker("  ...sending to #{sub.queue_name} queue with class #{sub.class_name} for app #{sub.app_key} because of subscription: #{sub.key}")
+
+          bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
+          ::QueueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
+        end
+      end
+    end
+
+  end
+end

data/lib/queue_bus/heartbeat.rb

@@ -0,0 +1,109 @@
+module QueueBus
+  # publishes event about the current time
+  class Heartbeat
+    include ::QueueBus::Worker
+
+    class << self
+
+      def lock_key
+        "bus:heartbeat:lock"
+      end
+
+      def lock_seconds
+        60
+      end
+
+      def lock!
+        now = Time.now.to_i
+        timeout = now + lock_seconds + 2
+
+        ::QueueBus.redis do |redis|
+          # return true if we successfully acquired the lock
+          return timeout if redis.setnx(lock_key, timeout)
+
+          # see if the existing timeout is still valid and return false if it is
+          # (we cannot acquire the lock during the timeout period)
+          return 0 if now <= redis.get(lock_key).to_i
+
+          # otherwise set the timeout and ensure that no other worker has
+          # acquired the lock
+          if now > redis.getset(lock_key, timeout).to_i
+            return timeout
+          else
+            return 0
+          end
+        end
+      end
+
+      def unlock!
+        ::QueueBus.redis { |redis| redis.del(lock_key) }
+      end
+
+
+      def redis_key
+        "bus:heartbeat:timestamp"
+      end
+
+      def environment_name
+        ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["BUS_ENV"]
+      end
+
+      def get_saved_minute!
+        key = ::QueueBus.redis { |redis| redis.get(redis_key) }
+        return nil if key.nil?
+        case environment_name
+        when 'development', 'test'
+          # only 3 minutes in development; otherwise, TONS of events if not run in a while
+          three_ago = Time.now.to_i - 3*60*60
+          key = three_ago if key.to_i < three_ago
+        end
+        return key.to_i
+      end
+
+      def set_saved_minute!(epoch_minute)
+        ::QueueBus.redis { |redis| redis.set(redis_key, epoch_minute) }
+      end
+
+      def perform
+        real_now = Time.now.to_i
+        run_until = lock! - 2
+        return if run_until < real_now
+
+        while((real_now = Time.now.to_i) < run_until)
+          minutes = real_now.to_i / 60
+          last = get_saved_minute!
+          if last
+            break if minutes <= last
+            minutes = last + 1
+          end
+
+          seconds = minutes * (60)
+          hours   = minutes / (60)
+          days    = minutes / (60*24)
+
+          now     = Time.at(seconds)
+
+          attributes = {}
+          attributes["epoch_seconds"] = seconds
+          attributes["epoch_minutes"] = minutes
+          attributes["epoch_hours"]   = hours
+          attributes["epoch_days"]    = days
+
+          attributes["minute"] = now.min
+          attributes["hour"]   = now.hour
+          attributes["day"]    = now.day
+          attributes["month"]  = now.month
+          attributes["year"]   = now.year
+          attributes["yday"]   = now.yday
+          attributes["wday"]   = now.wday
+
+          ::QueueBus.publish("heartbeat_minutes", attributes)
+          set_saved_minute!(minutes)
+        end
+
+        unlock!
+      end
+    end
+
+  end
+end

data/lib/queue_bus/local.rb

@@ -0,0 +1,38 @@
+module QueueBus
+  # only process local queues
+  class Local
+
+    class << self
+      def perform(attributes = {})
+        if ::QueueBus.local_mode == :suppress
+          ::QueueBus.log_worker("Suppressed: #{attributes.inspect}")
+          return  # not doing anything
+        end
+
+        ::QueueBus.log_worker("Local running: #{attributes.inspect}")
+
+        # looking for subscriptions, not queues
+        subscription_matches(attributes).each do |sub|
+           bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
+          to_publish = bus_attr.merge(attributes || {})
+          if ::QueueBus.local_mode == :standalone
+            ::QueueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
+          else  # defaults to inline mode
+            sub.execute!(to_publish)
+          end
+        end
+      end
+
+      # looking directly at subscriptions loaded into dispatcher
+      # so we don't need redis server up
+      def subscription_matches(attributes)
+        out = []
+        ::QueueBus.dispatchers.each do |dispatcher|
+          out.concat(dispatcher.subscription_matches(attributes))
+        end
+        out
+      end
+    end
+
+  end
+end

data/lib/queue_bus/matcher.rb

@@ -0,0 +1,81 @@
+module QueueBus
+  class Matcher
+    SPECIAL_PREPEND = "bus_special_value_"
+    attr_reader :filters
+    def initialize(hash)
+      @filters = encode(hash)
+    end
+    
+    def to_redis
+      @filters
+    end
+    
+    def match?(attribute_name, attributes)
+      mine = filters[attribute_name].to_s
+      return false if mine.size == 0
+      
+      given = attributes[attribute_name]
+      case mine
+      when "#{SPECIAL_PREPEND}key"
+        return true if attributes.has_key?(attribute_name)
+        return false
+      when "#{SPECIAL_PREPEND}blank"
+        return true if given.to_s.strip.size == 0
+        return false
+      when "#{SPECIAL_PREPEND}empty"
+        return false if given == nil
+        return true if given.to_s.size == 0
+        return false
+      when "#{SPECIAL_PREPEND}nil"
+        return true if given == nil
+        return false
+      when "#{SPECIAL_PREPEND}value"
+        return false if given == nil
+        return true
+      when "#{SPECIAL_PREPEND}present"
+        return true if given.to_s.strip.size > 0
+        return false
+      end
+      
+      given = given.to_s
+      
+      return true if mine == given
+      begin
+        # if it's already a regex, don't mess with it
+        # otherwise, it should have start and end line situation
+        if mine[0..6] == "(?-mix:"
+          regex = Regexp.new(mine)
+        else
+          regex = Regexp.new("^#{mine}$")
+        end
+        return !!regex.match(given)
+      rescue
+        return false
+      end
+    end
+    
+    def matches?(attributes)
+      return false if filters.empty?
+      return false if attributes == nil
+      
+      filters.keys.each do |key|
+        return false unless match?(key, attributes)
+      end
+      
+      true
+    end
+    
+    def encode(hash)
+      out = {}
+      hash.each do |key, value|
+        case value
+        when :key, :blank, :nil, :present, :empty, :value
+          value = "#{SPECIAL_PREPEND}#{value}"
+        end
+        out[key.to_s] = value.to_s
+      end      
+      out
+    end
+  end
+end
+

data/lib/queue_bus/publisher.rb

@@ -0,0 +1,23 @@
+module QueueBus
+  # publishes on a delay
+  class Publisher
+    include ::QueueBus::Worker
+
+    class << self
+      def perform(*args)
+        # TODO: move this to just resquebus for fallback
+        if args.size > 1
+          # handles older arguments
+          event_type = args.first
+          attributes = args.last
+        else
+          attributes = args.first
+          event_type = attributes["bus_event_type"]
+        end
+        ::QueueBus.log_worker("Publisher running: #{event_type} - #{attributes.inspect}")
+        ::QueueBus.publish(event_type, attributes)
+      end
+    end
+
+  end
+end

data/lib/queue_bus/publishing.rb

@@ -0,0 +1,80 @@
+module QueueBus
+  module Publishing
+
+    def with_global_attributes(attributes)
+      original_timezone = false
+      original_locale   = false
+
+      if attributes["bus_locale"] && defined?(I18n) && I18n.respond_to?(:locale=)
+        original_locale = I18n.locale if I18n.respond_to?(:locale)
+        I18n.locale = attributes["bus_locale"]
+      end
+
+      if attributes["bus_timezone"] && defined?(Time) && Time.respond_to?(:zone=)
+        original_timezone = Time.zone if Time.respond_to?(:zone)
+        Time.zone = attributes["bus_timezone"]
+      end
+
+      yield
+    ensure
+      I18n.locale = original_locale   unless original_locale   == false
+      Time.zone   = original_timezone unless original_timezone == false
+    end
+
+    def publish_metadata(event_type, attributes={})
+      # TODO: "bus_app_key" => application.app_key ?
+      bus_attr = {"bus_published_at" => Time.now.to_i, "bus_event_type" => event_type}
+      bus_attr["bus_id"]           = "#{Time.now.to_i}-#{generate_uuid}"
+      bus_attr["bus_app_hostname"] = ::QueueBus.hostname
+      bus_attr["bus_locale"]       = I18n.locale.to_s if defined?(I18n) && I18n.respond_to?(:locale) && I18n.locale
+      bus_attr["bus_timezone"]     = Time.zone.name   if defined?(Time) && Time.respond_to?(:zone)   && Time.zone
+      out = bus_attr.merge(attributes || {})
+      ::QueueBus.before_publish_callback(out)
+      out
+    end
+
+    def generate_uuid
+      require 'securerandom' unless defined?(SecureRandom)
+      return SecureRandom.uuid
+
+      rescue Exception => e
+        # secure random not there
+        # big random number a few times
+        n_bytes = [42].pack('i').size
+        n_bits = n_bytes * 8
+        max = 2 ** (n_bits - 2) - 1
+        return "#{rand(max)}-#{rand(max)}-#{rand(max)}"
+    end
+
+    def publish(event_type, attributes = {})
+      to_publish = publish_metadata(event_type, attributes)
+      ::QueueBus.log_application("Event published: #{event_type} #{to_publish.inspect}")
+      if local_mode
+        ::QueueBus::Local.perform(to_publish)
+      else
+        enqueue_to(::QueueBus.incoming_queue, Driver, to_publish)
+      end
+    end
+
+    def publish_at(timestamp_or_epoch, event_type, attributes = {})
+      to_publish = publish_metadata(event_type, attributes)
+      to_publish["bus_delayed_until"] ||= timestamp_or_epoch.to_i
+      to_publish.delete("bus_published_at") unless attributes["bus_published_at"] # will be put on when it actually does it
+
+      ::QueueBus.log_application("Event published:#{event_type} #{to_publish.inspect} publish_at: #{timestamp_or_epoch.to_i}")
+      delayed_enqueue_to(timestamp_or_epoch.to_i, incoming_queue, Publisher, to_publish)
+    end
+
+    def enqueue_to(queue_name, klass, hash)
+      ::QueueBus.adapter.enqueue(queue_name, klass, hash)
+    end
+
+    def delayed_enqueue_to(epoch_seconds, queue_name, klass, hash)
+      ::QueueBus.adapter.enqueue_at(epoch_seconds, queue_name, klass, hash)
+    end
+
+    def heartbeat!
+      ::QueueBus.adapter.setup_heartbeat!(incoming_queue)
+    end
+  end
+end

data/lib/queue_bus/rider.rb

@@ -0,0 +1,28 @@
+module QueueBus
+  # queue'd in each
+  class Rider
+    include ::QueueBus::Worker
+    
+    class << self
+      def perform(attributes = {})
+        sub_key = attributes["bus_rider_sub_key"]
+        app_key = attributes["bus_rider_app_key"]
+        raise "No application key passed" if app_key.to_s == ""
+        raise "No subcription key passed" if sub_key.to_s == ""
+        
+        attributes ||= {}
+        
+        ::QueueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
+        
+        # attributes that should be available
+        # attributes["bus_event_type"]
+        # attributes["bus_app_key"]
+        # attributes["bus_published_at"]
+        # attributes["bus_driven_at"]
+        
+        # (now running with the real app that subscribed)
+        ::QueueBus.dispatcher_execute(app_key, sub_key, attributes.merge("bus_executed_at" => Time.now.to_i))
+      end
+    end
+  end
+end

data/lib/queue_bus/subscriber.rb

@@ -0,0 +1,65 @@
+module QueueBus
+  module Subscriber
+
+    def self.included(base)
+      base.send(:include, ::QueueBus::Worker)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+
+      def application(app_key)
+        @app_key = ::QueueBus::Application.normalize(app_key)
+      end
+
+      def app_key
+        return @app_key if @app_key
+        @app_key = ::QueueBus.default_app_key
+        return @app_key if @app_key
+        # module or class_name
+        val = self.name.to_s.split("::").first
+        @app_key = ::QueueBus::Util.underscore(val)
+      end
+
+      def subscribe(method_name, matcher_hash = nil)
+        queue_name   = nil
+        queue_name ||= self.instance_variable_get(:@queue) || (self.respond_to?(:queue) && self.queue)
+        queue_name ||= ::QueueBus.default_queue
+        queue_name ||= "#{app_key}_default"
+        subscribe_queue(queue_name, method_name, matcher_hash)
+      end
+
+      def subscribe_queue(queue_name, method_name, matcher_hash = nil)
+        klass = self
+        matcher_hash ||= {"bus_event_type" => method_name}
+        sub_key = "#{self.name}.#{method_name}"
+        dispatcher = ::QueueBus.dispatcher_by_key(app_key)
+        dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash, lambda{ |att| klass.perform(att) })
+      end
+
+      def transform(method_name)
+        @transform = method_name
+      end
+
+      def perform(attributes)
+        ::QueueBus.with_global_attributes(attributes) do
+          sub_key = attributes["bus_rider_sub_key"]
+          meth_key = sub_key.split(".").last
+          queue_bus_execute(meth_key, attributes)
+        end
+      end
+
+      def queue_bus_execute(key, attributes)
+        args = attributes
+        args = send(@transform, attributes) if @transform
+        args = [args] unless args.is_a?(Array)
+        if self.respond_to?(:subscriber_with_attributes)
+          me = self.subscriber_with_attributes(attributes)
+        else
+          me = self.new
+        end
+        me.send(key, *args)
+      end
+    end
+  end
+end

data/lib/queue_bus/subscription.rb

@@ -0,0 +1,55 @@
+module QueueBus
+  class Subscription
+
+    class << self
+      def register(queue, key, class_name, matcher, block)
+        Subscription.new(queue, key, class_name, matcher, block)
+      end
+
+      def from_redis(hash)
+        queue_name = hash["queue_name"].to_s
+        key        = hash["key"].to_s
+        class_name = hash["class"].to_s
+        matcher    = hash["matcher"]
+        return nil if key.length == 0 || queue_name.length == 0
+        Subscription.new(queue_name, key, class_name, matcher, nil)
+      end
+
+      def normalize(val)
+        val.to_s.gsub(/\W/, "_").downcase
+      end
+    end
+
+    attr_reader :matcher, :executor, :queue_name, :key, :class_name
+    attr_accessor :app_key  # dyanmically set on return from subscription_matches
+
+    def initialize(queue_name, key, class_name, filters, executor=nil)
+      @queue_name = self.class.normalize(queue_name)
+      @key        = key.to_s
+      @class_name = class_name.to_s
+      @matcher    = Matcher.new(filters)
+      @executor   = executor
+    end
+
+    def execute!(attributes)
+      attributes = attributes.with_indifferent_access if attributes.respond_to?(:with_indifferent_access)
+      ::QueueBus.with_global_attributes(attributes) do
+        executor.call(attributes)
+      end
+    end
+
+    def matches?(attributes)
+      @matcher.matches?(attributes)
+    end
+
+    def to_redis
+      out = {}
+      out["queue_name"] = queue_name
+      out["key"]        = key
+      out["class"]      = class_name
+      out["matcher"]    = matcher.to_redis
+      out
+    end
+
+  end
+end

data/lib/queue_bus/subscription_list.rb

@@ -0,0 +1,53 @@
+module QueueBus
+  class SubscriptionList
+
+    class << self
+      def from_redis(redis_hash)
+        out = SubscriptionList.new
+        
+        redis_hash.each do |key, value|
+          sub = Subscription.from_redis(value)
+          out.add(sub) if sub
+        end
+
+        out
+      end
+    end
+
+    def to_redis
+      out = {}
+      @subscriptions.values.each do |sub|
+        out[sub.key] = sub.to_redis
+      end
+      out
+    end
+    
+    def initialize
+      @subscriptions = {}
+    end
+    
+    def add(sub)
+      @subscriptions[sub.key] = sub
+    end
+    
+    def size
+      @subscriptions.size
+    end
+    
+    def key(key)
+      @subscriptions[key.to_s]
+    end
+    
+    def all
+      @subscriptions.values
+    end
+    
+    def matches(attributes)
+      out = []
+      all.each do |sub|
+        out << sub if sub.matches?(attributes)
+      end
+      out
+    end
+  end
+end