dakwak 1.0.3 → 1.6.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 71fc14535973c5eeecdc876e3cb5c067599f044fa390c647fbafc0244e1e5c45
+  data.tar.gz: 79ffd7b76486f20b769074099e18188c48a425763c4571b1dd68d9850bb6809e
+SHA512:
+  metadata.gz: 3c589b38d8eb433f629e828d6eae6068d78a1f3ae3aa5216abb137417e87713a83e4186518c040197fbb808c8e130e6ffc9ed680579d61430fa699f03e9a0abf
+  data.tar.gz: 5f06e1f3a4f11b27d1743e1af801d930295ed24784cd297146daf2725e118bd8684c9ce59eb30e55be0389cc3aeb4889a33760cb291b4a587735c1e4343ba974

data/lib/dakwak/analytics/sheet.rb

@@ -0,0 +1,77 @@
+require 'json'
+
+module Dakwak
+  module Analytics # :nodoc:
+
+    ##
+    # An analytics sheet is a container of stats that can be submitted
+    # to the analytics platform, dakana.
+    #
+    # The usage is very simple. [] and []= operators are overloaded
+    # to allow you to access sheets just like you would a Hash. When
+    # your done collecting stats (the work is over), submit it using
+    # #commit!
+    #
+    # Here's an example:
+    #
+    #   class FlowerHunter
+    #     def initialize
+    #       # Prepare the sheet here
+    #       @sheet = Sheet.new({
+    #         :zombie_threat_eradicated => false,
+    #         :flowers  => {
+    #           :watered => 0,
+    #           :plucked => 0
+    #         }
+    #       })
+    #       super()
+    #     end
+    # 
+    #      def tend_to_flowers()
+    #        # water a flower
+    #        @sheet[:flowers][:watered] += 1
+    #        # pluck 5 for your special someone
+    #        @sheet[:flowers][:plucked] = 5
+    #      end
+    # 
+    #      def kill_zombies()
+    #        # ...
+    #        # BFG picked up
+    #        @sheet[:zombie_threat_eradicated] = true
+    #        # our work is done, let's submit the stats
+    #        @sheet.commit!
+    #      end
+    #   end    
+    class Sheet
+      attr_accessor :content # :nodoc:
+
+      # Convenience constructor for building a sheet with initial stats
+      def initialize(content = {})
+        @content = content
+      end
+
+      # Returns the value of the stat titled _key_ (if any)
+      def [](key)
+        @content[key]
+      end
+
+      # Defines a new stat as _key_ with any kind of value in _value_
+      def []=(key, value)
+        @content[key] = value
+      end
+
+      # Submits the sheet to the analytics platform.
+      def commit!
+        Dakwak.logger.log_debug "Comitting dakana sheet: #{serialize}" if Dakwak.debugging?
+        Communicator.new.publish(Message.new(serialize, {}), "analytics", Dakwak.app_name)
+      end
+
+      protected
+      # 
+      def serialize()
+        @content.to_json
+      end      
+    end
+
+  end
+end

data/lib/dakwak/configurable.rb

@@ -0,0 +1,22 @@
+module Dakwak
+  class Configurable
+    attr_reader :cfg
+
+    module Impl
+      def set_option(k, v)
+        @cfg ||= {}
+
+          # convert string keys to symbols
+          if v.is_a?(Hash) then
+            new_v = {}
+            v.each_pair { |_k, _v| new_v[_k.to_sym] = v[_k] }
+            v = new_v
+          end
+
+          @cfg[k.to_sym] = v
+        end
+    end
+
+    include Impl
+  end
+end

data/lib/dakwak/configurator.rb

@@ -0,0 +1,61 @@
+require 'dakwak/logger'
+require 'json'
+
+module Dakwak # :nodoc: all
+  class Configurator
+
+    class << self
+      attr_reader :subs
+
+      def subscribe(context, configurable)
+        if not configurable.respond_to?("set_option")
+          raise "Subscribing as a configurable must respond_to method 'set_option'"
+        end
+
+        @@subs          ||= {}
+        @@subs[context] ||= []; @@subs[context] << configurable
+      end
+    end
+
+    include Logger
+    # include Silencable
+
+    def initialize
+      logging_context("Configurator")
+      @@subs ||= {}
+      @@opts ||= {}
+
+      super()
+    end
+
+    def consume(json_feed)
+      begin
+        cfg = JSON.parse(json_feed)
+      rescue JSON::ParserError => e
+        log_error "Unable to parse JSON configuration feed; #{e.message}"
+        return nil
+      end
+
+
+      cfg.each_pair { |ctx, options|
+
+        if not @@subs[ctx] or @@subs[ctx].empty? then
+          log_warn "Context #{ctx} has no subscribers, ignoring." unless silent?
+          next
+        end
+
+        log_debug "Configuring context #{ctx} (#{@@subs[ctx].size} subscribers)" unless silent?
+
+        options.each_pair { |name, val|
+          @@subs[ctx] ||= []
+          @@subs[ctx].each { |configurable|
+            configurable.set_option(name, val)
+          }
+        }
+      }
+
+      true
+    end
+
+  end
+end

data/lib/dakwak/log_manager.rb

@@ -0,0 +1,52 @@
+require 'thread'
+require 'syslog'
+
+module Dakwak # :nodoc: all
+  class LogManager
+
+    def initialize
+      @log_mtx = Mutex.new
+      @@opts = { timestamp: true, vanilla: false }
+      # @sl = Syslog.open(Dakwak.app_name, Syslog::LOG_PID, Syslog::LOG_DAEMON)
+      @sl = Syslog.open(Dakwak.app_name, 0, Syslog::LOG_DAEMON)
+      @@levels ||= { }
+      LogLevel.constants.each { |c| @@levels[LogLevel.const_get(c)] = c.to_s.downcase }      
+      # for syslog compatibility
+      @@levels["W"] = "warning"
+      @@levels["E"] = "err"
+      super()
+    end
+
+    class << self
+      def disable_timestamps
+        @@opts[:timestamp] = false
+      end
+
+      def enable_timestamps
+        @@opts[:timestamp] = true
+      end
+    end
+
+    # Available options:
+    # => :vanilla    Message will be logged with no formatting (default: false )
+    # => :timestamp  Whether to prefix the message with a timestamp (default: true)
+    # opts: deprecated, use accessors instead
+    def log(msg, level, opts = {})
+      @log_mtx.synchronize do
+        puts "#{msg}" or return if @@opts[:vanilla]
+
+        stamp = Time.now.strftime("%m-%d-%Y %H:%M:%S ") unless @@opts[:timestamp] == false
+        puts "#{stamp}#{Socket.gethostbyname(Socket.gethostname).first} #{Dakwak.app_name} [#{level}] #{msg}"
+
+        expanded_level = @@levels[level]
+        # disabling syslog support for now
+        # if @sl.respond_to?(expanded_level) then
+        #  @sl.send :"#{expanded_level}", "[#{level}] #{msg}"
+        # end
+
+      end
+    end
+
+  end
+
+end

data/lib/dakwak/logger.rb

@@ -0,0 +1,127 @@
+module Dakwak
+
+  class LogLevel
+    Debug   = 'D'
+    Info    = 'I'
+    Notice  = 'N'
+    Warn    = 'W'
+    Error   = 'E'
+    Alert   = 'A'
+    Crit    = 'C'
+  end
+
+  module Silencable
+    module Impl
+      class << self
+        def silence(toggle)
+          @@opts ||= {}
+          @@opts[:silent] = toggle
+        end
+
+        def silent?
+          @@opts ||= {}
+          @@opts[:silent]
+        end
+      end
+    end
+
+    def silent?
+      if @opts && @opts.has_key?(:silent)
+        return @opts[:silent]
+      else
+        return Silencable::Impl.silent?
+      end
+    end
+
+    def silence(toggle)
+      @opts ||= {}
+      @opts[:silent] = toggle
+    end
+
+  end
+
+  # A thread-safe logging interface.
+  module Logger
+    include Silencable
+
+    # The logging context (NDC) of this instance (defaults to "Unnamed")
+    def logging_context
+      @__log_ctx
+    end
+
+    # Assigns the logging context (NDC) of this logger instance.
+    def logging_context(context)
+      @__log_ctx = context
+      @__log_indent ||= 0
+
+      unless context.empty?
+        @__log_prefix = "#{@__log_ctx}: "
+      else
+        @__log_prefix = ""
+      end
+    end
+
+    # Logs a message.
+    #
+    # @param msg    The message to log
+    # @param level  The logging level (see LogLevel)
+    # @param opts   Logging options (see LogManager::log)
+    #
+    def log(msg, level, opts = {})
+  
+      print("#{@__log_prefix}#{"\t" * (@__log_indent || 0)}#{msg}")
+      Dakwak.log_manager.log("#{@__log_prefix}#{"\t" * (@__log_indent || 0)}#{msg}", level, opts) unless silent?
+    end
+
+    def self.included(base) # :nodoc:
+      if defined?(Rails) then
+        @@levels ||= { }
+        LogLevel.constants.each { |c| @@levels[LogLevel.const_get(c)] = c.to_s.downcase }
+        send :define_method, :"log" do |msg, level, _|
+          Rails.logger.send(@@levels[level], msg)
+        end
+      end
+
+      LogLevel.constants.each { |level|
+        method = :"log_#{level.to_s.downcase}"
+        return if method_defined? method
+
+        send :define_method, :"log_#{level.to_s.downcase}" do |*args|
+          msg = args[0]
+          opts = args[1] || {}
+          log(msg, LogLevel.const_get(level), opts)
+        end
+      }
+    end
+
+    # Indents all messages logged in the future with whitespace by 1 step.
+    def log_indent_inc
+      @__log_indent ||= 0
+      @__log_indent += 1
+    end
+
+    # Resets 1 step of whitespace indentation from all messages logged in the future.
+    def log_indent_dec
+      @__log_indent ||= 0
+      @__log_indent -= 1
+    end
+
+    # Messages logged within the given block will be indented by 1 step
+    def log_indent(&block) # :yield:
+      log_indent_inc
+      if block_given? then yield end
+      log_indent_dec
+    end
+
+    def initialize
+      @__log_ctx    ||= "unnamed"
+      @__log_indent ||= 0
+      @__log_prefix ||= ""
+      super()
+    end
+
+    private
+    attr_reader :__log_ctx, :__log_indent, :__log_prefix
+  end
+
+end

data/lib/dakwak/messaging/channel.rb

@@ -0,0 +1,215 @@
+require 'dakwak/logger'
+
+module Dakwak
+
+  # Channels bind a number of queues which are consumed for messages sent by communicators.
+  #
+  # Dakwak::Communicator instances can _subscribe_ to channel queues to be notified
+  # of messages received, and can publish messages through channels directly.
+  #
+  # *Note*:
+  # Channels should not be managed directly, see Dakwak::Station for more info.
+  #
+  # *Note*:
+  # A channel is internally bound to an AMQP exchange entity.
+  class Channel
+    include Logger
+
+    attr_reader :session, :channel, :exchange
+
+    def name()
+      @__name
+    end
+
+    def opened?
+      @session && @session.connected?
+    end
+
+    # Initiates connection with the AMQP broker using connection info
+    # stored in the Dakwak::Station and binds to a direct exchange
+    # identified by the name of this channel.
+    #
+    # Params:
+    # &on_open::
+    #  A proc that will be called if and when the connection
+    #  with the broker succeeds and the exchange is bound.
+    #
+    # *Note*:
+    # This is an asynchronous method.
+    def open(opts = { }, &on_open)
+      if opened?
+        on_open.call(self) if on_open
+        return
+      end
+
+      opts = { :passive => true, :durable => true }.merge(opts)
+
+      AMQP.connect(Station.instance.connection_options) do |session|
+        @session = session
+        AMQP::Channel.new(@session, :auto_recovery => true) do |channel|
+          @channel  = channel
+          @exchange = @channel.direct(@__name, opts)
+          on_open.call(self) if on_open
+        end
+      end
+    end
+
+    # Terminates the connection with the AMQP broker and destroys
+    # all queue consumers.
+    #
+    # Attempting to close an already closed channel will log a warning
+    # and reject the request. However, if a block was passed, it will
+    # still be called.
+    #
+    # *Note*:
+    # This is an asynchronous method.
+    def close(&on_close)
+      unless opened?
+        log_warn "Attempting to close a closed channel."
+        on_close.call(self) if on_close
+        return nil
+      end
+
+      EventMachine.next_tick do
+        @session.close {
+          @queues = []
+          @session = nil
+
+          on_close.call(self) if on_close
+        }
+      end
+
+      nil
+    end
+
+    # Sends a message over this channel to the destination queue.
+    #
+    # *Note*:
+    # Messages published over a queue that is being consumed by this
+    # application will be IGNORED. The effect is analogous to AMQP's
+    # no-local queue binding option. What this means to you is that
+    # you won't have to worry about receiving self-messages.
+    #
+    # *Note*:
+    # Messages are automatically stamped with the current application's
+    # id and set in the message.meta.app_id so there's no need to do
+    # it manually.
+    def publish(msg, queue, &callback)
+      if not opened? then
+        return open { |c| c.publish(msg, queue) }
+      end
+
+      @exchange.publish(msg.payload, msg.meta.merge!({
+        :routing_key => queue,
+        :app_id => Dakwak.app_fqn(),
+        :timestamp => Time.now.to_i
+      })) { callback.call(msg) if callback }
+    end
+
+    # Subscribes a Communicator instance to messages received by a queue.
+    # The queue will automatically be consumed if it had not explicitly
+    # been marked for consumption.
+    #
+    # *Remark*:
+    # The subscriber doesn't necessarily have to be an instance of a
+    # Communicator, only that it responds to a method with the following signature:
+    # on_message_received(msg)
+    def subscribe(comm, queue, opts = {})
+      consume(queue, opts)
+      @queues[queue][:subscribers] << comm
+    end
+
+    # Messages received by this queue will no longer be dispatched to the
+    # Communicator.
+    def unsubscribe(comm, queue)
+      return unless consuming?(queue)
+
+      @queues[queue][:subscribers].delete_if { |v| v == comm }
+    end
+
+    # Removes all queue subscriptions for this Communicator. Messages
+    # over this channel will no longer be dispatched to it.
+    def unsubscribe_all(comm)
+      @queues.each { |q| unsubscribe(comm, q) }
+    end
+
+    # Binds the specified queue and consumes any messages routed to it.
+    #
+    # *Note*:
+    # There's little point in consuming a queue with no subscribed Dakwak::Communicator
+    # instances to it. To pull messages from a queue, use Communicator::subscribe.
+    #
+    # Params:
+    # routing_key::
+    #  The AMQP routing key to use when binding the new queue to the exchange.
+    # opts::
+    # - :exclusive
+    #   The queue consumption will be exclusive to this consumer.
+    # - :nowait
+    #   Does not wait for the broker to reply. If a binding error
+    #   occurs, a Channel exception will be thrown.
+    def consume(key, opts = {})
+      return if consuming?(key)
+
+
+      queue_name = "#{key}_queue"
+      if opts.has_key?(:queue_name) then
+        queue_name = opts[:queue_name]
+        opts.delete(:queue_name)
+      end
+      log_info "Consuming '#{key}##{queue_name}'"
+
+      # assign some defaults
+      opts = {
+        :exclusive => false,
+        :nowait => true,
+        :durable => true,
+        :passive => true
+      }.merge(opts)
+
+      @queues[key]  = { :queue => nil, :subscribers => [], :consumer => nil }
+      entry         = @queues[key]
+      entry[:queue] = @channel.queue(queue_name, opts)
+      # entry[:queue].bind(@exchange, { :routing_key => key })
+      entry[:queue].subscribe { |meta, payload|
+        msg = Message.new(payload, meta, key)
+        dispatch(msg)
+      }
+    end
+
+    # Is this queue being consumed?
+    def consuming?(queue)
+      return @queues.has_key?(queue)
+    end
+
+    # Creates a new channel with name as the AMQP exchange name. Do not create
+    # channel objects directly, use Dakwak::Station instead.
+    def initialize(name)
+      @__name = name # the name of the exchange this channel represents
+      logging_context("Channel[#{@__name}]")
+
+      @queues = {}
+      super()
+    end
+
+    private
+
+    # Dispatches the received message to subscribed Communicator instances
+    # if eligible.
+    #
+    # The following conditions must be met for a message to be eligible for
+    # dispatching:
+    # => 1. The message does not have an app_id equal to this application's (self-sent)
+    # => 2. The message has no reply_to field
+    # => 3. The message has a reply_to field that points to this application's id
+    def dispatch(msg)
+      # discard self-sent messages
+      return if msg.meta.app_id == Dakwak.app_fqn()
+      # discard messages that were not directed at this instance
+      return if msg.meta.reply_to && !msg.meta.reply_to.empty? && msg.meta.reply_to != DakTM.app_fqn()
+
+      log_info "Dispatching message #{msg}"
+      @queues[msg.queue][:subscribers].each { |c| c.on_message_received(msg) }
+    end
+  end
+end

data/lib/dakwak/messaging/communicator.rb

@@ -0,0 +1,101 @@
+require 'dakwak/logger'
+require 'bson'
+
+module Dakwak
+
+  # Communicator instances can send and receive messages across the platform.
+  class Communicator
+
+    # Subscribes this communicator to a number of queues in a channel. Messages routed
+    # through the indicated queues will be passed on to this instance.
+    #
+    # Params:
+    # channel:: The name of the channel to subscribe to
+    # queues::  An array of strings containing names of the queues to consume
+    #
+    # *Warning*:
+    # Chaining calls to subscribe will only work within blocks as this method
+    # is asynchronous; when it returns, it doesn't mean the subscription is active,
+    # only when the block is called.
+    def subscribe(channel, queues, exchange_opts = {}, queue_opts = {}, &callback)
+      return if !channel or channel.empty? or !queues
+
+      queues = [queues] if queues.is_a? String
+
+      if queues.empty?
+        callback.call if callback
+        return
+      end
+
+      Proc.new { |channel, queue, &block|
+        Station.instance.open_channel(channel, exchange_opts) { |c|
+          c.subscribe(self, queue, queue_opts)
+          block.call(c) if block
+        }
+      }.call(channel, queues.pop) { |c| subscribe(channel, queues) { callback.call if callback } }
+    end
+
+    def subscribe_exclusively(opts = {}, &callback)
+      return @private_channel if @private_channel
+
+      @private_channel = PrivateChannel.new
+      @private_channel.open do
+        @private_channel.consume(self, opts, &callback)
+      end
+
+      return @private_channel
+    end
+
+    def private_channel
+      @private_channel
+    end
+
+    def reply_to(original_message, payload = "")
+      publish(original_message.prepare_reply(payload), "")
+    end
+
+    # Removes the subscription to a given queue in a channel.
+    def unsubscribe(channel, queue)
+      Station.instance.get_channel(channel) { |c| c.unsubscribe(self, queue) }
+    end
+
+    # Removes the subscription to all queues in a channel.
+    def unsubscribe(channel)
+      Station.instance.get_channel(channel) { |c| c.unsubscribe_all(self) }
+    end
+
+    # Sends a message to a queue over a channel.
+    #
+    # If the queue isn't specified, it is taken from the message. This is used
+    # when acknowledging requests or sending responses. See Job::ack() for more info.
+    def publish(msg, channel, queue = nil, &callback)
+      queue ||= msg.queue
+
+      Station.instance.get_channel(channel) { |c|
+        c.publish(msg, queue, &callback)
+      }
+    end
+
+    def publish_and_expect_reply(payload, channel, queue = nil, &callback)
+      m = Message.new(payload)
+      m.meta[:message_id] ||= BSON::ObjectId.new.to_s
+      m.meta[:headers]["queue_name"] = private_channel.name
+
+      publish(m, channel, queue, &callback)
+    end
+
+    # Handler called whenever a message is received over one of the subscribed
+    # queues.
+    #
+    # Override this in your implementation.
+    def on_message_received(msg)
+      @callbacks.each { |c| c.call(msg) }
+    end
+
+    def attach_message_handler(callback)
+      @callbacks ||= []
+      @callbacks << callback
+    end
+  end
+
+end