smith 0.5.13.1 → 0.6.1

data/lib/smith/daemon.rb

@@ -19,11 +19,24 @@ module Smith
       unlink_pid_file
 
       if @daemonise
-        Daemonize::daemonize('/dev/null', @name)
-      else
+        fork && exit
+
+        unless Process.setsid
+          raise RuntimeException, 'cannot detach from controlling terminal'
+        end
+
         $0 = @name
+
+        # Be nice to unmount.
+        Dir.chdir "/"
+
+        STDIN.reopen("/dev/null")
+        STDOUT.reopen("/dev/null")
+        STDERR.reopen(STDOUT)
       end
 
+      $0 = @name
+
       @pid.pid = Process.pid
       logger.debug { "Pid file: #{@pid.filename}" }
     end
@@ -57,7 +70,7 @@ module Smith
       if dir
         dir
       else
-        if Smith.config.agency._has_key?(:pid_dir)
+        if Smith.config.agency.to_hash.has_key?(:pid_dir)
           Smith.config.agency.pid_dir
         else
           Dir.tmpdir

data/lib/smith/exceptions.rb

@@ -7,9 +7,12 @@ module Smith
   end
 
   module Messaging
-    class ACLTimeoutError < RuntimeError; end
+    class MessageTimeoutError < RuntimeError; end
+  end
 
-    class IncompletePayload < RuntimeError; end
-    class IncorrectPayloadType < RuntimeError; end
+  module ACL
+    class Error < RuntimeError; end
+    class IncorrectTypeError < Error; end
+    class UnknownError < Error; end
   end
 end

data/lib/smith/logger.rb

@@ -17,13 +17,10 @@ module Smith
     module Methods
       protected
 
-      @@__name = 'smith'
-      @@__pattern = Smith::Config.get.logging.default_pattern
-      @@__date_pattern = Smith::Config.get.logging.default_date_pattern
       @@__level = Smith::Config.get.logging.level
       @@__trace = Smith::Config.get.logging.trace
-      @@__appender = Smith::Config.get.logging.appender._data.clone
-      @@__appender[:type] = Logging::Appenders.const_get(Extlib::Inflection.camelize(Smith::Config.get.logging.appender.type))
+
+      @@appender = nil
 
       def log_level(level=nil)
         if level
@@ -37,27 +34,19 @@ module Smith
         Logging.logger.root.level = @@__level
       end
 
-      def log_appender(opts={})
-        if @appender.nil? || !opts.empty?
-          @@__name = opts[:name] if opts[:name]
-          @appender = @@__appender[:type].send(:new, @@__name, @@__appender.merge(:layout => log_pattern))
-          @reload = true
-        end
-        Logging.logger.root.appenders = @appender
-      end
+      def log_appender
+        unless @@appender
+          appender_type = Extlib::Inflection.camelize(Config.get.logging.appender.type)
+          pattern_opts = {
+            :pattern => Config.get.logging.default_pattern,
+            :date_pattern => Config.get.logging.default_date_pattern}
+
+          appender_opts = Config.get.logging.appender.clone.merge(:layout => Logging.layouts.pattern(pattern_opts))
 
-      def log_pattern(*pattern)
-        case pattern.size
-        when 1
-          @@__pattern = pattern.shift
-          @reload = true
-        when 2
-          @@__pattern = pattern.shift
-          @@__date_pattern = pattern.shift
-          @reload = true
+          @@appender = Logging::Appenders.const_get(appender_type).new('smith', appender_opts)
         end
 
-        Logging.layouts.pattern({:pattern => @@__pattern, :date_pattern => @@__date_pattern})
+        Logging.logger.root.appenders = @@appender
       end
 
       def log_trace(trace)
@@ -74,7 +63,6 @@ module Smith
       private
 
       def __setup
-        self.log_pattern
         self.log_appender
         self.log_level
         @reload = true

data/lib/smith/messaging/acl/agent_keepalive.proto

@@ -1,6 +1,6 @@
 package Smith.ACL;
 message AgentKeepalive {
-  required string name = 1;
-  optional int64 pid = 2;
+  optional string  uuid = 1;
+  required string name = 2;
   optional int64 time = 3;
 }

data/lib/smith/messaging/acl/agent_lifecycle.proto

@@ -1,12 +1,18 @@
 package Smith.ACL;
 
-// This should refactored into two: start and stop.
-message AgentLifecycle {
-  required string state = 1;
-  required string name = 2;
-  optional int64 pid = 3;
-  optional bool monitor = 4;
-  optional bool singleton = 5;
-  optional string metadata = 6;
-  optional int64 started_at = 7;
+message AgentAcknowledgeStart {
+  required string   uuid = 1;
+  required int32    pid = 2;
+  required int64    started_at = 3;
+  optional bool     singleton = 4;
+  optional bool     monitor = 5;
+  optional string   metadata = 6;
+}
+
+message AgentAcknowledgeStop {
+  required string   uuid = 1;
+}
+
+message AgentDead {
+  required string   uuid = 1;
 }

data/lib/smith/messaging/acl/agent_stats.proto

@@ -6,9 +6,10 @@ message AgentStats {
     required int32    length = 3;
   }
 
-  required string       agent_name = 1;
-  required int32        pid = 2;
-  optional int64        rss = 3;
-  required int64        up_time = 4;
-  repeated QueueStats   queues = 5;
+  required string       uuid = 1;
+  required string       agent_name = 2;
+  required int32        pid = 3;
+  optional int64        rss = 4;
+  required int64        up_time = 5;
+  repeated QueueStats   queues = 6;
 }

data/lib/smith/messaging/acl/default.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 
-require 'yajl'
+require 'multi_json'
 
 module Smith
   module ACL
@@ -15,11 +15,6 @@ module Smith
         @message = message
       end
 
-      # Always return true. There is no validation.
-      def initialized?
-        true
-      end
-
       def serialize_to_string
         Marshal.dump(@message)
       end
@@ -41,7 +36,7 @@ module Smith
       end
 
       def to_json
-        Yajl.dump(@message)
+        MultiJson.dump(@message)
       end
 
       def method_missing(method, args)

data/lib/smith/messaging/acl_type_cache.rb

@@ -0,0 +1,77 @@
+# -*- encoding: utf-8 -*-
+
+require 'set'
+require 'extlib'
+require 'singleton'
+require 'murmurhash3'
+
+module Smith
+  class AclTypeCache
+    include Singleton
+    include MurmurHash3
+
+    def initialize
+      clear!
+    end
+
+    def add(type)
+      if @types[type]
+        false
+      else
+        h = to_murmur32(type)
+        @types[type] = h
+        @hashes[h] = type
+        @legacy_types_by_hash[type.to_s.split(/::/)[-1].snake_case] = type
+        true
+      end
+    end
+
+    def get_by_hash(type)
+      if @hashes[type]
+        @hashes[type]
+      elsif @legacy_types_by_hash[type.to_s]
+        @legacy_types_by_hash[type.to_s]
+      end
+    end
+
+    def get_by_type(type)
+      @types[type]
+    end
+
+    # Look the key up in the cache. This defaults to the key being the hash.
+    # If :by_type => true is passed in as the second argument then it will
+    # perform the lookup in the type hash.
+    # 
+    def include?(key, opts={})
+      if opts[:by_type]
+        !get_by_type(key).nil?
+      else
+        !get_by_hash(key).nil?
+      end
+    end
+
+    # Clear the internal hashes.
+    def clear!
+      @types = {}
+      @hashes = {}
+      @legacy_types_by_hash = {}
+    end
+
+    # Dump the type hash
+    def dump_types
+      @types
+    end
+
+    # Dump the hashes hash
+    def dump_hashes
+      @hashes
+    end
+
+    private
+
+    # Convert the name to a base 36 murmur hash
+    def to_murmur32(type)
+      V32.murmur3_32_str_hash(type.to_s).to_s(36)
+    end
+  end
+end

data/lib/smith/messaging/factory.rb

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+module Smith
+
+  module ACL
+    class Factory
+      include Logger
+
+      class << self
+        def create(type, content=nil, &blk)
+          if type.respond_to?(:serialize_to_string)
+            return type
+          else
+            clazz = (type.is_a?(::Protobuf::Message)) ? type : get_clazz(type)
+
+            if blk
+              clazz.new.tap { |m| blk.call(m) }
+            else
+              (content.nil?) ? clazz.new : clazz.new(content)
+            end
+          end
+        end
+
+        def get_clazz(type)
+          type.to_s.split(/::/).inject(Kernel) { |acc, t| acc.const_get(t) }
+        end
+      end
+    end
+  end
+end

data/lib/smith/messaging/queue.rb

@@ -6,33 +6,35 @@ module Smith
       extend Util
 
       class << self
-        def messages?(queue_name, &blk)
-          number_of_messages(queue_name) do |n|
+        def messages?(queue_def, &blk)
+          number_of_messages(queue_def) do |n|
             yield n > 0
           end
         end
 
-        def consumers?(queue_name)
-          number_of_consumers(queue_name) do |n|
+        def consumers?(queue_def)
+          number_of_consumers(queue_def) do |n|
             yield n > 0
           end
         end
 
-        def number_of_consumers(queue_name)
-          status(queue_name) do |_, number_consumers|
+        def number_of_consumers(queue_def)
+          status(queue_def) do |_, number_consumers|
             yield number_consumers
           end
         end
 
-        def number_of_messages(queue_name)
-          status(queue_name) do |number_messages, _|
+        def number_of_messages(queue_def)
+          status(queue_def) do |number_messages, _|
             yield number_messages
           end
         end
 
-        def status(queue_name)
+        def status(queue_def)
           open_channel do |channel, ok|
-            channel.queue(normalise(queue_name), :passive => true).status do |number_messages, number_consumers|
+
+            queue_def = queue_def.is_a?(QueueDefinition) ? queue_def : QueueDefinition.new(queue_def, :passive => true)
+            channel.queue(*queue_def.to_a).status do |number_messages, number_consumers|
               yield number_messages, number_consumers
               channel.close
             end

data/lib/smith/messaging/queue_definition.rb

@@ -3,16 +3,33 @@
 module Smith
   class QueueDefinition
 
-    attr_reader :name, :options
+    attr_reader :options
 
     def initialize(name, options)
-      @name = name
+      @normalised_queue = "#{Smith.config.smith.namespace}.#{name}"
+      @denormalised_queue = "#{name}"
       @options = options
     end
 
+    def denormalise
+      @denormalised_queue
+    end
+
+    def name
+      @normalised_queue
+    end
+
+    def normalise
+      @normalised_queue
+    end
+
     # to_a is defined to make the splat operator work.
-    def to_a()
-      return @name, @options
+    def to_a
+      return @normalised_queue, @options
+    end
+
+    def to_s
+      "<#{self.class}: #{@denormalised_queue}, #{@options.inspect}>"
     end
   end
 end

data/lib/smith/messaging/receiver.rb

@@ -10,26 +10,25 @@ module Smith
       include Logger
       include Util
 
-      attr_accessor :queue_name
+      def initialize(queue_def, opts={}, &blk)
 
-      def initialize(queue_definition, opts={}, &blk)
+        # This is for backward compatibility.
+        @queue_def = queue_def.is_a?(QueueDefinition) ? queue_def : QueueDefinition.new(queue_def, opts)
 
-        @queue_name, opts = get_queue_name_and_options(queue_definition, opts)
-
-        @normalised_queue_name = normalise(@queue_name)
+        @acl_type_cache = AclTypeCache.instance
 
         @foo_options = {
-          :auto_ack => option_or_default(opts, :auto_ack, true),
-          :threading => option_or_default(opts, :threading, false)}
+          :auto_ack => option_or_default(@queue_def.options, :auto_ack, true),
+          :threading => option_or_default(@queue_def.options, :threading, false)}
 
-        @payload_type = option_or_default(opts, :type, []) {|v| [v].flatten }
+        @payload_type = Array(option_or_default(@queue_def.options, :type, []))
 
-        prefetch = option_or_default(opts, :prefetch, Smith.config.agent.prefetch)
+        prefetch = option_or_default(@queue_def.options, :prefetch, Smith.config.agent.prefetch)
 
-        @options = AmqpOptions.new(opts)
-        @options.routing_key = @normalised_queue_name
+        @options = AmqpOptions.new(@queue_def.options)
+        @options.routing_key = @queue_def.normalise
 
-        @message_counter = MessageCounter.new(@queue_name)
+        @message_counter = MessageCounter.new(@queue_def.denormalise)
 
         @channel_completion = EM::Completion.new
         @queue_completion = EM::Completion.new
@@ -40,55 +39,38 @@ module Smith
 
         open_channel(:prefetch => prefetch) do |channel|
           @channel_completion.succeed(channel)
-          channel.direct(@normalised_queue_name, @options.exchange) do |exchange|
+          channel.direct(@queue_def.normalise, @options.exchange) do |exchange|
             @exchange_completion.succeed(exchange)
           end
         end
 
         open_channel(:prefetch => prefetch) do |channel|
-          channel.queue(@normalised_queue_name, @options.queue) do |queue|
+          channel.queue(@queue_def.normalise, @options.queue) do |queue|
             @exchange_completion.completion do |exchange|
-              queue.bind(exchange, :routing_key => @queue_name)
+              queue.bind(exchange, :routing_key => @queue_def.normalise)
               @queue_completion.succeed(queue)
               @requeue_options_completion.succeed(:exchange => exchange, :queue => queue)
             end
           end
         end
 
-        start_garbage_collection
-
         blk.call(self) if blk
       end
 
-      def start_garbage_collection
-        logger.debug { "Starting the garbage collector." }
-        EM.add_periodic_timer(5) do
-          @reply_queues.each do |queue_name,queue|
-            queue.status do |number_of_messages, number_of_consumers|
-              if number_of_messages == 0 && number_of_consumers == 0
-                queue.delete do |delete_ok|
-                  @reply_queues.delete(queue_name)
-                  logger.debug { "Unused reply queue deleted: #{queue_name}" }
-                end
-              end
-            end
-          end
-        end
-      end
-
       def ack(multiple=false)
         @channel_completion.completion {|channel| channel.ack(multiple) }
       end
 
-      private :start_garbage_collection
-
       def setup_reply_queue(reply_queue_name, &blk)
         if @reply_queues[reply_queue_name]
           blk.call(@reply_queues[reply_queue_name])
         else
           @exchange_completion.completion do |exchange|
             logger.debug { "Attaching to reply queue: #{reply_queue_name}" }
-            Smith::Messaging::Sender.new(reply_queue_name, :auto_delete => false, :immediate => true, :mandatory => true) do |sender|
+
+            queue_def = QueueDefinition.new(reply_queue_name, :auto_delete => true, :immediate => true, :mandatory => true, :durable => false)
+
+            Smith::Messaging::Sender.new(queue_def) do |sender|
               @reply_queues[reply_queue_name] = sender
               blk.call(sender)
             end
@@ -104,21 +86,27 @@ module Smith
           @requeue_options_completion.completion do |requeue_options|
             if !queue.subscribed?
               opts = @options.subscribe
-              logger.debug { "Subscribing to: [queue]:#{@queue_name} [options]:#{opts}" }
+              logger.debug { "Subscribing to: [queue]:#{@queue_def.denormalise} [options]:#{opts}" }
               queue.subscribe(opts) do |metadata,payload|
                 if payload
                   on_message(metadata, payload, requeue_options, &blk)
                 else
-                  logger.verbose { "Received null message on: #{@queue_name} [options]:#{opts}" }
+                  logger.verbose { "Received null message on: #{@queue_def.denormalise} [options]:#{opts}" }
                 end
               end
             else
-              logger.error { "Queue is already subscribed too. Not listening on: #{@queue_name}" }
+              logger.error { "Queue is already subscribed too. Not listening on: #{@queue_def.denormalise}" }
             end
           end
         end
       end
 
+      def unsubscribe(&blk)
+        @queue_completion.completion do |queue|
+          queue.unsubscribe(&blk)
+        end
+      end
+
       # pops a message off the queue and passes the headers and payload
       # into the block. +pop+ will automatically acknowledge the message
       # unless the options sets :ack to false.
@@ -146,7 +134,7 @@ module Smith
       end
 
       def on_message(metadata, payload, requeue_options, &blk)
-        if @payload_type.empty? || @payload_type.include?(metadata.type.to_sym)
+        if @payload_type.empty? || @payload_type.include?(@acl_type_cache.get_by_hash(metadata.type))
           @message_counter.increment_counter
           if metadata.reply_to
             setup_reply_queue(metadata.reply_to) do |queue|
@@ -156,32 +144,31 @@ module Smith
             Foo.new(metadata, payload, @foo_options, requeue_options, &blk)
           end
         else
-          raise IncorrectPayloadType, "This queue can only accept the following payload types: #{@payload_type.to_a.to_s}"
+          allowable_acls = @payload_type.join(", ")
+          received_acl = @acl_type_cache.get_by_hash(metadata.type)
+          raise ACL::IncorrectTypeError, "Received ACL: #{received_acl} on queue: #{@queue_def.denormalise}. This queue can only accept the following ACLs: #{allowable_acls}"
         end
       end
 
       private :on_message
 
-      # def delete(&blk)
-      #   @queue_completion.completion do |queue|
-      #     queue.delete do
-      #       @exchange_completion.completion do |exchange|
-      #         exchange.delete do
-      #           @channel_completion.completion do |channel|
-      #             channel.close(&blk)
-      #           end
-      #         end
-      #       end
-      #     end
-      #   end
-      # end
-
-
-      # I've a feeling this should be replaced by the above code.
-      # TODO Check this is correct.
+      def queue_name
+        @queue_def.denormalise
+      end
+
       def delete(&blk)
-        @queue_completion.completion do |queue|
-          queue.delete(&blk)
+        @exchange_completion.completion do |exchange|
+          @queue_completion.completion do |queue|
+            @channel_completion.completion do |channel|
+              queue.unbind(exchange) do
+                queue.delete do
+                  exchange.delete do
+                    channel.close(&blk)
+                  end
+                end
+              end
+            end
+          end
         end
       end
 
@@ -212,7 +199,6 @@ module Smith
       end
     end
 
-
     class Foo
       include Smith::Logger
 
@@ -223,8 +209,15 @@ module Smith
         @reply_queue = opts[:reply_queue]
         @requeue_opts = requeue_opts
 
+        @acl_type_cache = AclTypeCache.instance
+
         @time = Time.now
-        @message = ACL::Payload.decode(data, metadata.type)
+
+        # TODO add some better error checking/diagnostics.
+        clazz = @acl_type_cache.get_by_hash(metadata.type)
+        raise ACL::UnknownError, "Unknown ACL: #{metadata.type}" if clazz.nil?
+
+        @message = clazz.new.parse_from_string(data)
 
         if opts[:threading]
           EM.defer do

data/lib/smith/messaging/requeue.rb

@@ -38,7 +38,7 @@ module Smith
           logger.verbose { "Requeuing to: #{@queue.name}. [options]: #{opts}" }
           logger.verbose { "Requeuing to: #{@queue.name}. [message]: #{@message}" }
 
-          @exchange.publish(ACL::Payload.new(ACL::Factory.create(@metadata.type, @message)).encode, opts)
+          @exchange.publish(ACL::Factory.create(@metadata.type, @message), opts)
         end
       end
 
@@ -82,10 +82,6 @@ module Smith
       def linear_strategy(delay)
         delay * (current_requeue_number + 1)
       end
-
-      def denormalise(name)
-        name.gsub(/^#{Regexp.escape(Smith.config.smith.namespace)}./, '')
-      end
     end
   end
 end