ezmobius-nanite 0.4.1.1 → 0.4.1.2

data/Rakefile

@@ -43,6 +43,8 @@ Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
 end
 
+task :default => :spec
+
 task :install => [:package] do
   sh %{sudo gem install pkg/#{GEM}-#{VER}}
 end

data/lib/nanite/actor_registry.rb

@@ -8,7 +8,9 @@ module Nanite
 
     def register(actor, prefix)
       raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
-      Nanite::Log.info("Registering #{actor.inspect} with prefix #{prefix.inspect}")
+      log_msg = "[actor] #{actor.class.to_s}"
+      log_msg += ", prefix #{prefix}" if prefix && !prefix.empty?
+      Nanite::Log.info(log_msg)
       prefix ||= actor.class.default_prefix
       actors[prefix.to_s] = actor
     end

data/lib/nanite/agent.rb

@@ -158,7 +158,7 @@ module Nanite
       actors = @options[:actors]
       Dir["#{actors_dir}/*.rb"].each do |actor|
         next if actors && !actors.include?(File.basename(actor, ".rb"))
-        Nanite::Log.info("loading actor: #{actor}")
+        Nanite::Log.info("[setup] loading #{actor}")
         require actor
       end
       init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
@@ -166,25 +166,27 @@ module Nanite
     end
 
     def receive(packet)
+      Nanite::Log.debug("RECV #{packet.to_s}")
       case packet
       when Advertise
-        Nanite::Log.debug("handling Advertise: #{packet.inspect}")
+        Nanite::Log.info("RECV #{packet.to_s}") unless Nanite::Log.level == Logger::DEBUG
         advertise_services
       when Request, Push
-        Nanite::Log.debug("handling Request: #{packet.inspect}")
         if @security && !@security.authorize(packet)
+          Nanite::Log.warn("RECV NOT AUTHORIZED #{packet.to_s}")
           if packet.kind_of?(Request)
             r = Result.new(packet.token, packet.reply_to, @deny_token, identity)
             amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
           end
         else
+          Nanite::Log.info("RECV #{packet.to_s([:from, :tags])}") unless Nanite::Log.level == Logger::DEBUG
           dispatcher.dispatch(packet)
         end
       when Result
-        Nanite::Log.debug("handling Result: #{packet.inspect}")
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == Logger::DEBUG
         @mapper_proxy.handle_result(packet)
       when IntermediateMessage
-        Nanite::Log.debug("handling Intermediate Result: #{packet.inspect}")
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == Logger::DEBUG
         @mapper_proxy.handle_intermediate_result(packet)
       end
     end
@@ -198,10 +200,9 @@ module Nanite
       amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
         begin
           info.ack
-          packet = serializer.load(msg)
-          receive(packet)
+          receive(serializer.load(msg))
         rescue Exception => e
-          Nanite::Log.error("Error handling packet: #{e.message}")
+          Nanite::Log.error("RECV #{e.message}")
         end
       end
     end
@@ -231,13 +232,15 @@ module Nanite
     def un_register
       unless @unregistered
         @unregistered = true
+        Nanite::Log.info("SEND [un_register]")
         amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity)))
       end
     end
 
     def advertise_services
-      Nanite::Log.debug("advertise_services: #{registry.services.inspect}")
-      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(Register.new(identity, registry.services, status_proc.call, self.tags)))
+      reg = Register.new(identity, registry.services, status_proc.call, self.tags)
+      Nanite::Log.info("SEND #{reg.to_s}")
+      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(reg))
     end
 
     def parse_uptime(up)

data/lib/nanite/cluster.rb

@@ -1,23 +1,17 @@
 module Nanite
   class Cluster
-    attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper
+    attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
 
-    def initialize(amq, agent_timeout, identity, serializer, mapper, redis=nil)
+    def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
       @amq = amq
       @agent_timeout = agent_timeout
       @identity = identity
       @serializer = serializer
       @mapper = mapper
-      @redis = redis
+      @state = state_configuration
       @security = SecurityProvider.get
-      if redis
-        Nanite::Log.info("using redis for state storage")
-        require 'nanite/state'
-        @nanites = ::Nanite::State.new(redis)
-      else
-        require 'nanite/local_state'
-        @nanites = Nanite::LocalState.new
-      end
+      @callbacks = callbacks
+      setup_state
       @reaper = Reaper.new(agent_timeout)
       setup_queues
     end
@@ -35,18 +29,27 @@ module Nanite
       case reg
       when Register
         if @security.authorize_registration(reg)
+          Nanite::Log.info("RECV #{reg.to_s}")
           nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags }
-          reaper.timeout(reg.identity, agent_timeout + 1) { nanites.delete(reg.identity) }
-          Nanite::Log.info("registered: #{reg.identity}, #{nanites[reg.identity].inspect}")
+          reaper.timeout(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
+          callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
         else
-          Nanite::Log.warning("registration of #{reg.inspect} not authorized")
+          Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
         end
       when UnRegister
+        Nanite::Log.info("RECV #{reg.to_s}")
         nanites.delete(reg.identity)
-        Nanite::Log.info("un-registering: #{reg.identity}")
+        callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
+      else
+        Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
       end
     end
 
+    def nanite_timed_out(token)
+      nanite = nanites.delete(token)
+      callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
+    end
+    
     def route(request, targets)
       EM.next_tick { targets.map { |target| publish(request, target) } }
     end
@@ -57,6 +60,7 @@ module Nanite
       begin
         old_target = request.target
         request.target = target unless target == 'mapper-offline'
+        Nanite::Log.info("SEND #{request.to_s([:from, :tags, :target])}")
         amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
       ensure
         request.target = old_target
@@ -68,36 +72,46 @@ module Nanite
     # updates nanite information (last ping timestamps, status)
     # when heartbeat message is received
     def handle_ping(ping)
-      if nanite = nanites[ping.identity]
-        nanite[:status] = ping.status
-        reaper.reset_with_autoregister_hack(ping.identity, agent_timeout + 1) { nanites.delete(ping.identity) }
-      else
-        amq.queue(ping.identity).publish(serializer.dump(Advertise.new))
+      begin
+        if nanite = nanites[ping.identity]
+          nanite[:status] = ping.status
+          reaper.reset_with_autoregister_hack(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
+        else
+          packet = Advertise.new
+          Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
+          amq.queue(ping.identity).publish(serializer.dump(packet))
+        end
       end
     end
     
     # forward request coming from agent
     def handle_request(request)
       if @security.authorize_request(request)
-        result = Result.new(request.token, request.from, nil, mapper.identity)
-        intm_handler = lambda do |res|
-          result.results = res
+        Nanite::Log.info("RECV #{request.to_s([:from, :target, :tags])}") unless Nanite::Log.level == Logger::DEBUG
+        Nanite::Log.debug("RECV #{request.to_s}")
+
+        intm_handler = lambda do |result, job|
+          result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
           forward_response(result, request.persistent)
         end
+        
+        result = Result.new(request.token, request.from, nil, mapper.identity)
         ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
           result.results = res
           forward_response(result, request.persistent)
         end
+
         if ok == false
           forward_response(result, request.persistent)
         end
       else
-        Nanite::Log.warning("request #{request.inspect} not authorized")
+        Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
       end
     end
     
     # forward response back to agent that originally made the request
     def forward_response(res, persistent)
+      Nanite::Log.info("SEND #{res.to_s([:to])}")
       amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
     end
     
@@ -150,14 +164,14 @@ module Nanite
       handler = lambda do |ping|
         begin
           ping = serializer.load(ping)
-          Nanite::Log.debug("got heartbeat from #{ping.identity}") if ping.respond_to?(:identity)
+          Nanite::Log.debug("RECV #{ping.to_s}") if ping.respond_to?(:to_s)
           handle_ping(ping)
         rescue Exception => e
-          Nanite::Log.error("Error handling heartbeat: #{e.message}")
+          Nanite::Log.error("RECV [ping] #{e.message}")
         end
       end
       hb_fanout = amq.fanout('heartbeat', :durable => true)
-      if @redis
+      if shared_state?
         amq.queue("heartbeat").bind(hb_fanout).subscribe &handler
       else
         amq.queue("heartbeat-#{identity}", :exclusive => true).bind(hb_fanout).subscribe &handler
@@ -167,15 +181,13 @@ module Nanite
     def setup_registration_queue
       handler = lambda do |msg|
         begin
-          msg = serializer.load(msg)
-          Nanite::Log.debug("got registration from #{msg.identity}")
-          register(msg)
+          register(serializer.load(msg))
         rescue Exception => e
-          Nanite::Log.error("Error handling registration: #{e.message}")
+          Nanite::Log.error("RECV [register] #{e.message}")
         end
       end
       reg_fanout = amq.fanout('registration', :durable => true)
-      if @redis
+      if shared_state?
         amq.queue("registration").bind(reg_fanout).subscribe &handler
       else
         amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
@@ -185,19 +197,36 @@ module Nanite
     def setup_request_queue
       handler = lambda do |msg|
         begin
-          msg = serializer.load(msg)
-          Nanite::Log.debug("got request from #{msg.from} of type #{msg.type}")
-          handle_request(msg)
+          handle_request(serializer.load(msg))
         rescue Exception => e
-          Nanite::Log.error("Error handling request: #{e.message}")
+          Nanite::Log.error("RECV [request] #{e.message}")
         end
       end
       req_fanout = amq.fanout('request', :durable => true)
-      if @redis
+      if shared_state?
         amq.queue("request").bind(req_fanout).subscribe &handler
       else
         amq.queue("request-#{identity}", :exclusive => true).bind(req_fanout).subscribe &handler
       end
     end
+
+    def setup_state
+      case @state
+      when String
+        # backwards compatibility, we assume redis if the configuration option
+        # was a string
+        Nanite::Log.info("[setup] using redis for state storage")
+        require 'nanite/state'
+        @nanites = Nanite::State.new(@state)
+      when Hash
+      else
+        require 'nanite/local_state'
+        @nanites = Nanite::LocalState.new
+      end
+    end
+    
+    def shared_state?
+      !@state.nil?
+    end
   end
 end

data/lib/nanite/dispatcher.rb

@@ -31,6 +31,7 @@ module Nanite
       callback = lambda do |r|
         if deliverable.kind_of?(Request)
           r = Result.new(deliverable.token, deliverable.reply_to, r, identity)
+          Nanite::Log.info("SEND #{r.to_s([])}")
           amq.queue(deliverable.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
         end
         r # For unit tests

data/lib/nanite/job.rb

@@ -14,8 +14,6 @@ module Nanite
     end
 
     def process(msg)
-      Nanite::Log.debug("processing message: #{msg.inspect}")
-
       if job = jobs[msg.token]
         job.process(msg)
 
@@ -29,6 +27,8 @@ module Nanite
           handler = job.intermediate_handler_for_key(key)
           if handler
             case handler.arity
+            when 2
+              handler.call(job.intermediate_state[msg.from][key].last, job)
             when 3
               handler.call(key, msg.from, job.intermediate_state[msg.from][key].last)
             when 4

data/lib/nanite/log.rb

@@ -13,7 +13,7 @@ module Nanite
       # Use Nanite::Logger.init when you want to set up the logger manually.
       # If this method is called with no arguments, it will log to STDOUT at the :info level.
       # It also configures the Logger instance it creates to use the custom Nanite::Log::Formatter class.
-      def init(identity, path = false)
+      def init(identity = nil, path = false)
         if path
           @file = File.join(path, "nanite.#{identity}.log")
         else
@@ -30,7 +30,8 @@ module Nanite
       def level=(loglevel)
         init() unless @logger
         loglevel = loglevel.intern if loglevel.is_a?(String)
-        @logger.info("Setting log level to #{loglevel.to_s.upcase}")
+        @logger.info("[setup] setting log level to #{loglevel.to_s.upcase}")
+        @level = loglevel
         case loglevel
         when :debug
           @logger.level = Logger::DEBUG
@@ -46,12 +47,12 @@ module Nanite
           raise ArgumentError, "Log level must be one of :debug, :info, :warn, :error, or :fatal"
         end
       end
-      
+
       # Passes any other method calls on directly to the underlying Logger object created with init. If
       # this method gets hit before a call to Nanite::Logger.init has been made, it will call 
       # Nanite::Logger.init() with no arguments.
       def method_missing(method_symbol, *args)
-        init(identity) unless @logger
+        init unless @logger
         if args.length > 0
           @logger.send(method_symbol, *args)
         else

data/lib/nanite/mapper.rb

@@ -24,7 +24,8 @@ module Nanite
     attr_reader :cluster, :identity, :job_warden, :options, :serializer, :amq
 
     DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'mapper', :identity => Identity.generate, :agent_timeout => 15,
-      :offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false}) unless defined?(DEFAULT_OPTIONS)
+      :offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false,
+      :callbacks => {} }) unless defined?(DEFAULT_OPTIONS)
 
     # Initializes a new mapper and establishes
     # AMQP connection. This must be used inside EM.run block or if EventMachine reactor
@@ -65,8 +66,9 @@ module Nanite
     #
     # persistent  : true instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the
     #               broker is restarted. Default is false. Can be overriden on a per-message basis using the request and push methods.
-    # 
+    #
     # secure      : use Security features of rabbitmq to restrict nanites to themselves
+    # prefetch    : Sets prefetch (only supported in RabbitMQ >= 1.6)
     #
     # Connection options:
     #
@@ -105,14 +107,9 @@ module Nanite
       @options[:file_root] ||= File.join(@options[:root], 'files')
       @options.freeze
     end
-    
+
     def run
-      log_path = false
-      if @options[:daemonize]
-        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
-      end
-      Nanite::Log.init(@identity, log_path)
-      Nanite::Log.level = @options[:log_level] if @options[:log_level]
+      setup_logging
       @serializer = Serializer.new(@options[:format])
       pid_file = PidFile.new(@identity, @options)
       pid_file.check
@@ -120,11 +117,13 @@ module Nanite
         daemonize
         pid_file.write
         at_exit { pid_file.remove }
+      else
+        trap("INT") {exit}
       end
       @amq = start_amqp(@options)
       @job_warden = JobWarden.new(@serializer)
-      @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis])
-      Nanite::Log.info('starting mapper')
+      setup_cluster
+      Nanite::Log.info('[setup] starting mapper')
       setup_queues
       start_console if @options[:console] && !@options[:daemonize]
     end
@@ -189,7 +188,7 @@ module Nanite
         false
       end
     end
-      
+
     # Make a nanite request which does not expect a response.
     #
     # ==== Parameters
@@ -235,6 +234,10 @@ module Nanite
     end
 
     def setup_queues
+      if amq.respond_to?(:prefetch) && @options.has_key?(:prefetch)
+        amq.prefetch(@options[:prefetch])
+      end
+
       setup_offline_queue
       setup_message_queue
     end
@@ -264,14 +267,28 @@ module Nanite
     def setup_message_queue
       amq.queue(identity, :exclusive => true).bind(amq.fanout(identity)).subscribe do |msg|
         begin
-          msg = serializer.load(msg)
-          Nanite::Log.debug("got result from #{msg.from}: #{msg.results.inspect}")
+          msg = serializer.load(msg)     
+          Nanite::Log.debug("RECV #{msg.to_s}")
+          Nanite::Log.info("RECV #{msg.to_s([:from])}") unless Nanite::Log.level == Logger::DEBUG
           job_warden.process(msg)
         rescue Exception => e
-          Nanite::Log.error("Error handling result: #{e.message}")
+          Nanite::Log.error("RECV [result] #{e.message}")
         end
       end
     end
+
+    def setup_logging
+      log_path = false
+      if @options[:daemonize]
+        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      Nanite::Log.init(@identity, log_path)
+      Nanite::Log.level = @options[:log_level] if @options[:log_level]
+    end
+
+    def setup_cluster
+      @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
+    end
   end
 end
 

data/lib/nanite/mapper_proxy.rb

@@ -37,6 +37,7 @@ module Nanite
       request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
       pending_requests[request.token] = 
         { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
+      Nanite::Log.info("SEND #{request.to_s([:tags, :target])}")
       amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
     end    
     

data/lib/nanite/packets.rb

@@ -2,58 +2,111 @@ module Nanite
   # Base class for all Nanite packets,
   # knows how to dump itself to JSON
   class Packet
+
+    attr_accessor :size
+
     def initialize
       raise NotImplementedError.new("#{self.class.name} is an abstract class.")
     end
+
     def to_json(*a)
-      {
+      js = {
         'json_class'   => self.class.name,
         'data'         => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
       }.to_json(*a)
+      js = js.chop + ",\"size\":#{js.size}}"
+      js
+    end
+
+    # Log representation
+    def to_s(filter=nil)
+      res = "[#{ self.class.to_s.split('::').last.
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        downcase }]"
+      res += " (#{size.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")} bytes)" if size && !size.to_s.empty?
+      res
+    end
+
+    # Log friendly name for given agent id
+    def id_to_s(id)
+      case id
+        when /^mapper-/ then 'mapper'
+        when /^nanite-(.*)/ then Regexp.last_match(1)
+        else id
+      end
+    end
+
+    # Wrap given string to given maximum number of characters per line
+    def wrap(txt, col=120)
+      txt.gsub(/(.{1,#{col}})( +|$\n?)|(.{1,#{col}})/, "\\1\\3\n").chomp
     end
+
   end
 
   # packet that means start of a file transfer
   # operation
   class FileStart < Packet
+
     attr_accessor :filename, :token, :dest
-    def initialize(filename, dest, token)
+
+    def initialize(filename, dest, token, size=nil)
       @filename = filename
       @dest = dest
       @token = token
+      @size = size
     end
 
     def self.json_create(o)
       i = o['data']
-      new(i['filename'], i['dest'], i['token'])
+      new(i['filename'], i['dest'], i['token'], o['size'])
+    end
+
+    def to_s
+      wrap("#{super} <#{token}> #{filename} to #{dest}")
     end
   end
 
   # packet that means end of a file transfer
   # operation
   class FileEnd < Packet
+
     attr_accessor :token, :meta
-    def initialize(token, meta)
+
+    def initialize(token, meta, size=nil)
       @token = token
       @meta  = meta
+      @size = size
     end
 
     def self.json_create(o)
       i = o['data']
-      new(i['token'], i['meta'])
+      new(i['token'], i['meta'], o['size'])
+    end
+
+    def to_s
+      wrap("#{super} <#{token}> meta #{meta}")
     end
   end
 
   # packet that carries data chunks during a file transfer
   class FileChunk < Packet
+
     attr_accessor :chunk, :token
-    def initialize(token, chunk=nil)
+
+    def initialize(token, size=nil, chunk=nil)
       @chunk = chunk
       @token = token
+      @size = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['token'], i['chunk'])
+      new(i['token'], o['size'], i['chunk'])
+    end
+
+    def to_s
+      "#{super} <#{token}>"
     end
   end
 
@@ -71,25 +124,43 @@ module Nanite
   # target   is the target nanite for the request
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Request < Packet
+
     attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
+
     DEFAULT_OPTIONS = {:selector => :least_loaded}
-    def initialize(type, payload, opts={})
+
+    def initialize(type, payload, size=nil, opts={})
       opts = DEFAULT_OPTIONS.merge(opts)
-      @type             = type
-      @payload          = payload
-      @from             = opts[:from]
-      @token            = opts[:token]
-      @reply_to         = opts[:reply_to]
-      @selector         = opts[:selector]
-      @target           = opts[:target]
-      @persistent       = opts[:persistent]
-      @tags             = opts[:tags] || []
+      @type       = type
+      @payload    = payload
+      @size       = size
+      @from       = opts[:from]
+      @token      = opts[:token]
+      @reply_to   = opts[:reply_to]
+      @selector   = opts[:selector]
+      @target     = opts[:target]
+      @persistent = opts[:persistent]
+      @tags       = opts[:tags] || []
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :reply_to => i['reply_to'], :selector => i['selector'],
-      :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
+      new(i['type'], i['payload'], o['size'], { :from     => i['from'],     :token      => i['token'],
+                                                :reply_to => i['reply_to'], :selector   => i['selector'],
+                                                :target   => i['target'],   :persistent => i['persistent'],
+                                                :tags     => i['tags'] })
+    end
+
+    def to_s(filter=nil)
+      log_msg = "#{super} <#{token}> #{type}"
+      log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
+      log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
+      log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
+      log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
+      log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
+      wrap(log_msg)
     end
+
   end
 
   # packet that means a work push from mapper
@@ -105,23 +176,38 @@ module Nanite
   # target   is the target nanite for the request
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Push < Packet
+
     attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
+
     DEFAULT_OPTIONS = {:selector => :least_loaded}
-    def initialize(type, payload, opts={})
+
+    def initialize(type, payload, size=nil, opts={})
       opts = DEFAULT_OPTIONS.merge(opts)
-      @type             = type
-      @payload          = payload
-      @from             = opts[:from]
-      @token            = opts[:token]
-      @selector         = opts[:selector]
-      @target           = opts[:target]
-      @persistent       = opts[:persistent]
-      @tags             = opts[:tags] || []
+      @type       = type
+      @payload    = payload
+      @size       = size
+      @from       = opts[:from]
+      @token      = opts[:token]
+      @selector   = opts[:selector]
+      @target     = opts[:target]
+      @persistent = opts[:persistent]
+      @tags       = opts[:tags] || []
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :selector => i['selector'],
-      :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
+      new(i['type'], i['payload'], o['size'], { :from       => i['from'],       :token  => i['token'],
+                                                :selector   => i['selector'],   :target => i['target'],
+                                                :persistent => i['persistent'], :tags   => i['tags'] })
+    end
+
+    def to_s(filter=nil)
+      log_msg = "#{super} <#{token}> #{type}"
+      log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
+      log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
+      log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
+      log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
+      wrap(log_msg)
     end
   end
 
@@ -132,16 +218,28 @@ module Nanite
   # token    is a generated request id that mapper uses to identify replies
   # to       is identity of the node result should be delivered to
   class Result < Packet
+
     attr_accessor :token, :results, :to, :from
-    def initialize(token, to, results, from)
+
+    def initialize(token, to, results, from, size=nil)
       @token = token
       @to = to
       @from = from
       @results = results
+      @size = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['token'], i['to'], i['results'], i['from'])
+      new(i['token'], i['to'], i['results'], i['from'], o['size'])
+    end
+
+    def to_s(filter=nil)
+      log_msg = "#{super} <#{token}>"
+      log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
+      log_msg += " to #{id_to_s(to)}" if filter.nil? || filter.include?(:to)
+      log_msg += " results: #{results.inspect}" if filter.nil? || filter.include?(:results)
+      wrap(log_msg)
     end
   end
 
@@ -153,17 +251,25 @@ module Nanite
   # token    is a generated request id that mapper uses to identify replies
   # to       is identity of the node result should be delivered to
   class IntermediateMessage < Packet
+
     attr_accessor :token, :messagekey, :message, :to, :from
-    def initialize(token, to, from, messagekey, message)
-      @token = token
-      @to = to
-      @from = from
+
+    def initialize(token, to, from, messagekey, message, size=nil)
+      @token      = token
+      @to         = to
+      @from       = from
       @messagekey = messagekey
-      @message = message
+      @message    = message
+      @size       = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['token'], i['to'], i['from'], i['messagekey'], i['message'])
+      new(i['token'], i['to'], i['from'], i['messagekey'], i['message'], o['size'])
+    end
+
+    def to_s
+      wrap("#{super} <#{token}> from #{id_to_s(from)}, key #{messagekey}")
     end
   end
 
@@ -174,16 +280,27 @@ module Nanite
   # status   is a load of the node by default, but may be any criteria
   #          agent may use to report it's availability, load, etc
   class Register < Packet
+
     attr_accessor :identity, :services, :status, :tags
-    def initialize(identity, services, status, tags)
-      @status = status
-      @tags = tags
+
+    def initialize(identity, services, status, tags, size=nil)
+      @status   = status
+      @tags     = tags
       @identity = identity
       @services = services
+      @size     = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['identity'], i['services'], i['status'], i['tags'])
+      new(i['identity'], i['services'], i['status'], i['tags'], o['size'])
+    end
+
+    def to_s
+      log_msg = "#{super} #{id_to_s(identity)}"
+      log_msg += ", services: #{services.join(', ')}" if services && !services.empty?
+      log_msg += ", tags: #{tags.join(', ')}" if tags && !tags.empty?
+      wrap(log_msg)
     end
   end
 
@@ -191,13 +308,21 @@ module Nanite
   #
   # from     is sender identity
   class UnRegister < Packet
+
     attr_accessor :identity
-    def initialize(identity)
+
+    def initialize(identity, size=nil)
       @identity = identity
+      @size = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['identity'])
+      new(i['identity'], o['size'])
+    end
+  
+    def to_s
+      "#{super} #{id_to_s(identity)}"
     end
   end
 
@@ -206,26 +331,40 @@ module Nanite
   # identity is sender's identity
   # status   is sender's status (see Register packet documentation)
   class Ping < Packet
+
     attr_accessor :identity, :status
-    def initialize(identity, status)
-      @status = status
+
+    def initialize(identity, status, size=nil)
+      @status   = status
       @identity = identity
+      @size     = size
     end
+
     def self.json_create(o)
       i = o['data']
-      new(i['identity'], i['status'])
+      new(i['identity'], i['status'], o['size'])
+    end
+
+    def to_s
+      "#{super} #{id_to_s(identity)} status #{status}"
     end
+
   end
 
   # packet that is sent by workers to the mapper
   # when worker initially comes online to advertise
   # it's services
   class Advertise < Packet
-    def initialize
+
+    def initialize(size=nil)
+      @size = size
     end
+    
     def self.json_create(o)
-      new
+      new(o['size'])
     end
+
   end
+ 
 end
 

data/lib/nanite/state.rb

@@ -30,7 +30,7 @@ module Nanite
     # of these two service tags
     
     def initialize(redis)
-      Nanite::Log.info("initializing redis state: #{redis}")
+      Nanite::Log.info("[setup] initializing redis state: #{redis}")
       host, port = redis.split(':')
       host ||= '127.0.0.1'
       port ||= '6379'

data/spec/actor_registry_spec.rb

@@ -49,14 +49,14 @@ describe Nanite::ActorRegistry do
 
   it "should log info message that actor was registered" do
     importer = WebDocumentImporter.new
-    Nanite::Log.should_receive(:info).with("Registering #{importer.inspect} with prefix nil")
+    Nanite::Log.should_receive(:info).with("[actor] #{importer.class.to_s}")
     @registry.register(importer, nil)
   end
 
   it "should handle actors registered with a custom prefix" do
     importer = WebDocumentImporter.new
     @registry.register(importer, 'monkey')
-    @registry.actors['monkey'].should == importer
+    @registry.actor_for('monkey').should == importer
   end
   
 end # Nanite::ActorRegistry

data/spec/cluster_spec.rb

@@ -2,6 +2,8 @@ require File.join(File.dirname(__FILE__), 'spec_helper')
 
 describe Nanite::Cluster do
 
+  include SpecHelpers
+  
   describe "Intialization" do
 
     before(:each) do
@@ -91,6 +93,26 @@ describe Nanite::Cluster do
 
     end # Reaper
 
+    describe "State" do
+      begin
+        require 'nanite/state'
+      rescue LoadError
+      end
+
+      if defined?(Redis)
+        it "should use a local state by default" do
+          cluster = Nanite::Cluster.new(@amq, 443, "the_identity", @serializer, @mapper)
+          cluster.nanites.instance_of?(Nanite::LocalState).should == true
+        end
+      
+        it "should set up a redis state when requested" do
+          state = Nanite::State.new("")
+          Nanite::State.should_receive(:new).with("localhost:1234").and_return(state)
+          cluster = Nanite::Cluster.new(@amq, 443, "the_identity", @serializer, @mapper, "localhost:1234")
+          cluster.nanites.instance_of?(Nanite::State).should == true
+        end
+      end
+    end
   end # Intialization
 
 
@@ -189,9 +211,94 @@ describe Nanite::Cluster do
       @cluster.register(@register_packet)
     end
 
+    describe "with registered callbacks" do
+      before(:each) do
+        @register_callback = lambda {|request, mapper|}
+        @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper, nil, :register =>  @register_callback)
+      end
+      
+      it "should call the registration callback" do
+        @register_callback.should_receive(:call).with("nanite_id", @mapper)
+        @cluster.register(@register_packet)
+      end
+    end
+    
+    describe "when sending an invalid packet to the registration queue" do
+      it "should log a message statement" do
+        Nanite::Log.logger.should_receive(:warn).with("RECV [register] Invalid packet type: Nanite::Ping")
+        @cluster.register(Nanite::Ping.new(nil, nil))
+      end
+    end
   end # Nanite Registration
 
-
+  describe "Unregister" do
+    before(:each) do
+      @fanout = mock("fanout")
+      @binding = mock("binding", :subscribe => true)
+      @queue = mock("queue", :bind => @binding)
+      @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
+      @serializer = mock("Serializer")
+      @reaper = mock("Reaper", :timeout => true)
+      Nanite::Log.stub!(:info)
+      Nanite::Reaper.stub!(:new).and_return(@reaper)
+      @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
+      @cluster.nanites["nanite_id"] = "nanite_id"
+      @unregister_packet = Nanite::UnRegister.new("nanite_id")
+    end
+    
+    it "should delete the nanite" do
+      @cluster.register(@unregister_packet)
+      @cluster.nanites["nanite_id"].should == nil
+    end
+    
+    describe "with registered callbacks" do
+      before(:each) do
+        @unregister_callback = lambda {|request, mapper| }
+        @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper, nil, :unregister => @unregister_callback)
+        @cluster.nanites["nanite_id"] = "nanite_id"
+      end
+      
+      it "should call the unregister callback" do
+        @unregister_callback.should_receive(:call).with("nanite_id", @mapper)
+        @cluster.register(@unregister_packet)
+      end
+    end
+  end
+  
+  describe "Nanite timed out" do
+    before(:each) do
+      @fanout = mock("fanout")
+      @binding = mock("binding", :subscribe => true)
+      @queue = mock("queue", :bind => @binding)
+      @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
+      @serializer = mock("Serializer")
+      Nanite::Log.stub!(:info)
+      @register_packet = Nanite::Register.new("nanite_id", ["the_nanite_services"], "nanite_status",[])
+    end
+    
+    it "should remove the nanite when timed out" do
+      EM.run do
+        @cluster = Nanite::Cluster.new(@amq, 0.01, "the_identity", @serializer, @mapper)
+        @cluster.register(@register_packet)
+        EM.add_timer(1.1) {
+          @cluster.nanites["nanite_id"].should == nil
+          EM.stop_event_loop
+        }
+      end
+    end
+    
+    it "should call the timed out callback handler when registered" do
+      EM.run do
+        @cluster = Nanite::Cluster.new(@amq, 0.01, "the_identity", @serializer, @mapper)
+        @cluster.register(@register_packet)
+        EM.add_timer(1.1) {
+          @cluster.nanites["nanite_id"].should == nil
+          EM.stop_event_loop
+        }
+      end
+    end
+  end
+  
   describe "Route" do
 
     before(:each) do
@@ -231,7 +338,7 @@ describe Nanite::Cluster do
       @reaper = mock("Reaper")
       Nanite::Reaper.stub!(:new).and_return(@reaper)
       @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
-      @request = mock("Request", :persistent => true)
+      @request = mock("Request", :persistent => true, :target => nil, :target= => nil, :to_s => nil)
       @target = mock("Target of Request")
     end
 
@@ -274,9 +381,10 @@ describe Nanite::Cluster do
       @reaper = mock("Reaper")
       Nanite::Reaper.stub!(:new).and_return(@reaper)
       @request_without_target = mock("Request", :target => nil, :token => "Token",
-       :reply_to => "Reply To", :from => "From", :persistent => true, :identity => "Identity")
+       :reply_to => "Reply To", :from => "From", :persistent => true, :identity => "Identity",
+       :payload => "Payload", :to_s => nil)
       @request_with_target = mock("Request", :target => "Target", :token => "Token",
-       :reply_to => "Reply To", :from => "From", :persistent => true)
+       :reply_to => "Reply To", :from => "From", :persistent => true, :payload => "Payload", :to_s => nil)
       @mapper_with_target = mock("Mapper", :identity => "id")
       @mapper_without_target = mock("Mapper", :request => false, :identity => @request_without_target.identity)
       @cluster_with_target = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper_with_target)
@@ -292,9 +400,78 @@ describe Nanite::Cluster do
     it "should reply back with nil results for requests with no target when offline queue is disabled" do
       @mapper_without_target.should_receive(:send_request).with(@request_without_target, anything())
       Nanite::Result.should_receive(:new).with(@request_without_target.token, @request_without_target.from, nil, @request_without_target.identity)
-       @cluster_without_target.__send__(:handle_request, @request_without_target)
+      @cluster_without_target.__send__(:handle_request, @request_without_target)
     end
     
+    it "should hand in an intermediate handler" do
+      @mapper_with_target.should_receive(:send_request) do |request, opts|
+        opts[:intermediate_handler].should be_instance_of(Proc)
+      end
+      
+      @cluster_with_target.__send__(:handle_request, @request_with_target)
+    end
+
+    it "should forward the message when send_request failed" do
+      @mapper_with_target.stub!(:send_request).and_return(false)
+      @cluster_with_target.should_receive(:forward_response)
+      @cluster_with_target.__send__(:handle_request, @request_with_target)
+    end
   end # Agent Request Handling
 
+  describe "Heartbeat" do
+    before(:each) do
+      @fanout = mock("fanout")
+      @binding = mock("binding", :subscribe => true)
+      @queue = mock("queue", :bind => @binding, :publish => true)
+      @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
+      @serializer = mock("Serializer", :dump => "dumped_value")
+      Nanite::Log.stub!(:info)
+      @ping = stub("ping", :status => 0.3, :identity => "nanite_id")
+    end
+    
+    it "should update the nanite status" do
+      run_in_em do
+        @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
+        @cluster.nanites["nanite_id"] = {:status => "nanite_status"}
+        @cluster.send :handle_ping, @ping
+        @cluster.nanites["nanite_id"][:status].should == 0.3
+      end
+    end
+    
+    it "should reset the agent time out" do
+      run_in_em do
+        @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
+        @cluster.reaper.should_receive(:reset_with_autoregister_hack).with("nanite_id", 33)
+        @cluster.nanites["nanite_id"] = {:status => "nanite_status"}
+        @cluster.send :handle_ping, @ping
+      end
+    end
+    
+    describe "when timing out after a heartbeat" do
+      it "should remove the nanite" do
+        run_in_em(false) do
+          @cluster = Nanite::Cluster.new(@amq, 0.1, "the_identity", @serializer, @mapper)
+          @cluster.nanites["nanite_id"] = {:status => "nanite_status"}
+          @cluster.send :handle_ping, @ping
+          EM.add_timer(1.5) do
+            @cluster.nanites["nanite_id"].should == nil
+            EM.stop_event_loop
+          end
+        end
+      end
+      
+      it "should call the timeout callback when defined" do
+        run_in_em(false) do
+          @timeout_callback = lambda {|nanite, mapper| }
+          @timeout_callback.should_receive(:call).with("nanite_id", @mapper)
+          @cluster = Nanite::Cluster.new(@amq, 0.1, "the_identity", @serializer, @mapper, nil, :timeout => @timeout_callback)
+          @cluster.nanites["nanite_id"] = {:status => "nanite_status"}
+          @cluster.send :handle_ping, @ping
+          EM.add_timer(1.5) do
+            EM.stop_event_loop
+          end
+        end
+      end
+    end
+  end
 end # Nanite::Cluster

data/spec/job_spec.rb

@@ -33,7 +33,44 @@ describe Nanite::JobWarden do
 
   end # Creating a new Job
 
-
+  describe "Processing an intermediate message" do
+    before(:each) do
+      @intm_handler = lambda {|arg1, arg2, arg3| puts 'ehlo'}
+      @message = mock("Message", :token => "3faba24fcc", :from => 'nanite-agent')
+      @serializer = mock("Serializer", :load => @message)
+      @warden = Nanite::JobWarden.new(@serializer)
+      @job = Nanite::Job.new(stub("request", :token => "3faba24fcc"), [], @intm_handler)
+      @job.instance_variable_set(:@pending_keys, ["defaultkey"])
+      @job.instance_variable_set(:@intermediate_state, {"nanite-agent" => {"defaultkey" => [1]}})
+      @warden.jobs[@job.token] = @job
+      Nanite::Log.stub!(:debug)
+      Nanite::Log.stub!(:error)
+    end
+    
+    it "should call the intermediate handler with three parameters" do
+      @intm_handler.should_receive(:call).with("defaultkey", "nanite-agent", 1)
+      @warden.process(@message)
+    end
+    
+    it "should call the intermediate handler with four parameters" do
+      @intm_handler.stub!(:arity).and_return(4)
+      @intm_handler.should_receive(:call).with("defaultkey", "nanite-agent", 1, @job)
+      @warden.process(@message)
+    end
+    
+    it "should not call the intermediate handler when it can't be found for the specified key" do
+      @intm_handler.should_not_receive(:call)
+      @job.instance_variable_set(:@intermediate_handler, nil)
+      @warden.process(@message)
+    end
+    
+    it "should call the intermediate handler with one parameter which needs to be the result" do
+      @intm_handler.should_receive(:call).with(1, @job)
+      @intm_handler.stub!(:arity).and_return(2)
+      @warden.process(@message)
+    end
+  end
+  
   describe "Processing a Message" do
 
     before(:each) do
@@ -44,11 +81,6 @@ describe Nanite::JobWarden do
       Nanite::Log.stub!(:debug)
     end
 
-    it "should log debug message about message to be processed" do
-      Nanite::Log.should_receive(:debug)
-      @warden.process(@message)
-    end
-
     it "should hand over processing to job" do
       Nanite::Job.stub!(:new).and_return(@job)
       @job.should_receive(:process).with(@message)

data/spec/spec_helper.rb

@@ -6,7 +6,10 @@ require 'spec'
 require 'nanite'
 
 module SpecHelpers
-  
+
+  # Initialize logger so it writes to file instead of STDOUT
+  Nanite::Log.init('test', File.join(File.dirname(__FILE__)))
+
   # Create test certificate
   def issue_cert
     test_dn = { 'C'  => 'US',
@@ -20,4 +23,11 @@ module SpecHelpers
     [ Nanite::Certificate.new(key, dn, dn), key ]
   end
 
+  def run_in_em(stop_event_loop = true)
+    EM.run do
+      yield
+      EM.stop_event_loop if stop_event_loop
+    end
+  end
+  
 end  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ezmobius-nanite
 version: !ruby/object:Gem::Version 
-  version: 0.4.1.1
+  version: 0.4.1.2
 platform: ruby
 authors: 
 - Ezra Zygmuntowicz
@@ -102,6 +102,7 @@ files:
 - spec/distinguished_name_spec.rb
 has_rdoc: true
 homepage: http://github.com/ezmobius/nanite
+licenses: 
 post_install_message: 
 rdoc_options: []
 
@@ -122,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 2
 summary: self assembling fabric of ruby daemons