nofxx-nanite 0.4.1.2 → 0.4.1.15

data/Rakefile

@@ -10,7 +10,7 @@ end
 require 'rake/clean'
 require 'lib/nanite'
 
-GEM = "nanite"
+GEM = "nofxx-nanite"
 AUTHOR = "Ezra Zygmuntowicz"
 EMAIL = "ezra@engineyard.com"
 HOMEPAGE = "http://github.com/ezmobius/nanite"
@@ -35,6 +35,7 @@ spec = Gem::Specification.new do |s|
   s.executables  = %w( nanite-agent nanite-mapper nanite-admin )
 
   s.add_dependency('amqp', '>= 0.6.0')
+  s.add_dependency('json', '>= 1.1.7')
 
   s.require_path = 'lib'
   s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*")
@@ -70,7 +71,7 @@ task :docs => :rdoc do
   case RUBY_PLATFORM
   when /darwin/       ; sh 'open rdoc/index.html'
   when /mswin|mingw/  ; sh 'start rdoc\index.html'
-  else 
+  else
     sh 'firefox rdoc/index.html'
   end
 end

data/bin/nanite-agent

@@ -42,6 +42,9 @@ opts = OptionParser.new do |opts|
     options[:threadpool_size] = tps
   end
   
+  opts.on("--prefetch COUNT", Integer, "The number of messages stuffed into the queue at anytime.  Set this to a value of 1 or so for longer running jobs (1 or more seconds), so the agent does not get overwhelmed.  Default is unlimited.") do |pref|
+    options[:prefetch] = pref
+  end
 end
 
 opts.parse!

data/lib/nanite.rb

@@ -5,6 +5,7 @@ require 'json'
 require 'logger'
 require 'yaml'
 require 'openssl'
+require 'fileutils'
 
 $:.unshift File.dirname(__FILE__)
 require 'nanite/amqp'
@@ -20,7 +21,7 @@ require 'nanite/mapper'
 require 'nanite/actor'
 require 'nanite/actor_registry'
 require 'nanite/streaming'
-require 'nanite/dispatcher'
+require 'nanite/nanite_dispatcher'
 require 'nanite/agent'
 require 'nanite/cluster'
 require 'nanite/reaper'
@@ -39,7 +40,7 @@ require 'nanite/security/static_certificate_store'
 require 'nanite/serializer'
 
 module Nanite
-  VERSION = '0.4.1.2' unless defined?(Nanite::VERSION)
+  VERSION = '0.4.1.15' unless defined?(Nanite::VERSION)
 
   class MapperNotRunning < StandardError; end
 
@@ -54,6 +55,11 @@ module Nanite
       @mapper = Nanite::Mapper.start(options)
     end
 
+    def start_mapper_proxy(options = {})
+      identity = options[:identity] || Nanite::Identity.generate
+      @mapper = Nanite::MapperProxy.new(identity, options)
+    end
+      
     def request(*args, &blk)
       ensure_mapper
       @mapper.request(*args, &blk)

data/lib/nanite/agent.rb

@@ -55,6 +55,9 @@ module Nanite
     # services    : list of services provided by this agent, by default
     #               all methods exposed by actors are listed
     #
+    # prefetch    : Sets prefetch (only supported in RabbitMQ >= 1.6).  Use value of 1 for long
+    #               running jobs (greater than a second) to avoid slamming/stalling your agent.
+    #
     # single_threaded: Run all operations in one thread
     #
     # threadpool_size: Number of threads to run operations in
@@ -92,18 +95,14 @@ module Nanite
     end
     
     def run
-      log_path = false
-      if @options[:daemonize]
-        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
-      end
-      Log.init(@identity, log_path)
+      Log.init(@identity, @options[:log_path])
       Log.level = @options[:log_level] if @options[:log_level]
       @serializer = Serializer.new(@options[:format])
       @status_proc = lambda { parse_uptime(`uptime 2> /dev/null`) rescue 'no status' }
       pid_file = PidFile.new(@identity, @options)
       pid_file.check
       if @options[:daemonize]
-        daemonize
+        daemonize(@identity, @options)
         pid_file.write
         at_exit { pid_file.remove }
       end
@@ -149,6 +148,11 @@ module Nanite
       opts.delete(:identity) unless opts[:identity]
       @options.update(custom_config.merge(opts))
       @options[:file_root] ||= File.join(@options[:root], 'files')
+      @options[:log_path] = false
+      if @options[:daemonize]
+        @options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      
       return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
       token = Identity.generate
       @identity = "nanite-#{token}"
@@ -160,6 +164,7 @@ module Nanite
     def load_actors
       return unless options[:root]
       actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
+      Nanite::Log.warn("Actors dir #{actors_dir} does not exist or is not reachable") unless File.directory?(actors_dir)
       actors = @options[:actors]
       Dir["#{actors_dir}/*.rb"].each do |actor|
         next if actors && !actors.include?(File.basename(actor, ".rb"))
@@ -167,14 +172,17 @@ module Nanite
         require actor
       end
       init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
-      instance_eval(File.read(init_path), init_path) if File.exist?(init_path)
+      if File.exist?(init_path)
+        instance_eval(File.read(init_path), init_path) 
+      else
+        Nanite::Log.warn("init.rb #{init_path} does not exist or is not reachable") unless File.exists?(init_path)
+      end
     end
 
     def receive(packet)
       Nanite::Log.debug("RECV #{packet.to_s}")
       case packet
       when Advertise
-        Nanite::Log.info("RECV #{packet.to_s}") unless Nanite::Log.level == :debug
         advertise_services
       when Request, Push
         if @security && !@security.authorize(packet)
@@ -184,14 +192,11 @@ module Nanite
             amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
           end
         else
-          Nanite::Log.info("RECV #{packet.to_s([:from, :tags])}") unless Nanite::Log.level == :debug
           dispatcher.dispatch(packet)
         end
       when Result
-        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
         @mapper_proxy.handle_result(packet)
       when IntermediateMessage
-        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
         @mapper_proxy.handle_intermediate_result(packet)
       end
     end
@@ -202,6 +207,9 @@ module Nanite
     end
 
     def setup_queue
+      if amq.respond_to?(:prefetch) && @options.has_key?(:prefetch)
+        amq.prefetch(@options[:prefetch])
+      end
       amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
         begin
           info.ack

data/lib/nanite/amqp.rb

@@ -1,5 +1,18 @@
 class MQ
   class Queue
+    
+    # Monkey patch to add :no_declare => true for new queue objects. See the
+    # explanation for MQ::Exchange#initialize below.
+    def initialize mq, name, opts = {}
+      @mq = mq
+      @opts = opts
+      @bindings ||= {}
+      @mq.queues[@name = name] ||= self
+      @mq.callback{
+      @mq.send Protocol::Queue::Declare.new({ :queue => name,
+                                              :nowait => true }.merge(opts))
+      } unless opts[:no_declare]
+    end
     # Asks the broker to redeliver all unacknowledged messages on a
     # specifieid channel. Zero or more messages may be redelivered.
     #
@@ -46,7 +59,11 @@ module Nanite
         :host => options[:host],
         :port => (options[:port] || ::AMQP::PORT).to_i,
         :insist => options[:insist] || false,
-        :retry => options[:retry] || 5
+        :retry => options[:retry] || 5,
+        :connection_status => options[:connection_callback] || proc {|event| 
+          Nanite::Log.debug("Connected to MQ") if event == :connected
+          Nanite::Log.debug("Disconnected from MQ") if event == :disconnected
+        }
       })
       MQ.new(connection)
     end

data/lib/nanite/cluster.rb

@@ -29,15 +29,16 @@ module Nanite
       case reg
       when Register
         if @security.authorize_registration(reg)
-          Nanite::Log.info("RECV #{reg.to_s}")
-          nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags }
-          reaper.timeout(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
+          Nanite::Log.debug("RECV #{reg.to_s}")
+          nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
+          reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
           callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
         else
           Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
         end
       when UnRegister
         Nanite::Log.info("RECV #{reg.to_s}")
+        reaper.unregister(reg.identity)
         nanites.delete(reg.identity)
         callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
       else
@@ -46,8 +47,13 @@ module Nanite
     end
 
     def nanite_timed_out(token)
-      nanite = nanites.delete(token)
-      callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
+      nanite = nanites[token]
+      if nanite && timed_out?(nanite)
+        Nanite::Log.info("Nanite #{token} timed out")
+        nanite = nanites.delete(token)
+        callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
+        true
+      end
     end
 
     def route(request, targets)
@@ -60,8 +66,8 @@ module Nanite
       begin
         old_target = request.target
         request.target = target unless target == 'mapper-offline'
-        Nanite::Log.info("SEND #{request.to_s([:from, :tags, :target])}")
-        amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
+        Nanite::Log.debug("SEND #{request.to_s([:from, :tags, :target])}")
+        amq.queue(target).publish(serializer.dump(request, enforce_format?(target)), :persistent => request.persistent)
       ensure
         request.target = old_target
       end
@@ -69,16 +75,20 @@ module Nanite
 
     protected
 
+    def enforce_format?(target)
+      target == 'mapper-offline' ? :insecure : nil
+    end
+    
     # updates nanite information (last ping timestamps, status)
     # when heartbeat message is received
     def handle_ping(ping)
       begin
         if nanite = nanites[ping.identity]
-          nanite[:status] = ping.status
-          reaper.reset_with_autoregister_hack(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
+          nanites.update_status(ping.identity, ping.status)
+          reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
         else
-          packet = Advertise.new
-          Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
+          packet = Advertise.new(nil, ping.identity)
+          Nanite::Log.debug("SEND #{packet.to_s} to #{ping.identity}")
           amq.queue(ping.identity).publish(serializer.dump(packet))
         end
       end
@@ -87,7 +97,6 @@ module Nanite
     # forward request coming from agent
     def handle_request(request)
       if @security.authorize_request(request)
-        Nanite::Log.info("RECV #{request.to_s([:from, :target, :tags])}") unless Nanite::Log.level == :debug
         Nanite::Log.debug("RECV #{request.to_s}")
         case request
         when Push
@@ -115,7 +124,7 @@ module Nanite
 
     # forward response back to agent that originally made the request
     def forward_response(res, persistent)
-      Nanite::Log.info("SEND #{res.to_s([:to])}")
+      Nanite::Log.debug("SEND #{res.to_s([:to])}")
       amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
     end
 
@@ -152,10 +161,21 @@ module Nanite
       @last[service] += 1
       [candidate]
     end
+    
+    def timed_out?(nanite)
+      nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
+    end
 
     # returns all nanites that provide the given service
     def nanites_providing(service, *tags)
-      nanites.nanites_for(service, *tags)
+      nanites.nanites_for(service, *tags).delete_if do |nanite|
+        nanite_id, nanite_attributes = nanite
+        if timed_out?(nanite_attributes)
+          reaper.unregister(nanite_id)
+          nanites.delete(nanite_id)
+          Nanite::Log.debug("Nanite #{nanite_id} timed out - ignoring in target selection and deleting from state - last seen at #{nanite_attributes[:timestamp]}")
+        end
+      end
     end
 
     def setup_queues

data/lib/nanite/config.rb

@@ -31,7 +31,7 @@ module Nanite
       opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do
         options[:offline_failsafe] = true
       end
-      
+
       opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis|
         redishost, redisport = redis.split(':')
         redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?)

data/lib/nanite/daemonize.rb

@@ -1,13 +1,13 @@
 module Nanite
   module DaemonizeHelper
-    def daemonize
+    def daemonize(identity, options = {})
       exit if fork
       Process.setsid
       exit if fork
-      File.umask 0000
       STDIN.reopen "/dev/null"
-      STDOUT.reopen "/dev/null", "a"
-      STDERR.reopen STDOUT
+      STDOUT.reopen "#{options[:log_path]}/nanite.#{identity}.out", "a"
+      STDERR.reopen "#{options[:log_path]}/nanite.#{identity}.err", "a"
+      File.umask 0000
     end
   end
 end

data/lib/nanite/local_state.rb

@@ -24,6 +24,10 @@ module Nanite
       end.to_a
     end
 
+    def update_status(name, status)
+      self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
+    end
+    
     private
 
     def all(key)

data/lib/nanite/log.rb

@@ -35,7 +35,7 @@ module Nanite
       # Throws an ArgumentError if you feed it a bogus log level (that is not
       # one of :debug, :info, :warn, :error, :fatal or the corresponding strings or a valid Logger level)
       def level=(loglevel)
-        init() unless @logger
+        init unless @logger
         lvl = case loglevel
           when String  then loglevel.intern
           when Integer then LEVELS.invert[loglevel]

data/lib/nanite/mapper.rb

@@ -117,21 +117,18 @@ module Nanite
       @options.update(custom_config.merge(options))
       @identity = "mapper-#{@options[:identity]}"
       @options[:file_root] ||= File.join(@options[:root], 'files')
+      @options[:log_path] = false
+      if @options[:daemonize]
+        @options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
       @options.freeze
+      @offline_queue = 'mapper-offline'
     end
 
     def run
       setup_logging
       @serializer = Serializer.new(@options[:format])
-      pid_file = PidFile.new(@identity, @options)
-      pid_file.check
-      if @options[:daemonize]
-        daemonize
-        pid_file.write
-        at_exit { pid_file.remove }
-      else
-        trap("INT") {exit}
-      end
+      setup_process
       @amq = start_amqp(@options)
       @job_warden = JobWarden.new(@serializer)
       setup_cluster
@@ -192,9 +189,9 @@ module Nanite
         job = job_warden.new_job(request, targets, intm_handler, blk)
         cluster.route(request, job.targets)
         job
-      elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
+      elsif offline_failsafe?(opts)
         job_warden.new_job(request, [], intm_handler, blk)
-        cluster.publish(request, 'mapper-offline')
+        cluster.publish(request, @offline_queue)
         :offline
       else
         false
@@ -231,14 +228,22 @@ module Nanite
       if !targets.empty?
         cluster.route(push, targets)
         true
-      elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
-        cluster.publish(push, 'mapper-offline')
+      elsif offline_failsafe?(opts)
+        cluster.publish(push, @offline_queue)
         :offline
       else
         false
       end
     end
 
+    # <<<<<<< HEAD
+
+    def offline_failsafe?(opts)
+      opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
+    end
+
+    # =======
+
     private
 
     def build_deliverable(deliverable_type, type, payload, opts)
@@ -259,11 +264,12 @@ module Nanite
     end
 
     def setup_offline_queue
-      offline_queue = amq.queue('mapper-offline', :durable => true)
+      offline_queue = amq.queue(@offline_queue, :durable => true)
       offline_queue.subscribe(:ack => true) do |info, deliverable|
-        deliverable = serializer.load(deliverable)
+        deliverable = serializer.load(deliverable, :insecure)
         targets = cluster.targets_for(deliverable)
         unless targets.empty?
+          Nanite::Log.debug("Recovering message from offline queue: #{deliverable.to_s([:from, :tags, :target])}")
           info.ack
           if deliverable.kind_of?(Request)
             if job = job_warden.jobs[deliverable.token]
@@ -285,7 +291,6 @@ module Nanite
         begin
           msg = serializer.load(msg)
           Nanite::Log.debug("RECV #{msg.to_s}")
-          Nanite::Log.info("RECV #{msg.to_s([:from])}") unless Nanite::Log.level == :debug
           job_warden.process(msg)
         rescue Exception => e
           Nanite::Log.error("RECV [result] #{e.message}")
@@ -294,17 +299,25 @@ module Nanite
     end
 
     def setup_logging
-      log_path = false
-      if @options[:daemonize]
-        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
-      end
-      Nanite::Log.init(@identity, log_path)
+      Nanite::Log.init(@identity, @options[:log_path])
       Nanite::Log.level = @options[:log_level] if @options[:log_level]
     end
 
     def setup_cluster
       @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
     end
+
+    def setup_process
+      pid_file = PidFile.new(@identity, @options)
+      pid_file.check
+      if @options[:daemonize]
+        daemonize(@identity, @options)
+        pid_file.write
+        at_exit { pid_file.remove }
+      else
+        trap("INT") {exit}
+      end
+    end
   end
 end