shift-nanite 0.4.1.2

data/lib/nanite/cluster.rb

@@ -0,0 +1,250 @@
+module Nanite
+  class Cluster
+    attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
+
+    def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
+      @amq = amq
+      @agent_timeout = agent_timeout
+      @identity = identity
+      @serializer = serializer
+      @mapper = mapper
+      @state = state_configuration
+      @security = SecurityProvider.get
+      @callbacks = callbacks
+      setup_state
+      @reaper = Reaper.new(agent_timeout)
+      setup_queues
+    end
+
+    # determine which nanites should receive the given request
+    def targets_for(request)
+      return [request.target] if request.target
+      __send__(request.selector, request.type, request.tags).collect {|name, state| name }
+    end
+
+    # adds nanite to nanites map: key is nanite's identity
+    # and value is a services/status pair implemented
+    # as a hash
+    def register(reg)
+      case reg
+      when Register
+        if @security.authorize_registration(reg)
+          Nanite::Log.info("RECV #{reg.to_s}")
+          nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
+          reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
+          callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
+        else
+          Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
+        end
+      when UnRegister
+        Nanite::Log.info("RECV #{reg.to_s}")
+        reaper.unregister(reg.identity)
+        nanites.delete(reg.identity)
+        callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
+      else
+        Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
+      end
+    end
+
+    def nanite_timed_out(token)
+      nanite = nanites[token]
+      if nanite && timed_out?(nanite)
+        Nanite::Log.info("Nanite #{token} timed out")
+        nanite = nanites.delete(token)
+        callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
+        true
+      end
+    end
+    
+    def route(request, targets)
+      EM.next_tick { targets.map { |target| publish(request, target) } }
+    end
+
+    def publish(request, target)
+      # We need to initialize the 'target' field of the request object so that the serializer has
+      # access to it.
+      begin
+        old_target = request.target
+        request.target = target unless target == 'mapper-offline'
+        Nanite::Log.info("SEND #{request.to_s([:from, :tags, :target])}")
+        amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
+      ensure
+        request.target = old_target
+      end
+    end
+
+    protected
+
+    # updates nanite information (last ping timestamps, status)
+    # when heartbeat message is received
+    def handle_ping(ping)
+      begin
+        if nanite = nanites[ping.identity]
+          nanites.update_status(ping.identity, ping.status)
+          reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
+        else
+          packet = Advertise.new
+          Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
+          amq.queue(ping.identity).publish(serializer.dump(packet))
+        end
+      end
+    end
+    
+    # forward request coming from agent
+    def handle_request(request)
+      if @security.authorize_request(request)
+        Nanite::Log.info("RECV #{request.to_s([:from, :target, :tags])}") unless Nanite::Log.level == :debug
+        Nanite::Log.debug("RECV #{request.to_s}")
+        case request
+        when Push
+          mapper.send_push(request)
+        else
+          intm_handler = lambda do |result, job|
+            result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
+            forward_response(result, request.persistent)
+          end
+        
+          result = Result.new(request.token, request.from, nil, mapper.identity)
+          ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
+            result.results = res
+            forward_response(result, request.persistent)
+          end
+          
+          if ok == false
+            forward_response(result, request.persistent)
+          end
+        end
+      else
+        Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
+      end
+    end
+    
+    # forward response back to agent that originally made the request
+    def forward_response(res, persistent)
+      Nanite::Log.info("SEND #{res.to_s([:to])}")
+      amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
+    end
+    
+    # returns least loaded nanite that provides given service
+    def least_loaded(service, tags=[])
+      candidates = nanites_providing(service,tags)
+      return [] if candidates.empty?
+
+      [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
+    end
+
+    # returns all nanites that provide given service
+    def all(service, tags=[])
+      nanites_providing(service,tags)
+    end
+
+    # returns a random nanite
+    def random(service, tags=[])
+      candidates = nanites_providing(service,tags)
+      return [] if candidates.empty?
+
+      [candidates[rand(candidates.size)]]
+    end
+
+    # selects next nanite that provides given service
+    # using round robin rotation
+    def rr(service, tags=[])
+      @last ||= {}
+      @last[service] ||= 0
+      candidates = nanites_providing(service,tags)
+      return [] if candidates.empty?
+      @last[service] = 0 if @last[service] >= candidates.size
+      candidate = candidates[@last[service]]
+      @last[service] += 1
+      [candidate]
+    end
+    
+    def timed_out?(nanite)
+      nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
+    end
+
+    # returns all nanites that provide the given service
+    def nanites_providing(service, *tags)
+      nanites.nanites_for(service, *tags).delete_if do |nanite| 
+        if timed_out?(nanite[1])
+          Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
+        end
+      end
+    end
+
+    def setup_queues
+      setup_heartbeat_queue
+      setup_registration_queue
+      setup_request_queue
+    end
+
+    def setup_heartbeat_queue
+      handler = lambda do |ping|
+        begin
+          ping = serializer.load(ping)
+          Nanite::Log.debug("RECV #{ping.to_s}") if ping.respond_to?(:to_s)
+          handle_ping(ping)
+        rescue Exception => e
+          Nanite::Log.error("RECV [ping] #{e.message}")
+        end
+      end
+      hb_fanout = amq.fanout('heartbeat', :durable => true)
+      if shared_state?
+        amq.queue("heartbeat").bind(hb_fanout).subscribe &handler
+      else
+        amq.queue("heartbeat-#{identity}", :exclusive => true).bind(hb_fanout).subscribe &handler
+      end
+    end
+
+    def setup_registration_queue
+      handler = lambda do |msg|
+        begin
+          register(serializer.load(msg))
+        rescue Exception => e
+          Nanite::Log.error("RECV [register] #{e.message}")
+        end
+      end
+      reg_fanout = amq.fanout('registration', :durable => true)
+      if shared_state?
+        amq.queue("registration").bind(reg_fanout).subscribe &handler
+      else
+        amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
+      end
+    end
+    
+    def setup_request_queue
+      handler = lambda do |msg|
+        begin
+          handle_request(serializer.load(msg))
+        rescue Exception => e
+          Nanite::Log.error("RECV [request] #{e.message}")
+        end
+      end
+      req_fanout = amq.fanout('request', :durable => true)
+      if shared_state?
+        amq.queue("request").bind(req_fanout).subscribe &handler
+      else
+        amq.queue("request-#{identity}", :exclusive => true).bind(req_fanout).subscribe &handler
+      end
+    end
+
+    def setup_state
+      case @state
+      when String
+        # backwards compatibility, we assume redis if the configuration option
+        # was a string
+        Nanite::Log.info("[setup] using redis for state storage")
+        require 'nanite/state'
+        @nanites = Nanite::State.new(@state)
+      when Hash
+      else
+        require 'nanite/local_state'
+        @nanites = Nanite::LocalState.new
+      end
+    end
+    
+    def shared_state?
+      !@state.nil?
+    end
+  end
+end

data/lib/nanite/config.rb

@@ -0,0 +1,112 @@
+module Nanite
+
+  COMMON_DEFAULT_OPTIONS = {
+    :pass => 'testing',
+    :vhost => '/nanite',
+    :secure => false,
+    :host => '0.0.0.0',
+    :log_level => :info,
+    :format => :marshal,
+    :daemonize => false,
+    :console => false,
+    :root => Dir.pwd,
+    :insist => true
+  }
+
+  module CommonConfig
+    def setup_mapper_options(opts, options)
+      setup_common_options(opts, options, 'mapper')
+
+      opts.on("-a", "--agent-timeout TIMEOUT", "How long to wait before an agent is considered to be offline and thus removed from the list of available agents.") do |timeout|
+        options[:agent_timeout] = timeout
+      end
+
+      opts.on("-r", "--offline-redelivery-frequency", "The frequency in seconds that messages stored in the offline queue will be retrieved for attempted redelivery to the nanites. Default is 10 seconds.") do |frequency|
+        options[:offline_redelivery_frequency] = frequency
+      end
+
+      opts.on("--persistent", "Instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the broker is restarted. Can be overriden on a per-message basis using the request and push methods.") do
+        options[:persistent] = true
+      end
+
+      opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do
+        options[:offline_failsafe] = true
+      end
+      
+      opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis|
+        redishost, redisport = redis.split(':')
+        redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?)
+        redisport = '6379' if (redishost.nil? || redishost.empty?)
+        redis = "#{redishost}:#{redisport}"
+        options[:redis] = redis
+      end
+      
+    end
+    
+    def setup_common_options(opts, options, type)
+      opts.version = Nanite::VERSION
+
+      opts.on("-i", "--irb-console", "Start #{type} in irb console mode.") do |console|
+        options[:console] = 'irb'
+      end
+
+      opts.on("-u", "--user USER", "Specify the rabbitmq username.") do |user|
+        options[:user] = user
+      end
+
+      opts.on("-h", "--host HOST", "Specify the rabbitmq hostname.") do |host|
+        options[:host] = host
+      end
+
+      opts.on("-P", "--port PORT", "Specify the rabbitmq PORT, default 5672.") do |port|
+        options[:port] = port
+      end
+
+      opts.on("-p", "--pass PASSWORD", "Specify the rabbitmq password") do |pass|
+        options[:pass] = pass
+      end
+
+      opts.on("-t", "--token IDENITY", "Specify the #{type} identity.") do |ident|
+        options[:identity] = ident
+      end
+
+      opts.on("-v", "--vhost VHOST", "Specify the rabbitmq vhost") do |vhost|
+        options[:vhost] = vhost
+      end
+
+      opts.on("-s", "--secure", "Use Security features of rabbitmq to restrict nanites to themselves") do
+        options[:secure] = true
+      end
+
+      opts.on("-f", "--format FORMAT", "The serialization type to use for transfering data. Can be marshal, json or yaml. Default is marshal") do |fmt|
+        options[:format] = fmt.to_sym
+      end
+
+      opts.on("-d", "--daemonize", "Run #{type} as a daemon") do |d|
+        options[:daemonize] = true
+      end
+      
+      opts.on("--pid-dir PATH", "Specify the pid path, only used with daemonize") do |dir|
+        options[:pid_dir] = dir
+      end
+
+      opts.on("-l", "--log-level LEVEL", "Specify the log level (fatal, error, warn, info, debug). Default is info") do |level|
+        options[:log_level] = level
+      end
+      
+      opts.on("--log-dir PATH", "Specify the log path") do |dir|
+        options[:log_dir] = dir
+      end
+
+      opts.on("--tag TAG", "Specify a tag.  Can issue multiple times.") do |tag|
+        options[:tag] ||= []
+        options[:tag] << tag
+      end
+
+      opts.on("--version", "Show the nanite version number") do |res|
+        puts "Nanite Version #{opts.version}"
+        exit
+      end
+    end
+  end
+end

data/lib/nanite/console.rb

@@ -0,0 +1,39 @@
+module Nanite
+  module ConsoleHelper
+    def self.included(base)
+      @@base = base
+    end
+    
+    def start_console
+      puts "Starting #{@@base.name.split(":").last.downcase} console (#{self.identity}) (Nanite #{Nanite::VERSION})"
+      Thread.new do
+        Console.start(self)
+      end
+    end
+  end
+  
+  module Console
+    class << self; attr_accessor :instance; end
+
+    def self.start(binding)
+      require 'irb'
+      old_args = ARGV.dup
+      ARGV.replace ["--simple-prompt"]
+
+      IRB.setup(nil)
+      self.instance = IRB::Irb.new(IRB::WorkSpace.new(binding))
+
+      @CONF = IRB.instance_variable_get(:@CONF)
+      @CONF[:IRB_RC].call self.instance.context if @CONF[:IRB_RC]
+      @CONF[:MAIN_CONTEXT] = self.instance.context
+
+      catch(:IRB_EXIT) { self.instance.eval_input }
+    ensure
+      ARGV.replace old_args
+      # Clean up tty settings in some evil, evil cases
+      begin; catch(:IRB_EXIT) { irb_exit }; rescue Exception; end
+      # Make nanite exit when irb does
+      EM.stop if EM.reactor_running?
+    end
+  end
+end

data/lib/nanite/daemonize.rb

@@ -0,0 +1,13 @@
+module Nanite
+  module DaemonizeHelper
+    def daemonize(identity, options = {})
+      exit if fork
+      Process.setsid
+      exit if fork
+      STDIN.reopen "/dev/null"
+      STDOUT.reopen "#{options[:log_path]}/nanite.#{identity}.out", "a"
+      STDERR.reopen "#{options[:log_path]}/nanite.#{identity}.err", "a"
+      File.umask 0000
+    end
+  end
+end

data/lib/nanite/identity.rb

@@ -0,0 +1,16 @@
+module Nanite
+  module Identity
+    def self.generate
+      values = [
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x1000000),
+        rand(0x1000000),
+      ]
+      "%04x%04x%04x%04x%04x%06x%06x" % values
+    end
+  end
+end

data/lib/nanite/job.rb

@@ -0,0 +1,104 @@
+module Nanite
+  class JobWarden
+    attr_reader :serializer, :jobs
+
+    def initialize(serializer)
+      @serializer = serializer
+      @jobs = {}
+    end
+
+    def new_job(request, targets, inthandler = nil, blk = nil)
+      job = Job.new(request, targets, inthandler, blk)
+      jobs[job.token] = job
+      job
+    end
+
+    def process(msg)
+      if job = jobs[msg.token]
+        job.process(msg)
+
+        if job.intermediate_handler && (job.pending_keys.size > 0)
+
+          unless job.pending_keys.size == 1
+            raise "IntermediateMessages are currently dispatched as they arrive, shouldn't have more than one key in pending_keys: #{job.pending_keys.inspect}"
+          end
+
+          key = job.pending_keys.first
+          handler = job.intermediate_handler_for_key(key)
+          if handler
+            case handler.arity
+            when 2
+              handler.call(job.intermediate_state[msg.from][key].last, job)
+            when 3
+              handler.call(key, msg.from, job.intermediate_state[msg.from][key].last)
+            when 4
+              handler.call(key, msg.from, job.intermediate_state[msg.from][key].last, job)
+            end
+          end
+
+          job.reset_pending_intermediate_state_keys
+        end
+
+        if job.completed?
+          jobs.delete(job.token)
+          if job.completed
+            case job.completed.arity
+            when 1
+              job.completed.call(job.results)
+            when 2
+              job.completed.call(job.results, job)
+            end
+          end
+        end
+      end
+    end
+  end # JobWarden
+
+  class Job
+    attr_reader :results, :request, :token, :completed, :intermediate_state, :pending_keys, :intermediate_handler
+    attr_accessor :targets # This can be updated when a request gets picked up from the offline queue
+
+    def initialize(request, targets, inthandler = nil, blk = nil)
+      @request = request
+      @targets = targets
+      @token = @request.token
+      @results = {}
+      @intermediate_handler = inthandler
+      @pending_keys = []
+      @completed = blk
+      @intermediate_state = {}
+    end
+
+    def process(msg)
+      case msg
+      when Result
+        results[msg.from] = msg.results
+        targets.delete(msg.from)
+      when IntermediateMessage
+        intermediate_state[msg.from] ||= {}
+        intermediate_state[msg.from][msg.messagekey] ||= []
+        intermediate_state[msg.from][msg.messagekey] << msg.message
+        @pending_keys << msg.messagekey
+      end
+    end
+
+    def intermediate_handler_for_key(key)
+      return nil unless @intermediate_handler
+      case @intermediate_handler
+      when Proc
+        @intermediate_handler
+      when Hash
+        @intermediate_handler[key] || @intermediate_handler['*']
+      end
+    end
+
+    def reset_pending_intermediate_state_keys
+      @pending_keys = []
+    end
+
+    def completed?
+      targets.empty?
+    end
+  end # Job
+
+end # Nanite