ezmobius-nanite 0.4.0 → 0.4.1.1

data/lib/nanite/log/formatter.rb

@@ -0,0 +1,39 @@
+require 'logger'
+require 'time'
+
+module Nanite
+  class Log
+    class Formatter < Logger::Formatter
+      @@show_time = true
+      
+      def self.show_time=(show=false)
+        @@show_time = show
+      end
+      
+      # Prints a log message as '[time] severity: message' if Nanite::Log::Formatter.show_time == true.
+      # Otherwise, doesn't print the time.
+      def call(severity, time, progname, msg)
+        if @@show_time
+          sprintf("[%s] %s: %s\n", time.rfc2822(), severity, msg2str(msg))
+        else
+          sprintf("%s: %s\n", severity, msg2str(msg))
+        end
+      end
+      
+      # Converts some argument to a Logger.severity() call to a string.  Regular strings pass through like
+      # normal, Exceptions get formatted as "message (class)\nbacktrace", and other random stuff gets 
+      # put through "object.inspect"
+      def msg2str(msg)
+        case msg
+        when ::String
+          msg
+        when ::Exception
+          "#{ msg.message } (#{ msg.class })\n" <<
+            (msg.backtrace || []).join("\n")
+        else
+          msg.inspect
+        end
+      end
+    end
+  end
+end

data/lib/nanite/mapper.rb

@@ -0,0 +1,277 @@
+module Nanite
+  # Mappers are control nodes in nanite clusters. Nanite clusters
+  # can follow peer-to-peer model of communication as well as client-server,
+  # and mappers are nodes that know who to send work requests to agents.
+  #
+  # Mappers can reside inside a front end web application written in Merb/Rails
+  # and distribute heavy lifting to actors that register with the mapper as soon
+  # as they go online.
+  #
+  # Each mapper tracks nanites registered with it. It periodically checks
+  # when the last time a certain nanite sent a heartbeat notification,
+  # and removes those that have timed out from the list of available workers.
+  # As soon as a worker goes back online again it re-registers itself
+  # and the mapper adds it to the list and makes it available to
+  # be called again.
+  #
+  # This makes Nanite clusters self-healing and immune to individual node
+  # failures.
+  class Mapper
+    include AMQPHelper
+    include ConsoleHelper
+    include DaemonizeHelper
+
+    attr_reader :cluster, :identity, :job_warden, :options, :serializer, :amq
+
+    DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'mapper', :identity => Identity.generate, :agent_timeout => 15,
+      :offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false}) unless defined?(DEFAULT_OPTIONS)
+
+    # Initializes a new mapper and establishes
+    # AMQP connection. This must be used inside EM.run block or if EventMachine reactor
+    # is already started, for instance, by a Thin server that your Merb/Rails
+    # application runs on.
+    #
+    # Mapper options:
+    #
+    # identity    : identity of this mapper, may be any string
+    #
+    # format      : format to use for packets serialization. Can be :marshal, :json or :yaml or :secure.
+    #               Defaults to Ruby's Marshall format. For interoperability with
+    #               AMQP clients implemented in other languages, use JSON.
+    #
+    #               Note that Nanite uses JSON gem,
+    #               and ActiveSupport's JSON encoder may cause clashes
+    #               if ActiveSupport is loaded after JSON gem.
+    #
+    #               Also using the secure format requires prior initialization of the serializer, see
+    #               SecureSerializer.init
+    #
+    # log_level   : the verbosity of logging, can be debug, info, warn, error or fatal.
+    #
+    # agent_timeout   : how long to wait before an agent is considered to be offline
+    #                   and thus removed from the list of available agents.
+    #
+    # log_dir    : log file path, defaults to the current working directory.
+    #
+    # console     : true tells mapper to start interactive console
+    #
+    # daemonize   : true tells mapper to daemonize
+    #
+    # pid_dir     : path to the directory where the agent stores its pid file (only if daemonized)
+    #               defaults to the root or the current working directory.
+    #
+    # offline_redelivery_frequency : The frequency in seconds that messages stored in the offline queue will be retrieved
+    #                                for attempted redelivery to the nanites. Default is 10 seconds.
+    #
+    # persistent  : true instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the
+    #               broker is restarted. Default is false. Can be overriden on a per-message basis using the request and push methods.
+    # 
+    # secure      : use Security features of rabbitmq to restrict nanites to themselves
+    #
+    # Connection options:
+    #
+    # vhost    : AMQP broker vhost that should be used
+    #
+    # user     : AMQP broker user
+    #
+    # pass     : AMQP broker password
+    #
+    # host     : host AMQP broker (or node of interest) runs on,
+    #            defaults to 0.0.0.0
+    #
+    # port     : port AMQP broker (or node of interest) runs on,
+    #            this defaults to 5672, port used by some widely
+    #            used AMQP brokers (RabbitMQ and ZeroMQ)
+    #
+    # @api :public:
+    def self.start(options = {})
+      mapper = new(options)
+      mapper.run
+      mapper
+    end
+
+    def initialize(options)
+      @options = DEFAULT_OPTIONS.clone.merge(options)
+      root = options[:root] || @options[:root]
+      custom_config = if root
+        file = File.expand_path(File.join(root, 'config.yml'))
+        File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {}
+      else
+        {}
+      end
+      options.delete(:identity) unless options[:identity]
+      @options.update(custom_config.merge(options))
+      @identity = "mapper-#{@options[:identity]}"
+      @options[:file_root] ||= File.join(@options[:root], 'files')
+      @options.freeze
+    end
+    
+    def run
+      log_path = false
+      if @options[:daemonize]
+        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      Nanite::Log.init(@identity, log_path)
+      Nanite::Log.level = @options[:log_level] if @options[:log_level]
+      @serializer = Serializer.new(@options[:format])
+      pid_file = PidFile.new(@identity, @options)
+      pid_file.check
+      if @options[:daemonize]
+        daemonize
+        pid_file.write
+        at_exit { pid_file.remove }
+      end
+      @amq = start_amqp(@options)
+      @job_warden = JobWarden.new(@serializer)
+      @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis])
+      Nanite::Log.info('starting mapper')
+      setup_queues
+      start_console if @options[:console] && !@options[:daemonize]
+    end
+
+    # Make a nanite request which expects a response.
+    #
+    # ==== Parameters
+    # type<String>:: The dispatch route for the request
+    # payload<Object>:: Payload to send.  This will get marshalled en route
+    #
+    # ==== Options
+    # :selector<Symbol>:: Method for selecting an actor.  Default is :least_loaded.
+    #   :least_loaded:: Pick the nanite which has the lowest load.
+    #   :all:: Send the request to all nanites which respond to the service.
+    #   :random:: Randomly pick a nanite.
+    #   :rr: Select a nanite according to round robin ordering.
+    # :target<String>:: Select a specific nanite via identity, rather than using
+    #   a selector.
+    # :offline_failsafe<Boolean>:: Store messages in an offline queue when all
+    #   the nanites are offline. Messages will be redelivered when nanites come online.
+    #   Default is false unless the mapper was started with the --offline-failsafe flag.
+    # :persistent<Boolean>:: Instructs the AMQP broker to save the message to persistent
+    #   storage so that it isnt lost when the broker is restarted.
+    #   Default is false unless the mapper was started with the --persistent flag.
+    # :intermediate_handler:: Takes a lambda to call when an IntermediateMessage
+    #   event arrives from a nanite.  If passed a Hash, hash keys should correspond to
+    #   the IntermediateMessage keys provided by the nanite, and each should have a value
+    #   that is a lambda/proc taking the parameters specified here.  Can supply a key '*'
+    #   as a catch-all for unmatched keys.
+    #
+    # ==== Block Parameters for intermediate_handler
+    # key<String>:: array of unique keys for which intermediate state has been received
+    #   since the last call to this block.
+    # nanite<String>:: nanite which sent the message.
+    # state:: most recently delivered intermediate state for the key provided.
+    # job:: (optional) -- if provided, this parameter gets the whole job object, if there's
+    #   a reason to do more complex work with the job.
+    #
+    # ==== Block Parameters
+    # :results<Object>:: The returned value from the nanite actor.
+    #
+    # @api :public:
+    def request(type, payload = '', opts = {}, &blk)
+      request = build_deliverable(Request, type, payload, opts)
+      send_request(request, opts, &blk)
+    end
+
+    # Send request with pre-built request instance
+    def send_request(request, opts = {}, &blk)
+      request.reply_to = identity
+      intm_handler = opts.delete(:intermediate_handler)
+      targets = cluster.targets_for(request)
+      if !targets.empty?
+        job = job_warden.new_job(request, targets, intm_handler, blk)
+        cluster.route(request, job.targets)
+        job
+      elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
+        job_warden.new_job(request, [], intm_handler, blk)
+        cluster.publish(request, 'mapper-offline')
+        :offline
+      else
+        false
+      end
+    end
+      
+    # Make a nanite request which does not expect a response.
+    #
+    # ==== Parameters
+    # type<String>:: The dispatch route for the request
+    # payload<Object>:: Payload to send.  This will get marshalled en route
+    #
+    # ==== Options
+    # :selector<Symbol>:: Method for selecting an actor.  Default is :least_loaded.
+    #   :least_loaded:: Pick the nanite which has the lowest load.
+    #   :all:: Send the request to all nanites which respond to the service.
+    #   :random:: Randomly pick a nanite.
+    #   :rr: Select a nanite according to round robin ordering.
+    # :offline_failsafe<Boolean>:: Store messages in an offline queue when all
+    #   the nanites are offline. Messages will be redelivered when nanites come online.
+    #   Default is false unless the mapper was started with the --offline-failsafe flag.
+    # :persistent<Boolean>:: Instructs the AMQP broker to save the message to persistent
+    #   storage so that it isnt lost when the broker is restarted.
+    #   Default is false unless the mapper was started with the --persistent flag.
+    #
+    # @api :public:
+    def push(type, payload = '', opts = {})
+      push = build_deliverable(Push, type, payload, opts)
+      targets = cluster.targets_for(push)
+      if !targets.empty?
+        cluster.route(push, targets)
+        true
+      elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
+        cluster.publish(push, 'mapper-offline')
+        :offline
+      else
+        false
+      end
+    end
+
+    private
+
+    def build_deliverable(deliverable_type, type, payload, opts)
+      deliverable = deliverable_type.new(type, payload, opts)
+      deliverable.from = identity
+      deliverable.token = Identity.generate
+      deliverable.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
+      deliverable
+    end
+
+    def setup_queues
+      setup_offline_queue
+      setup_message_queue
+    end
+
+    def setup_offline_queue
+      offline_queue = amq.queue('mapper-offline', :durable => true)
+      offline_queue.subscribe(:ack => true) do |info, deliverable|
+        deliverable = serializer.load(deliverable)
+        targets = cluster.targets_for(deliverable)
+        unless targets.empty?
+          info.ack
+          if deliverable.kind_of?(Request)
+            if job = job_warden.jobs[deliverable.token]
+              job.targets = targets
+            else
+              deliverable.reply_to = identity
+              job_warden.new_job(deliverable, targets)
+            end
+          end
+          cluster.route(deliverable, targets)
+        end
+      end
+
+      EM.add_periodic_timer(options[:offline_redelivery_frequency]) { offline_queue.recover }
+    end
+
+    def setup_message_queue
+      amq.queue(identity, :exclusive => true).bind(amq.fanout(identity)).subscribe do |msg|
+        begin
+          msg = serializer.load(msg)
+          Nanite::Log.debug("got result from #{msg.from}: #{msg.results.inspect}")
+          job_warden.process(msg)
+        rescue Exception => e
+          Nanite::Log.error("Error handling result: #{e.message}")
+        end
+      end
+    end
+  end
+end
+

data/lib/nanite/mapper_proxy.rb

@@ -0,0 +1,56 @@
+module Nanite
+
+  # This class allows sending requests to nanite agents without having
+  # to run a local mapper.
+  # It is used by Actor.request which can be used by actors than need
+  # to send requests to remote agents.
+  # All requests go through the mapper for security purposes.
+  class MapperProxy
+        
+    $:.push File.dirname(__FILE__)
+    require 'amqp'
+  
+    include AMQPHelper
+    
+    attr_accessor :pending_requests, :identity, :options, :amqp, :serializer
+
+    # Accessor for actor
+    def self.instance
+      @@instance
+    end
+
+    def initialize(id, opts)
+      @options = opts || {}
+      @identity = id
+      @pending_requests = {}
+      @amqp = start_amqp(options)
+      @serializer = Serializer.new(options[:format])
+      @@instance = self
+    end
+
+    # Send request to given agent through the mapper
+    def request(type, payload = '', opts = {}, &blk)
+      raise "Mapper proxy not initialized" unless identity && options
+      request = Request.new(type, payload, opts)
+      request.from = identity
+      request.token = Identity.generate
+      request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
+      pending_requests[request.token] = 
+        { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
+      amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
+    end    
+    
+    # Handle intermediary result
+    def handle_intermediate_result(res)
+      handlers = pending_requests[res.token]
+      handlers[:intermediate_handler].call(res) if handlers && handlers[:intermediate_handler]
+    end
+    
+    # Handle final result
+    def handle_result(res)
+      handlers = pending_requests.delete(res.token)
+      handlers[:result_handler].call(res) if handlers && handlers[:result_handler]
+    end
+
+  end
+end

data/lib/nanite/packets.rb

@@ -0,0 +1,231 @@
+module Nanite
+  # Base class for all Nanite packets,
+  # knows how to dump itself to JSON
+  class Packet
+    def initialize
+      raise NotImplementedError.new("#{self.class.name} is an abstract class.")
+    end
+    def to_json(*a)
+      {
+        'json_class'   => self.class.name,
+        'data'         => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
+      }.to_json(*a)
+    end
+  end
+
+  # packet that means start of a file transfer
+  # operation
+  class FileStart < Packet
+    attr_accessor :filename, :token, :dest
+    def initialize(filename, dest, token)
+      @filename = filename
+      @dest = dest
+      @token = token
+    end
+
+    def self.json_create(o)
+      i = o['data']
+      new(i['filename'], i['dest'], i['token'])
+    end
+  end
+
+  # packet that means end of a file transfer
+  # operation
+  class FileEnd < Packet
+    attr_accessor :token, :meta
+    def initialize(token, meta)
+      @token = token
+      @meta  = meta
+    end
+
+    def self.json_create(o)
+      i = o['data']
+      new(i['token'], i['meta'])
+    end
+  end
+
+  # packet that carries data chunks during a file transfer
+  class FileChunk < Packet
+    attr_accessor :chunk, :token
+    def initialize(token, chunk=nil)
+      @chunk = chunk
+      @token = token
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['token'], i['chunk'])
+    end
+  end
+
+  # packet that means a work request from mapper
+  # to actor node
+  #
+  # type     is a service name
+  # payload  is arbitrary data that is transferred from mapper to actor
+  #
+  # Options:
+  # from     is sender identity
+  # token    is a generated request id that mapper uses to identify replies
+  # reply_to is identity of the node actor replies to, usually a mapper itself
+  # selector is the selector used to route the request
+  # target   is the target nanite for the request
+  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
+  class Request < Packet
+    attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
+    DEFAULT_OPTIONS = {:selector => :least_loaded}
+    def initialize(type, payload, opts={})
+      opts = DEFAULT_OPTIONS.merge(opts)
+      @type             = type
+      @payload          = payload
+      @from             = opts[:from]
+      @token            = opts[:token]
+      @reply_to         = opts[:reply_to]
+      @selector         = opts[:selector]
+      @target           = opts[:target]
+      @persistent       = opts[:persistent]
+      @tags             = opts[:tags] || []
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :reply_to => i['reply_to'], :selector => i['selector'],
+      :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
+    end
+  end
+
+  # packet that means a work push from mapper
+  # to actor node
+  #
+  # type     is a service name
+  # payload  is arbitrary data that is transferred from mapper to actor
+  #
+  # Options:
+  # from     is sender identity
+  # token    is a generated request id that mapper uses to identify replies
+  # selector is the selector used to route the request
+  # target   is the target nanite for the request
+  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
+  class Push < Packet
+    attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
+    DEFAULT_OPTIONS = {:selector => :least_loaded}
+    def initialize(type, payload, opts={})
+      opts = DEFAULT_OPTIONS.merge(opts)
+      @type             = type
+      @payload          = payload
+      @from             = opts[:from]
+      @token            = opts[:token]
+      @selector         = opts[:selector]
+      @target           = opts[:target]
+      @persistent       = opts[:persistent]
+      @tags             = opts[:tags] || []
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :selector => i['selector'],
+      :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
+    end
+  end
+
+  # packet that means a work result notification sent from actor to mapper
+  #
+  # from     is sender identity
+  # results  is arbitrary data that is transferred from actor, a result of actor's work
+  # token    is a generated request id that mapper uses to identify replies
+  # to       is identity of the node result should be delivered to
+  class Result < Packet
+    attr_accessor :token, :results, :to, :from
+    def initialize(token, to, results, from)
+      @token = token
+      @to = to
+      @from = from
+      @results = results
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['token'], i['to'], i['results'], i['from'])
+    end
+  end
+
+  # packet that means an intermediate status notification sent from actor to mapper. is appended to a list of messages matching messagekey.
+  #
+  # from     is sender identity
+  # messagekey is a string that can become part of a redis key, which identifies the name under which the message is stored
+  # message  is arbitrary data that is transferred from actor, an intermediate result of actor's work
+  # token    is a generated request id that mapper uses to identify replies
+  # to       is identity of the node result should be delivered to
+  class IntermediateMessage < Packet
+    attr_accessor :token, :messagekey, :message, :to, :from
+    def initialize(token, to, from, messagekey, message)
+      @token = token
+      @to = to
+      @from = from
+      @messagekey = messagekey
+      @message = message
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['token'], i['to'], i['from'], i['messagekey'], i['message'])
+    end
+  end
+
+  # packet that means an availability notification sent from actor to mapper
+  #
+  # from     is sender identity
+  # services is a list of services provided by the node
+  # status   is a load of the node by default, but may be any criteria
+  #          agent may use to report it's availability, load, etc
+  class Register < Packet
+    attr_accessor :identity, :services, :status, :tags
+    def initialize(identity, services, status, tags)
+      @status = status
+      @tags = tags
+      @identity = identity
+      @services = services
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['identity'], i['services'], i['status'], i['tags'])
+    end
+  end
+
+  # packet that means deregister an agent from the mappers
+  #
+  # from     is sender identity
+  class UnRegister < Packet
+    attr_accessor :identity
+    def initialize(identity)
+      @identity = identity
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['identity'])
+    end
+  end
+
+  # heartbeat packet
+  #
+  # identity is sender's identity
+  # status   is sender's status (see Register packet documentation)
+  class Ping < Packet
+    attr_accessor :identity, :status
+    def initialize(identity, status)
+      @status = status
+      @identity = identity
+    end
+    def self.json_create(o)
+      i = o['data']
+      new(i['identity'], i['status'])
+    end
+  end
+
+  # packet that is sent by workers to the mapper
+  # when worker initially comes online to advertise
+  # it's services
+  class Advertise < Packet
+    def initialize
+    end
+    def self.json_create(o)
+      new
+    end
+  end
+end
+