ezmobius-nanite 0.4.0 → 0.4.1.1

data/lib/nanite/config.rb

@@ -0,0 +1,102 @@
+module Nanite
+
+  COMMON_DEFAULT_OPTIONS = {:pass => 'testing', :vhost => '/nanite', :secure => false, :host => '0.0.0.0',
+    :log_level => :info, :format => :marshal, :daemonize => false, :console => false, :root => Dir.pwd}
+
+  module CommonConfig
+    def setup_mapper_options(opts, options)
+      setup_common_options(opts, options, 'mapper')
+
+      opts.on("-a", "--agent-timeout TIMEOUT", "How long to wait before an agent is considered to be offline and thus removed from the list of available agents.") do |timeout|
+        options[:agent_timeout] = timeout
+      end
+
+      opts.on("-r", "--offline-redelivery-frequency", "The frequency in seconds that messages stored in the offline queue will be retrieved for attempted redelivery to the nanites. Default is 10 seconds.") do |frequency|
+        options[:offline_redelivery_frequency] = frequency
+      end
+
+      opts.on("--persistent", "Instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the broker is restarted. Can be overriden on a per-message basis using the request and push methods.") do
+        options[:persistent] = true
+      end
+
+      opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do
+        options[:offline_failsafe] = true
+      end
+      
+      opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis|
+        redishost, redisport = redis.split(':')
+        redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?)
+        redisport = '6379' if (redishost.nil? || redishost.empty?)
+        redis = "#{redishost}:#{redisport}"
+        options[:redis] = redis
+      end
+      
+    end
+    
+    def setup_common_options(opts, options, type)
+      opts.version = Nanite::VERSION
+
+      opts.on("-i", "--irb-console", "Start #{type} in irb console mode.") do |console|
+        options[:console] = 'irb'
+      end
+
+      opts.on("-u", "--user USER", "Specify the rabbitmq username.") do |user|
+        options[:user] = user
+      end
+
+      opts.on("-h", "--host HOST", "Specify the rabbitmq hostname.") do |host|
+        options[:host] = host
+      end
+
+      opts.on("-P", "--port PORT", "Specify the rabbitmq PORT, default 5672.") do |port|
+        options[:port] = port
+      end
+
+      opts.on("-p", "--pass PASSWORD", "Specify the rabbitmq password") do |pass|
+        options[:pass] = pass
+      end
+
+      opts.on("-t", "--token IDENITY", "Specify the #{type} identity.") do |ident|
+        options[:identity] = ident
+      end
+
+      opts.on("-v", "--vhost VHOST", "Specify the rabbitmq vhost") do |vhost|
+        options[:vhost] = vhost
+      end
+
+      opts.on("-s", "--secure", "Use Security features of rabbitmq to restrict nanites to themselves") do
+        options[:secure] = true
+      end
+
+      opts.on("-f", "--format FORMAT", "The serialization type to use for transfering data. Can be marshal, json or yaml. Default is marshal") do |fmt|
+        options[:format] = fmt.to_sym
+      end
+
+      opts.on("-d", "--daemonize", "Run #{type} as a daemon") do |d|
+        options[:daemonize] = true
+      end
+      
+      opts.on("--pid-dir PATH", "Specify the pid path, only used with daemonize") do |dir|
+        options[:pid_dir] = dir
+      end
+
+      opts.on("-l", "--log-level LEVEL", "Specify the log level (fatal, error, warn, info, debug). Default is info") do |level|
+        options[:log_level] = level
+      end
+      
+      opts.on("--log-dir PATH", "Specify the log path") do |dir|
+        options[:log_dir] = dir
+      end
+
+      opts.on("--tag TAG", "Specify a tag.  Can issue multiple times.") do |tag|
+        options[:tag] ||= []
+        options[:tag] << tag
+      end
+
+      opts.on("--version", "Show the nanite version number") do |res|
+        puts "Nanite Version #{opts.version}"
+        exit
+      end
+    end
+  end
+end

data/lib/nanite/console.rb

@@ -0,0 +1,39 @@
+module Nanite
+  module ConsoleHelper
+    def self.included(base)
+      @@base = base
+    end
+    
+    def start_console
+      puts "Starting #{@@base.name.split(":").last.downcase} console (#{self.identity}) (Nanite #{Nanite::VERSION})"
+      Thread.new do
+        Console.start(self)
+      end
+    end
+  end
+  
+  module Console
+    class << self; attr_accessor :instance; end
+
+    def self.start(binding)
+      require 'irb'
+      old_args = ARGV.dup
+      ARGV.replace ["--simple-prompt"]
+
+      IRB.setup(nil)
+      self.instance = IRB::Irb.new(IRB::WorkSpace.new(binding))
+
+      @CONF = IRB.instance_variable_get(:@CONF)
+      @CONF[:IRB_RC].call self.instance.context if @CONF[:IRB_RC]
+      @CONF[:MAIN_CONTEXT] = self.instance.context
+
+      catch(:IRB_EXIT) { self.instance.eval_input }
+    ensure
+      ARGV.replace old_args
+      # Clean up tty settings in some evil, evil cases
+      begin; catch(:IRB_EXIT) { irb_exit }; rescue Exception; end
+      # Make nanite exit when irb does
+      EM.stop if EM.reactor_running?
+    end
+  end
+end

data/lib/nanite/daemonize.rb

@@ -0,0 +1,13 @@
+module Nanite
+  module DaemonizeHelper
+    def daemonize
+      exit if fork
+      Process.setsid
+      exit if fork
+      File.umask 0000
+      STDIN.reopen "/dev/null"
+      STDOUT.reopen "/dev/null", "a"
+      STDERR.reopen STDOUT
+    end
+  end
+end

data/lib/nanite/dispatcher.rb

@@ -0,0 +1,90 @@
+module Nanite
+  class Dispatcher
+    attr_reader :registry, :serializer, :identity, :amq, :options
+    attr_accessor :evmclass
+
+    def initialize(amq, registry, serializer, identity, options)
+      @amq = amq
+      @registry = registry
+      @serializer = serializer
+      @identity = identity
+      @options = options
+      @evmclass = EM
+    end
+
+    def dispatch(deliverable)
+      prefix, meth = deliverable.type.split('/')[1..-1]
+      meth ||= :index
+      actor = registry.actor_for(prefix)
+
+      operation = lambda do
+        begin
+          intermediate_results_proc = lambda { |*args| self.handle_intermediate_results(actor, meth, deliverable, *args) }
+          args = [ deliverable.payload ]
+          args.push(deliverable) if actor.method(meth).arity == 2
+          actor.send(meth, *args, &intermediate_results_proc)
+        rescue Exception => e
+          handle_exception(actor, meth, deliverable, e)
+        end
+      end
+      
+      callback = lambda do |r|
+        if deliverable.kind_of?(Request)
+          r = Result.new(deliverable.token, deliverable.reply_to, r, identity)
+          amq.queue(deliverable.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
+        end
+        r # For unit tests
+      end
+
+      if @options[:single_threaded]
+        @evmclass.next_tick { callback.call(operation.call) }
+      else
+        @evmclass.defer(operation, callback)
+      end
+    end
+
+    protected
+
+    def handle_intermediate_results(actor, meth, deliverable, *args)
+      messagekey = case args.size
+      when 1
+        'defaultkey'
+      when 2
+        args.first.to_s
+      else
+        raise ArgumentError, "handle_intermediate_results passed unexpected number of arguments (#{args.size})"
+      end
+      message = args.last
+      @evmclass.defer(lambda {
+        [deliverable.reply_to, IntermediateMessage.new(deliverable.token, deliverable.reply_to, identity, messagekey, message)]
+      }, lambda { |r|
+        amq.queue(r.first, :no_declare => options[:secure]).publish(serializer.dump(r.last))
+      })
+    end
+
+    private
+
+    def describe_error(e)
+      "#{e.class.name}: #{e.message}\n #{e.backtrace.join("\n  ")}"
+    end
+
+    def handle_exception(actor, meth, deliverable, e)
+      error = describe_error(e)
+      Nanite::Log.error(error)
+      begin
+        if actor.class.exception_callback
+          case actor.class.exception_callback
+          when Symbol, String
+            actor.send(actor.class.exception_callback, meth.to_sym, deliverable, e)
+          when Proc
+            actor.instance_exec(meth.to_sym, deliverable, e, &actor.class.exception_callback)
+          end
+        end
+      rescue Exception => e1
+        error = describe_error(e1)
+        Nanite::Log.error(error)
+      end
+      error
+    end
+  end
+end

data/lib/nanite/identity.rb

@@ -0,0 +1,16 @@
+module Nanite
+  module Identity
+    def self.generate
+      values = [
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x0010000),
+        rand(0x1000000),
+        rand(0x1000000),
+      ]
+      "%04x%04x%04x%04x%04x%06x%06x" % values
+    end
+  end
+end

data/lib/nanite/job.rb

@@ -0,0 +1,104 @@
+module Nanite
+  class JobWarden
+    attr_reader :serializer, :jobs
+
+    def initialize(serializer)
+      @serializer = serializer
+      @jobs = {}
+    end
+
+    def new_job(request, targets, inthandler = nil, blk = nil)
+      job = Job.new(request, targets, inthandler, blk)
+      jobs[job.token] = job
+      job
+    end
+
+    def process(msg)
+      Nanite::Log.debug("processing message: #{msg.inspect}")
+
+      if job = jobs[msg.token]
+        job.process(msg)
+
+        if job.intermediate_handler && (job.pending_keys.size > 0)
+
+          unless job.pending_keys.size == 1
+            raise "IntermediateMessages are currently dispatched as they arrive, shouldn't have more than one key in pending_keys: #{job.pending_keys.inspect}"
+          end
+
+          key = job.pending_keys.first
+          handler = job.intermediate_handler_for_key(key)
+          if handler
+            case handler.arity
+            when 3
+              handler.call(key, msg.from, job.intermediate_state[msg.from][key].last)
+            when 4
+              handler.call(key, msg.from, job.intermediate_state[msg.from][key].last, job)
+            end
+          end
+
+          job.reset_pending_intermediate_state_keys
+        end
+
+        if job.completed?
+          jobs.delete(job.token)
+          if job.completed
+            case job.completed.arity
+            when 1
+              job.completed.call(job.results)
+            when 2
+              job.completed.call(job.results, job)
+            end
+          end
+        end
+      end
+    end
+  end # JobWarden
+
+  class Job
+    attr_reader :results, :request, :token, :completed, :intermediate_state, :pending_keys, :intermediate_handler
+    attr_accessor :targets # This can be updated when a request gets picked up from the offline queue
+
+    def initialize(request, targets, inthandler = nil, blk = nil)
+      @request = request
+      @targets = targets
+      @token = @request.token
+      @results = {}
+      @intermediate_handler = inthandler
+      @pending_keys = []
+      @completed = blk
+      @intermediate_state = {}
+    end
+
+    def process(msg)
+      case msg
+      when Result
+        results[msg.from] = msg.results
+        targets.delete(msg.from)
+      when IntermediateMessage
+        intermediate_state[msg.from] ||= {}
+        intermediate_state[msg.from][msg.messagekey] ||= []
+        intermediate_state[msg.from][msg.messagekey] << msg.message
+        @pending_keys << msg.messagekey
+      end
+    end
+
+    def intermediate_handler_for_key(key)
+      return nil unless @intermediate_handler
+      case @intermediate_handler
+      when Proc
+        @intermediate_handler
+      when Hash
+        @intermediate_handler[key] || @intermediate_handler['*']
+      end
+    end
+
+    def reset_pending_intermediate_state_keys
+      @pending_keys = []
+    end
+
+    def completed?
+      targets.empty?
+    end
+  end # Job
+
+end # Nanite

data/lib/nanite/local_state.rb

@@ -0,0 +1,34 @@
+module Nanite
+  class LocalState < ::Hash
+    def initialize(hsh={})
+      hsh.each do |k,v|
+        self[k] = v
+      end
+    end
+
+    def all_services
+      all(:services)
+    end
+
+    def all_tags
+      all(:tags)
+    end
+
+    def nanites_for(service, *tags)
+      tags = tags.dup.flatten
+      nanites = select { |name, state| state[:services].include?(service) }
+      unless tags.empty?
+        nanites.select { |a| !(a[1][:tags] & tags).empty? }
+      else
+        nanites
+      end
+    end
+
+    private
+
+    def all(key)
+      map { |n,s| s[key] }.flatten.uniq.compact
+    end
+
+  end # LocalState
+end # Nanite

data/lib/nanite/log.rb

@@ -0,0 +1,64 @@
+require 'nanite/config'
+require 'nanite/log/formatter'
+require 'logger'
+
+module Nanite
+  class Log
+  
+    @logger = nil
+    
+    class << self
+      attr_accessor :logger, :level, :file #:nodoc
+      
+      # Use Nanite::Logger.init when you want to set up the logger manually.
+      # If this method is called with no arguments, it will log to STDOUT at the :info level.
+      # It also configures the Logger instance it creates to use the custom Nanite::Log::Formatter class.
+      def init(identity, path = false)
+        if path
+          @file = File.join(path, "nanite.#{identity}.log")
+        else
+          @file = STDOUT
+        end
+        @logger = Logger.new(file)
+        @logger.formatter = Nanite::Log::Formatter.new
+        level = @log_level = :info
+      end
+      
+      # Sets the level for the Logger by symbol or by command line argument.
+      # Throws an ArgumentError if you feed it a bogus log level (that is not
+      # one of :debug, :info, :warn, :error, :fatal or the corresponding strings)
+      def level=(loglevel)
+        init() unless @logger
+        loglevel = loglevel.intern if loglevel.is_a?(String)
+        @logger.info("Setting log level to #{loglevel.to_s.upcase}")
+        case loglevel
+        when :debug
+          @logger.level = Logger::DEBUG
+        when :info
+          @logger.level = Logger::INFO
+        when :warn
+          @logger.level = Logger::WARN
+        when :error
+          @logger.level = Logger::ERROR
+        when :fatal
+          @logger.level = Logger::FATAL
+        else
+          raise ArgumentError, "Log level must be one of :debug, :info, :warn, :error, or :fatal"
+        end
+      end
+      
+      # Passes any other method calls on directly to the underlying Logger object created with init. If
+      # this method gets hit before a call to Nanite::Logger.init has been made, it will call 
+      # Nanite::Logger.init() with no arguments.
+      def method_missing(method_symbol, *args)
+        init(identity) unless @logger
+        if args.length > 0
+          @logger.send(method_symbol, *args)
+        else
+          @logger.send(method_symbol)
+        end
+      end
+      
+    end # class << self
+  end
+end