rightscale-nanite 0.4.1 → 0.4.1.1

data/lib/nanite.rb

@@ -0,0 +1,71 @@
+require 'rubygems'
+require 'amqp'
+require 'mq'
+require 'json'
+require 'logger'
+require 'yaml'
+require 'openssl'
+
+$:.unshift File.dirname(__FILE__)
+require 'nanite/amqp'
+require 'nanite/util'
+require 'nanite/config'
+require 'nanite/packets'
+require 'nanite/identity'
+require 'nanite/console'
+require 'nanite/daemonize'
+require 'nanite/pid_file'
+require 'nanite/job'
+require 'nanite/mapper'
+require 'nanite/actor'
+require 'nanite/actor_registry'
+require 'nanite/streaming'
+require 'nanite/dispatcher'
+require 'nanite/agent'
+require 'nanite/cluster'
+require 'nanite/reaper'
+require 'nanite/log'
+require 'nanite/mapper_proxy'
+require 'nanite/security_provider'
+require 'nanite/security/cached_certificate_store_proxy'
+require 'nanite/security/certificate'
+require 'nanite/security/certificate_cache'
+require 'nanite/security/distinguished_name'
+require 'nanite/security/encrypted_document'
+require 'nanite/security/rsa_key_pair'
+require 'nanite/security/secure_serializer'
+require 'nanite/security/signature'
+require 'nanite/security/static_certificate_store'
+require 'nanite/serializer'
+
+module Nanite
+  VERSION = '0.4.0' unless defined?(Nanite::VERSION)
+
+  class MapperNotRunning < StandardError; end
+
+  class << self
+    attr_reader :mapper, :agent
+
+    def start_agent(options = {})
+      @agent = Nanite::Agent.start(options)
+    end
+
+    def start_mapper(options = {})
+      @mapper = Nanite::Mapper.start(options)
+    end
+
+    def request(*args, &blk)
+      ensure_mapper
+      @mapper.request(*args, &blk)
+    end
+
+    def push(*args)
+      ensure_mapper
+      @mapper.push(*args)
+    end
+
+    def ensure_mapper
+      raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper') unless @mapper
+    end
+  end
+end

data/lib/nanite/actor.rb

@@ -0,0 +1,60 @@
+module Nanite
+  # This mixin provides Nanite actor functionality.
+  #
+  # To use it simply include it your class containing the functionality to be exposed:
+  #
+  #   class Foo
+  #     include Nanite::Actor
+  #     expose :bar
+  #
+  #     def bar(payload)
+  #       # ...
+  #     end
+  #
+  #   end
+  module Actor
+
+    def self.included(base)
+      base.class_eval do 
+        include Nanite::Actor::InstanceMethods
+        extend  Nanite::Actor::ClassMethods
+      end # base.class_eval
+    end # self.included
+    
+    module ClassMethods
+      def default_prefix
+        to_s.to_const_path
+      end
+
+      def expose(*meths)
+        @exposed ||= []
+        meths.each do |meth|
+          @exposed << meth unless @exposed.include?(meth)
+        end
+      end
+
+      def provides_for(prefix)
+        return [] unless @exposed
+        @exposed.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
+      end
+
+      def on_exception(proc = nil, &blk)
+        raise 'No callback provided for on_exception' unless proc || blk
+        @exception_callback = proc || blk
+      end
+
+      def exception_callback
+        @exception_callback
+      end
+      
+    end # ClassMethods     
+    
+    module InstanceMethods
+      # send nanite request to another agent (through the mapper)
+      def request(*args, &blk)
+        MapperProxy.instance.request(*args, &blk)
+      end
+    end # InstanceMethods
+    
+  end # Actor
+end # Nanite

data/lib/nanite/actor_registry.rb

@@ -0,0 +1,24 @@
+module Nanite
+  class ActorRegistry
+    attr_reader :actors
+
+    def initialize
+      @actors = {}
+    end
+
+    def register(actor, prefix)
+      raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
+      Nanite::Log.info("Registering #{actor.inspect} with prefix #{prefix.inspect}")
+      prefix ||= actor.class.default_prefix
+      actors[prefix.to_s] = actor
+    end
+
+    def services
+      actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
+    end
+
+    def actor_for(prefix)
+      actor = actors[prefix]
+    end
+  end # ActorRegistry
+end # Nanite 

data/lib/nanite/admin.rb

@@ -0,0 +1,153 @@
+require 'rack'
+
+module Nanite
+  # This is a Rack app for nanite-admin.  You need to have an async capable
+  # version of Thin installed for this to work.  See bin/nanite-admin for install
+  # instructions.
+  class Admin
+    def initialize(mapper)
+      @mapper = mapper
+    end
+
+    AsyncResponse = [-1, {}, []].freeze
+
+    def call(env)
+      req = Rack::Request.new(env)
+      if cmd = req.params['command']
+        @command = cmd
+        @selection = req.params['type'] if req.params['type']
+        
+        options = {}
+        case @selection
+        when 'least_loaded', 'random', 'all', 'rr'
+          options[:selector] = @selection
+        else
+          options[:target] = @selection
+        end
+
+        @mapper.request(cmd, req.params['payload'], options) do |response, responsejob|
+          env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]]
+        end
+        AsyncResponse
+      else
+        [200, {'Content-Type' => 'text/html'}, layout]
+      end
+    end
+
+    def services
+      buf = "<select name='command'>"
+      @mapper.cluster.nanites.all_services.each do |srv|
+        buf << "<option value='#{srv}' #{@command == srv ? 'selected="true"' : ''}>#{srv}</option>"
+      end
+      buf << "</select>"
+      buf
+    end
+
+    def ul(hash, job)
+      buf = "<ul>"
+      hash.each do |k,v|
+        buf << "<li><div class=\"nanite\">#{k}:</div><div class=\"response\">#{v.inspect}</div>"
+        if job.intermediate_state && job.intermediate_state[k]
+          buf << "<div class=\"intermediatestates\"><span class=\"statenote\">intermediate state:</span> #{job.intermediate_state[k].inspect}</div>"
+        end
+        buf << "</li>"
+      end
+      buf << "</ul>"
+      buf
+    end
+
+    def layout(content=nil)
+      %Q{
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns='http://www.w3.org/1999/xhtml'>
+          <head>
+            <meta content='text/html; charset=utf-8' http-equiv='Content-Type' />
+            <meta content='en' http-equiv='Content-Language' />
+            <meta content='Engineyard' name='author' />
+            <title>Nanite Control Tower</title>
+
+            <!-- Google AJAX Libraries API -->
+            <script src="http://www.google.com/jsapi"></script>
+            <script type="text/javascript">
+              google.load("jquery", "1");
+            </script>
+
+            <script type="text/javascript">
+            $(document).ready(function(){
+
+              // set the focus to the payload field
+              $("#payload").focus();
+
+            });
+            </script>
+
+            <style>
+              body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
+              ul {margin: 0; padding: 0; margin-left: 10px}
+              li {list-style-type: none; margin-bottom: 6px}
+              li .nanite {font-weight: bold; font-size: 12px}
+              li .response {padding: 8px}
+              li .intermediatestates {padding: 8px; font-size: 10px;}
+              li .intermediatestates span.statenote {font-style: italic;}
+              h1, h2, h3 {margin-top: none; padding: none; margin-left: 40px;}
+              h1 {font-size: 22px; margin-top: 40px; margin-bottom: 30px; border-bottom: 1px solid #ddd; padding-bottom: 6px;
+                margin-right: 40px}
+              h2 {font-size: 16px;}
+              h3 {margin-left: 0; font-size: 14px}
+              .section {border: 1px solid #ccc; background-color: #fefefe; padding: 10px; margin: 20px 40px; padding: 20px;
+                font-size: 14px}
+              #footer {text-align: center; color: #AAA; font-size: 12px}
+            </style>
+
+          </head>
+
+          <body>
+            <div id="header">
+              <h1>Nanite Control Tower</h1>
+            </div>
+
+            <h2>#{@mapper.options[:vhost]}</h2>
+            <div class="section">
+              <form method="post" action="/">
+                <input type="hidden" value="POST" name="_method"/>
+
+                  <label>Send</label>
+                  <select name="type">
+                    <option #{@selection == 'least_loaded' ? 'selected="true"' : ''} value="least_loaded">the least loaded nanite</option>
+                    <option #{@selection == 'random' ? 'selected="true"' : ''} value="random">a random nanite</option>
+                    <option #{@selection == 'all' ? 'selected="true"' : ''} value="all">all nanites</option>
+                    <option #{@selection == 'rr' ? 'selected="true"' : ''} value="rr">a nanite chosen by round robin</option>
+                    #{@mapper.cluster.nanites.map {|k,v| "<option #{@selection == k ? 'selected="true"' : ''} value='#{k}'>#{k}</option>" }.join}
+                  </select>
+
+                  <label>providing service</label>
+                  #{services}
+
+                  <label>the payload</label>
+                  <input type="text" class="text" name="payload" id="payload"/>
+
+                  <input type="submit" class="submit" value="Go!" name="submit"/>
+              </form>
+
+              #{"<h3>Responses</h3>" if content}
+              #{content}
+            </div>
+
+            <h2>Running nanites</h2>
+            <div class="section">
+              #{"No nanites online." if @mapper.cluster.nanites.size == 0}
+              <ul>
+                #{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}</li>" }.join}
+              </ul>
+            </div>
+            <div id="footer">
+              Nanite #{Nanite::VERSION}
+              <br />
+              &copy; 2009 a bunch of random geeks
+            </div>
+          </body>
+        </html>
+      }
+    end
+  end
+end

data/lib/nanite/agent.rb

@@ -0,0 +1,250 @@
+module Nanite
+  class Agent
+    include AMQPHelper
+    include FileStreaming
+    include ConsoleHelper
+    include DaemonizeHelper
+
+    attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags
+    attr_accessor :status_proc
+
+    DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'nanite', :ping_time => 15,
+      :default_services => []}) unless defined?(DEFAULT_OPTIONS)
+
+    # Initializes a new agent and establishes AMQP connection.
+    # This must be used inside EM.run block or if EventMachine reactor
+    # is already started, for instance, by a Thin server that your Merb/Rails
+    # application runs on.
+    #
+    # Agent options:
+    #
+    # identity    : identity of this agent, may be any string
+    #
+    # status_proc : a callable object that returns agent load as a string,
+    #               defaults to load averages string extracted from `uptime`
+    # format      : format to use for packets serialization. One of the three:
+    #               :marshall, :json, or :yaml. Defaults to
+    #               Ruby's Marshall format. For interoperability with
+    #               AMQP clients implemented in other languages, use JSON.
+    #
+    #               Note that Nanite uses JSON gem,
+    #               and ActiveSupport's JSON encoder may cause clashes
+    #               if ActiveSupport is loaded after JSON gem.
+    #
+    # root        : application root for this agent, defaults to Dir.pwd
+    #
+    # log_dir     : path to directory where agent stores it's log file
+    #               if not given, app_root is used.
+    #
+    # file_root   : path to directory to files this agent provides
+    #               defaults to app_root/files
+    #
+    # ping_time   : time interval in seconds between two subsequent heartbeat messages
+    #               this agent broadcasts. Default value is 15.
+    #
+    # console     : true tells Nanite to start interactive console
+    #
+    # daemonize   : true tells Nanite to daemonize
+    #
+    # pid_dir     : path to the directory where the agent stores its pid file (only if daemonized)
+    #               defaults to the root or the current working directory.
+    #
+    # services    : list of services provided by this agent, by default
+    #               all methods exposed by actors are listed
+    #
+    # single_threaded: Run all operations in one thread
+    #
+    # Connection options:
+    #
+    # vhost    : AMQP broker vhost that should be used
+    #
+    # user     : AMQP broker user
+    #
+    # pass     : AMQP broker password
+    #
+    # host     : host AMQP broker (or node of interest) runs on,
+    #            defaults to 0.0.0.0
+    #
+    # port     : port AMQP broker (or node of interest) runs on,
+    #            this defaults to 5672, port used by some widely
+    #            used AMQP brokers (RabbitMQ and ZeroMQ)
+    #
+    # On start Nanite reads config.yml, so it is common to specify
+    # options in the YAML file. However, when both Ruby code options
+    # and YAML file specify option, Ruby code options take precedence.
+    def self.start(options = {})
+      agent = new(options)
+      agent.run
+      agent
+    end
+
+    def initialize(opts)
+      set_configuration(opts)
+      @tags = []
+      @tags << opts[:tag]
+      @tags.flatten!
+      @options.freeze
+    end
+    
+    def run
+      log_path = false
+      if @options[:daemonize]
+        log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      Log.init(@identity, log_path)
+      Log.level = @options[:log_level] if @options[:log_level]
+      @serializer = Serializer.new(@options[:format])
+      @status_proc = lambda { parse_uptime(`uptime`) rescue 'no status' }
+      pid_file = PidFile.new(@identity, @options)
+      pid_file.check
+      if @options[:daemonize]
+        daemonize
+        pid_file.write
+        at_exit { pid_file.remove }
+      end
+      @amq = start_amqp(@options)
+      @registry = ActorRegistry.new
+      @dispatcher = Dispatcher.new(@amq, @registry, @serializer, @identity, @options)
+      setup_mapper_proxy
+      load_actors
+      setup_traps
+      setup_queue
+      advertise_services
+      setup_heartbeat
+      at_exit { un_register } unless $TESTING
+      start_console if @options[:console] && !@options[:daemonize]
+    end
+
+    def register(actor, prefix = nil)
+      registry.register(actor, prefix)
+    end
+
+    # Can be used in agent's initialization file to register a security module
+    # This security module 'authorize' method will be called back whenever the
+    # agent receives a request and will be given the corresponding deliverable.
+    # It should return 'true' for the request to proceed.
+    # Requests will return 'deny_token' or the string "Denied" by default when
+    # 'authorize' does not return 'true'.
+    def register_security(security, deny_token = "Denied")
+      @security = security
+      @deny_token = deny_token
+    end
+
+    protected
+
+    def set_configuration(opts)
+      @options = DEFAULT_OPTIONS.clone
+      root = opts[:root] || @options[:root]
+      custom_config = if root
+        file = File.expand_path(File.join(root, 'config.yml'))
+        File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {}
+      else
+        {}
+      end
+      opts.delete(:identity) unless opts[:identity]
+      @options.update(custom_config.merge(opts))
+      @options[:file_root] ||= File.join(@options[:root], 'files')
+      return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
+      token = Identity.generate
+      @identity = "nanite-#{token}"
+      File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd|
+        fd.write(YAML.dump(custom_config.merge(:identity => token)))
+      end
+    end
+
+    def load_actors
+      return unless options[:root]
+      actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
+      actors = @options[:actors]
+      Dir["#{actors_dir}/*.rb"].each do |actor|
+        next if actors && !actors.include?(File.basename(actor, ".rb"))
+        Nanite::Log.info("loading actor: #{actor}")
+        require actor
+      end
+      init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
+      instance_eval(File.read(init_path), init_path) if File.exist?(init_path)
+    end
+
+    def receive(packet)
+      case packet
+      when Advertise
+        Nanite::Log.debug("handling Advertise: #{packet.inspect}")
+        advertise_services
+      when Request, Push
+        Nanite::Log.debug("handling Request: #{packet.inspect}")
+        if @security && !@security.authorize(packet)
+          if packet.kind_of?(Request)
+            r = Result.new(packet.token, packet.reply_to, @deny_token, identity)
+            amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
+          end
+        else
+          dispatcher.dispatch(packet)
+        end
+      when Result
+        Nanite::Log.debug("handling Result: #{packet.inspect}")
+        @mapper_proxy.handle_result(packet)
+      when IntermediateMessage
+        Nanite::Log.debug("handling Intermediate Result: #{packet.inspect}")
+        @mapper_proxy.handle_intermediate_result(packet)
+      end
+    end
+    
+    def tag(*tags)
+      tags.each {|t| @tags << t}
+      @tags.uniq!
+    end
+
+    def setup_queue
+      amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
+        begin
+          info.ack
+          packet = serializer.load(msg)
+          receive(packet)
+        rescue Exception => e
+          Nanite::Log.error("Error handling packet: #{e.message}")
+        end
+      end
+    end
+
+    def setup_heartbeat
+      EM.add_periodic_timer(options[:ping_time]) do
+        amq.fanout('heartbeat', :no_declare => options[:secure]).publish(serializer.dump(Ping.new(identity, status_proc.call)))
+      end
+    end
+    
+    def setup_mapper_proxy
+      @mapper_proxy = MapperProxy.new(identity, options)
+    end
+    
+    def setup_traps
+      ['INT', 'TERM'].each do |sig|
+        old = trap(sig) do
+          un_register
+          amq.instance_variable_get('@connection').close do
+            EM.stop
+            old.call if old.is_a? Proc
+          end
+        end
+      end
+    end
+    
+    def un_register
+      unless @unregistered
+        @unregistered = true
+        amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity)))
+      end
+    end
+
+    def advertise_services
+      Nanite::Log.debug("advertise_services: #{registry.services.inspect}")
+      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(Register.new(identity, registry.services, status_proc.call, self.tags)))
+    end
+
+    def parse_uptime(up)
+      if up =~ /load averages?: (.*)/
+        a,b,c = $1.split(/\s+|,\s+/)
+        (a.to_f + b.to_f + c.to_f) / 3
+      end
+    end
+  end
+end