RubyGems - shift-nanite - Versions diffs - 0.4.1.2 - Mend

shift-nanite 0.4.1.2

Files changed (63) hide show

data/LICENSE +201 -0
data/README.rdoc +430 -0
data/Rakefile +76 -0
data/TODO +24 -0
data/bin/nanite-admin +65 -0
data/bin/nanite-agent +79 -0
data/bin/nanite-mapper +50 -0
data/lib/nanite.rb +74 -0
data/lib/nanite/actor.rb +71 -0
data/lib/nanite/actor_registry.rb +26 -0
data/lib/nanite/admin.rb +138 -0
data/lib/nanite/agent.rb +264 -0
data/lib/nanite/amqp.rb +58 -0
data/lib/nanite/cluster.rb +250 -0
data/lib/nanite/config.rb +112 -0
data/lib/nanite/console.rb +39 -0
data/lib/nanite/daemonize.rb +13 -0
data/lib/nanite/identity.rb +16 -0
data/lib/nanite/job.rb +104 -0
data/lib/nanite/local_state.rb +38 -0
data/lib/nanite/log.rb +66 -0
data/lib/nanite/log/formatter.rb +39 -0
data/lib/nanite/mapper.rb +309 -0
data/lib/nanite/mapper_proxy.rb +67 -0
data/lib/nanite/nanite_dispatcher.rb +92 -0
data/lib/nanite/packets.rb +365 -0
data/lib/nanite/pid_file.rb +52 -0
data/lib/nanite/reaper.rb +39 -0
data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
data/lib/nanite/security/certificate.rb +55 -0
data/lib/nanite/security/certificate_cache.rb +66 -0
data/lib/nanite/security/distinguished_name.rb +34 -0
data/lib/nanite/security/encrypted_document.rb +46 -0
data/lib/nanite/security/rsa_key_pair.rb +53 -0
data/lib/nanite/security/secure_serializer.rb +68 -0
data/lib/nanite/security/signature.rb +46 -0
data/lib/nanite/security/static_certificate_store.rb +35 -0
data/lib/nanite/security_provider.rb +47 -0
data/lib/nanite/serializer.rb +52 -0
data/lib/nanite/state.rb +168 -0
data/lib/nanite/streaming.rb +125 -0
data/lib/nanite/util.rb +58 -0
data/spec/actor_registry_spec.rb +60 -0
data/spec/actor_spec.rb +77 -0
data/spec/agent_spec.rb +240 -0
data/spec/cached_certificate_store_proxy_spec.rb +34 -0
data/spec/certificate_cache_spec.rb +49 -0
data/spec/certificate_spec.rb +27 -0
data/spec/cluster_spec.rb +622 -0
data/spec/distinguished_name_spec.rb +24 -0
data/spec/encrypted_document_spec.rb +21 -0
data/spec/job_spec.rb +251 -0
data/spec/local_state_spec.rb +130 -0
data/spec/nanite_dispatcher_spec.rb +136 -0
data/spec/packet_spec.rb +220 -0
data/spec/rsa_key_pair_spec.rb +33 -0
data/spec/secure_serializer_spec.rb +41 -0
data/spec/serializer_spec.rb +107 -0
data/spec/signature_spec.rb +30 -0
data/spec/spec_helper.rb +33 -0
data/spec/static_certificate_store_spec.rb +30 -0
data/spec/util_spec.rb +63 -0
metadata +129 -0

@@ -0,0 +1,138 @@
+require 'rack'
+module Nanite
+  # This is a Rack app for nanite-admin.  You need to have an async capable
+  # version of Thin installed for this to work.  See bin/nanite-admin for install
+  # instructions.
+  class Admin
+    def initialize(mapper)
+      @mapper = mapper
+    end
+    AsyncResponse = [-1, {}, []].freeze
+    def call(env)
+      req = Rack::Request.new(env)
+      if cmd = req.params['command']
+        @command = cmd
+        @selection = req.params['type'] if req.params['type']
+        options = {}
+        case @selection
+        when 'least_loaded', 'random', 'all', 'rr'
+          options[:selector] = @selection
+        else
+          options[:target] = @selection
+        end
+        @mapper.request(cmd, req.params['payload'], options) do |response, responsejob|
+          env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]]
+        end
+        AsyncResponse
+      else
+        [200, {'Content-Type' => 'text/html'}, layout]
+      end
+    end
+    def services
+      buf = "<select name='command'>"
+      @mapper.cluster.nanites.all_services.each do |srv|
+        buf << "<option value='#{srv}' #{@command == srv ? 'selected="true"' : ''}>#{srv}</option>"
+      end
+      buf << "</select>"
+      buf
+    end
+    def ul(hash, job)
+      buf = "<ul>"
+      hash.each do |k,v|
+        buf << "<li><div class=\"nanite\">#{k}:</div><div class=\"response\">#{v.inspect}</div>"
+        if job.intermediate_state && job.intermediate_state[k]
+          buf << "<div class=\"intermediatestates\"><span class=\"statenote\">intermediate state:</span> #{job.intermediate_state[k].inspect}</div>"
+        end
+        buf << "</li>"
+      end
+      buf << "</ul>"
+      buf
+    end
+    def layout(content=nil)
+      %Q{
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns='http://www.w3.org/1999/xhtml'>
+          <head>
+            <meta content='text/html; charset=utf-8' http-equiv='Content-Type' />
+            <meta content='en' http-equiv='Content-Language' />
+            <meta content='Engineyard' name='author' />
+            <title>Nanite Control Tower</title>
+            <style>
+              body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
+              ul {margin: 0; padding: 0; margin-left: 10px}
+              li {list-style-type: none; margin-bottom: 6px}
+              li .nanite {font-weight: bold; font-size: 12px}
+              li .response {padding: 8px}
+              li .intermediatestates {padding: 8px; font-size: 10px;}
+              li .intermediatestates span.statenote {font-style: italic;}
+              h1, h2, h3 {margin-top: none; padding: none; margin-left: 40px;}
+              h1 {font-size: 22px; margin-top: 40px; margin-bottom: 30px; border-bottom: 1px solid #ddd; padding-bottom: 6px;
+                margin-right: 40px}
+              h2 {font-size: 16px;}
+              h3 {margin-left: 0; font-size: 14px}
+              .section {border: 1px solid #ccc; background-color: #fefefe; padding: 10px; margin: 20px 40px; padding: 20px;
+                font-size: 14px}
+              #footer {text-align: center; color: #AAA; font-size: 12px}
+            </style>
+          </head>
+          <body>
+            <div id="header">
+              <h1>Nanite Control Tower</h1>
+            </div>
+            <h2>#{@mapper.options[:vhost]}</h2>
+            <div class="section">
+              <form method="post" action="/">
+                <input type="hidden" value="POST" name="_method"/>
+                  <label>Send</label>
+                  <select name="type">
+                    <option #{@selection == 'least_loaded' ? 'selected="true"' : ''} value="least_loaded">the least loaded nanite</option>
+                    <option #{@selection == 'random' ? 'selected="true"' : ''} value="random">a random nanite</option>
+                    <option #{@selection == 'all' ? 'selected="true"' : ''} value="all">all nanites</option>
+                    <option #{@selection == 'rr' ? 'selected="true"' : ''} value="rr">a nanite chosen by round robin</option>
+                    #{@mapper.cluster.nanites.map {|k,v| "<option #{@selection == k ? 'selected="true"' : ''} value='#{k}'>#{k}</option>" }.join}
+                  </select>
+                  <label>providing service</label>
+                  #{services}
+                  <label>the payload</label>
+                  <input type="text" class="text" name="payload" id="payload"/>
+                  <input type="submit" class="submit" value="Go!" name="submit"/>
+              </form>
+              #{"<h3>Responses</h3>" if content}
+              #{content}
+            </div>
+            <h2>Running nanites</h2>
+            <div class="section">
+              #{"No nanites online." if @mapper.cluster.nanites.size == 0}
+              <ul>
+                #{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}<br />tags: #{v[:tags].to_a.inspect}</li>" }.join}
+              </ul>
+            </div>
+            <div id="footer">
+              Nanite #{Nanite::VERSION}
+              <br />
+              &copy; 2009 a bunch of random geeks
+            </div>
+          </body>
+        </html>
+      }
+    end
+  end
+end

data/lib/nanite/agent.rb ADDED

@@ -0,0 +1,264 @@
+module Nanite
+  class Agent
+    include AMQPHelper
+    include FileStreaming
+    include ConsoleHelper
+    include DaemonizeHelper
+    attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags
+    attr_accessor :status_proc
+    DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({
+      :user => 'nanite',
+      :ping_time => 15,
+      :default_services => []
+    }) unless defined?(DEFAULT_OPTIONS)
+    # Initializes a new agent and establishes AMQP connection.
+    # This must be used inside EM.run block or if EventMachine reactor
+    # is already started, for instance, by a Thin server that your Merb/Rails
+    # application runs on.
+    #
+    # Agent options:
+    #
+    # identity    : identity of this agent, may be any string
+    #
+    # status_proc : a callable object that returns agent load as a string,
+    #               defaults to load averages string extracted from `uptime`
+    # format      : format to use for packets serialization. One of the three:
+    #               :marshall, :json, or :yaml. Defaults to
+    #               Ruby's Marshall format. For interoperability with
+    #               AMQP clients implemented in other languages, use JSON.
+    #
+    #               Note that Nanite uses JSON gem,
+    #               and ActiveSupport's JSON encoder may cause clashes
+    #               if ActiveSupport is loaded after JSON gem.
+    #
+    # root        : application root for this agent, defaults to Dir.pwd
+    #
+    # log_dir     : path to directory where agent stores it's log file
+    #               if not given, app_root is used.
+    #
+    # file_root   : path to directory to files this agent provides
+    #               defaults to app_root/files
+    #
+    # ping_time   : time interval in seconds between two subsequent heartbeat messages
+    #               this agent broadcasts. Default value is 15.
+    #
+    # console     : true tells Nanite to start interactive console
+    #
+    # daemonize   : true tells Nanite to daemonize
+    #
+    # pid_dir     : path to the directory where the agent stores its pid file (only if daemonized)
+    #               defaults to the root or the current working directory.
+    #
+    # services    : list of services provided by this agent, by default
+    #               all methods exposed by actors are listed
+    #
+    # single_threaded: Run all operations in one thread
+    #
+    # threadpool_size: Number of threads to run operations in
+    #
+    # Connection options:
+    #
+    # vhost    : AMQP broker vhost that should be used
+    #
+    # user     : AMQP broker user
+    #
+    # pass     : AMQP broker password
+    #
+    # host     : host AMQP broker (or node of interest) runs on,
+    #            defaults to 0.0.0.0
+    #
+    # port     : port AMQP broker (or node of interest) runs on,
+    #            this defaults to 5672, port used by some widely
+    #            used AMQP brokers (RabbitMQ and ZeroMQ)
+    #
+    # On start Nanite reads config.yml, so it is common to specify
+    # options in the YAML file. However, when both Ruby code options
+    # and YAML file specify option, Ruby code options take precedence.
+    def self.start(options = {})
+      agent = new(options)
+      agent.run
+      agent
+    end
+    def initialize(opts)
+      set_configuration(opts)
+      @tags = []
+      @tags << opts[:tag]
+      @tags.flatten!
+      @options.freeze
+    end
+    def run
+      Log.init(@identity, @options[:log_path])
+      Log.level = @options[:log_level] if @options[:log_level]
+      @serializer = Serializer.new(@options[:format])
+      @status_proc = lambda { parse_uptime(`uptime 2> /dev/null`) rescue 'no status' }
+      pid_file = PidFile.new(@identity, @options)
+      pid_file.check
+      if @options[:daemonize]
+        daemonize(@identity, @options)
+        pid_file.write
+        at_exit { pid_file.remove }
+      end
+      @amq = start_amqp(@options)
+      @registry = ActorRegistry.new
+      @dispatcher = Dispatcher.new(@amq, @registry, @serializer, @identity, @options)
+      setup_mapper_proxy
+      load_actors
+      setup_traps
+      setup_queue
+      advertise_services
+      setup_heartbeat
+      at_exit { un_register } unless $TESTING
+      start_console if @options[:console] && !@options[:daemonize]
+    end
+    def register(actor, prefix = nil)
+      registry.register(actor, prefix)
+    end
+    # Can be used in agent's initialization file to register a security module
+    # This security module 'authorize' method will be called back whenever the
+    # agent receives a request and will be given the corresponding deliverable.
+    # It should return 'true' for the request to proceed.
+    # Requests will return 'deny_token' or the string "Denied" by default when
+    # 'authorize' does not return 'true'.
+    def register_security(security, deny_token = "Denied")
+      @security = security
+      @deny_token = deny_token
+    end
+    protected
+    def set_configuration(opts)
+      @options = DEFAULT_OPTIONS.clone
+      root = opts[:root] || @options[:root]
+      custom_config = if root
+        file = File.expand_path(File.join(root, 'config.yml'))
+        File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {}
+      else
+        {}
+      end
+      opts.delete(:identity) unless opts[:identity]
+      @options.update(custom_config.merge(opts))
+      @options[:file_root] ||= File.join(@options[:root], 'files')
+      @options[:log_path] = false
+      if @options[:daemonize]
+        @options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
+      token = Identity.generate
+      @identity = "nanite-#{token}"
+      File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd|
+        fd.write(YAML.dump(custom_config.merge(:identity => token)))
+      end
+    end
+    def load_actors
+      return unless options[:root]
+      actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
+      Nanite::Log.warn("Actors dir #{actors_dir} does not exist or is not reachable") unless File.directory?(actors_dir)
+      actors = @options[:actors]
+      Dir["#{actors_dir}/*.rb"].each do |actor|
+        next if actors && !actors.include?(File.basename(actor, ".rb"))
+        Nanite::Log.info("[setup] loading #{actor}")
+        require actor
+      end
+      init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
+      if File.exist?(init_path)
+        instance_eval(File.read(init_path), init_path)
+      else
+        Nanite::Log.warn("init.rb #{init_path} does not exist or is not reachable") unless File.exists?(init_path)
+      end
+    end
+    def receive(packet)
+      Nanite::Log.debug("RECV #{packet.to_s}")
+      case packet
+      when Advertise
+        Nanite::Log.info("RECV #{packet.to_s}") unless Nanite::Log.level == :debug
+        advertise_services
+      when Request, Push
+        if @security && !@security.authorize(packet)
+          Nanite::Log.warn("RECV NOT AUTHORIZED #{packet.to_s}")
+          if packet.kind_of?(Request)
+            r = Result.new(packet.token, packet.reply_to, @deny_token, identity)
+            amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
+          end
+        else
+          Nanite::Log.info("RECV #{packet.to_s([:from, :tags])}") unless Nanite::Log.level == :debug
+          dispatcher.dispatch(packet)
+        end
+      when Result
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
+        @mapper_proxy.handle_result(packet)
+      when IntermediateMessage
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
+        @mapper_proxy.handle_intermediate_result(packet)
+      end
+    end
+    def tag(*tags)
+      tags.each {|t| @tags << t}
+      @tags.uniq!
+    end
+    def setup_queue
+      amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
+        begin
+          info.ack
+          receive(serializer.load(msg))
+        rescue Exception => e
+          Nanite::Log.error("RECV #{e.message}")
+        end
+      end
+    end
+    def setup_heartbeat
+      EM.add_periodic_timer(options[:ping_time]) do
+        amq.fanout('heartbeat', :no_declare => options[:secure]).publish(serializer.dump(Ping.new(identity, status_proc.call)))
+      end
+    end
+    def setup_mapper_proxy
+      @mapper_proxy = MapperProxy.new(identity, options)
+    end
+    def setup_traps
+      ['INT', 'TERM'].each do |sig|
+        old = trap(sig) do
+          un_register
+          amq.instance_variable_get('@connection').close do
+            EM.stop
+            old.call if old.is_a? Proc
+          end
+        end
+      end
+    end
+    def un_register
+      unless @unregistered
+        @unregistered = true
+        Nanite::Log.info("SEND [un_register]")
+        amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity)))
+      end
+    end
+    def advertise_services
+      reg = Register.new(identity, registry.services, status_proc.call, self.tags)
+      Nanite::Log.info("SEND #{reg.to_s}")
+      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(reg))
+    end
+    def parse_uptime(up)
+      if up =~ /load averages?: (.*)/
+        a,b,c = $1.split(/\s+|,\s+/)
+        (a.to_f + b.to_f + c.to_f) / 3
+      end
+    end
+  end
+end

data/lib/nanite/amqp.rb ADDED

@@ -0,0 +1,58 @@
+class MQ
+  class Queue
+    # Asks the broker to redeliver all unacknowledged messages on a
+    # specifieid channel. Zero or more messages may be redelivered.
+    #
+    # * requeue (default false)
+    # If this parameter is false, the message will be redelivered to the original recipient.
+    # If this flag is true, the server will attempt to requeue the message, potentially then
+    # delivering it to an alternative subscriber.
+    #
+    def recover requeue = false
+      @mq.callback{
+        @mq.send Protocol::Basic::Recover.new({ :requeue => requeue })
+      }
+      self
+    end
+  end
+end
+# monkey patch to the amqp gem that adds :no_declare => true option for new
+# Exchange objects. This allows us to send messeages to exchanges that are
+# declared by the mappers and that we have no configuration priviledges on.
+# temporary until we get this into amqp proper
+MQ::Exchange.class_eval do
+  def initialize mq, type, name, opts = {}
+    @mq = mq
+    @type, @name, @opts = type, name, opts
+    @mq.exchanges[@name = name] ||= self
+    @key = opts[:key]
+    @mq.callback{
+      @mq.send AMQP::Protocol::Exchange::Declare.new({ :exchange => name,
+                                                 :type => type,
+                                                 :nowait => true }.merge(opts))
+    } unless name == "amq.#{type}" or name == ''  or opts[:no_declare]
+  end
+end
+module Nanite
+  module AMQPHelper
+    def start_amqp(options)
+      connection = AMQP.connect({
+        :user => options[:user],
+        :pass => options[:pass],
+        :vhost => options[:vhost],
+        :host => options[:host],
+        :port => (options[:port] || ::AMQP::PORT).to_i,
+        :insist => options[:insist] || false,
+        :retry => options[:retry] || 5,
+        :connection_status => options[:connection_callback] || proc {|event|
+          Nanite::Log.debug("CONNECTED to MQ") if event == :connected
+          Nanite::Log.debug("DISCONNECTED from MQ") if event == :disconnected
+        }
+      })
+      MQ.new(connection)
+    end
+  end
+end