RubyGems - shift-nanite - Versions diffs - 0.4.1.2 - Mend

shift-nanite 0.4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (63) hide show

data/LICENSE +201 -0
data/README.rdoc +430 -0
data/Rakefile +76 -0
data/TODO +24 -0
data/bin/nanite-admin +65 -0
data/bin/nanite-agent +79 -0
data/bin/nanite-mapper +50 -0
data/lib/nanite.rb +74 -0
data/lib/nanite/actor.rb +71 -0
data/lib/nanite/actor_registry.rb +26 -0
data/lib/nanite/admin.rb +138 -0
data/lib/nanite/agent.rb +264 -0
data/lib/nanite/amqp.rb +58 -0
data/lib/nanite/cluster.rb +250 -0
data/lib/nanite/config.rb +112 -0
data/lib/nanite/console.rb +39 -0
data/lib/nanite/daemonize.rb +13 -0
data/lib/nanite/identity.rb +16 -0
data/lib/nanite/job.rb +104 -0
data/lib/nanite/local_state.rb +38 -0
data/lib/nanite/log.rb +66 -0
data/lib/nanite/log/formatter.rb +39 -0
data/lib/nanite/mapper.rb +309 -0
data/lib/nanite/mapper_proxy.rb +67 -0
data/lib/nanite/nanite_dispatcher.rb +92 -0
data/lib/nanite/packets.rb +365 -0
data/lib/nanite/pid_file.rb +52 -0
data/lib/nanite/reaper.rb +39 -0
data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
data/lib/nanite/security/certificate.rb +55 -0
data/lib/nanite/security/certificate_cache.rb +66 -0
data/lib/nanite/security/distinguished_name.rb +34 -0
data/lib/nanite/security/encrypted_document.rb +46 -0
data/lib/nanite/security/rsa_key_pair.rb +53 -0
data/lib/nanite/security/secure_serializer.rb +68 -0
data/lib/nanite/security/signature.rb +46 -0
data/lib/nanite/security/static_certificate_store.rb +35 -0
data/lib/nanite/security_provider.rb +47 -0
data/lib/nanite/serializer.rb +52 -0
data/lib/nanite/state.rb +168 -0
data/lib/nanite/streaming.rb +125 -0
data/lib/nanite/util.rb +58 -0
data/spec/actor_registry_spec.rb +60 -0
data/spec/actor_spec.rb +77 -0
data/spec/agent_spec.rb +240 -0
data/spec/cached_certificate_store_proxy_spec.rb +34 -0
data/spec/certificate_cache_spec.rb +49 -0
data/spec/certificate_spec.rb +27 -0
data/spec/cluster_spec.rb +622 -0
data/spec/distinguished_name_spec.rb +24 -0
data/spec/encrypted_document_spec.rb +21 -0
data/spec/job_spec.rb +251 -0
data/spec/local_state_spec.rb +130 -0
data/spec/nanite_dispatcher_spec.rb +136 -0
data/spec/packet_spec.rb +220 -0
data/spec/rsa_key_pair_spec.rb +33 -0
data/spec/secure_serializer_spec.rb +41 -0
data/spec/serializer_spec.rb +107 -0
data/spec/signature_spec.rb +30 -0
data/spec/spec_helper.rb +33 -0
data/spec/static_certificate_store_spec.rb +30 -0
data/spec/util_spec.rb +63 -0
metadata +129 -0

data/lib/nanite/admin.rb ADDED

@@ -0,0 +1,138 @@
+require 'rack'
+module Nanite
+  # This is a Rack app for nanite-admin.  You need to have an async capable
+  # version of Thin installed for this to work.  See bin/nanite-admin for install
+  # instructions.
+  class Admin
+    def initialize(mapper)
+      @mapper = mapper
+    end
+    AsyncResponse = [-1, {}, []].freeze
+    def call(env)
+      req = Rack::Request.new(env)
+      if cmd = req.params['command']
+        @command = cmd
+        @selection = req.params['type'] if req.params['type']
+        options = {}
+        case @selection
+        when 'least_loaded', 'random', 'all', 'rr'
+          options[:selector] = @selection
+        else
+          options[:target] = @selection
+        end
+        @mapper.request(cmd, req.params['payload'], options) do |response, responsejob|
+          env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]]
+        end
+        AsyncResponse
+      else
+        [200, {'Content-Type' => 'text/html'}, layout]
+      end
+    end
+    def services
+      buf = "<select name='command'>"
+      @mapper.cluster.nanites.all_services.each do |srv|
+        buf << "<option value='#{srv}' #{@command == srv ? 'selected="true"' : ''}>#{srv}</option>"
+      end
+      buf << "</select>"
+      buf
+    end
+    def ul(hash, job)
+      buf = "<ul>"
+      hash.each do |k,v|
+        buf << "<li><div class=\"nanite\">#{k}:</div><div class=\"response\">#{v.inspect}</div>"
+        if job.intermediate_state && job.intermediate_state[k]
+          buf << "<div class=\"intermediatestates\"><span class=\"statenote\">intermediate state:</span> #{job.intermediate_state[k].inspect}</div>"
+        end
+        buf << "</li>"
+      end
+      buf << "</ul>"
+      buf
+    end
+    def layout(content=nil)
+      %Q{
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns='http://www.w3.org/1999/xhtml'>
+          <head>
+            <meta content='text/html; charset=utf-8' http-equiv='Content-Type' />
+            <meta content='en' http-equiv='Content-Language' />
+            <meta content='Engineyard' name='author' />
+            <title>Nanite Control Tower</title>
+            <style>
+              body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
+              ul {margin: 0; padding: 0; margin-left: 10px}
+              li {list-style-type: none; margin-bottom: 6px}
+              li .nanite {font-weight: bold; font-size: 12px}
+              li .response {padding: 8px}
+              li .intermediatestates {padding: 8px; font-size: 10px;}
+              li .intermediatestates span.statenote {font-style: italic;}
+              h1, h2, h3 {margin-top: none; padding: none; margin-left: 40px;}
+              h1 {font-size: 22px; margin-top: 40px; margin-bottom: 30px; border-bottom: 1px solid #ddd; padding-bottom: 6px;
+                margin-right: 40px}
+              h2 {font-size: 16px;}
+              h3 {margin-left: 0; font-size: 14px}
+              .section {border: 1px solid #ccc; background-color: #fefefe; padding: 10px; margin: 20px 40px; padding: 20px;
+                font-size: 14px}
+              #footer {text-align: center; color: #AAA; font-size: 12px}
+            </style>
+          </head>
+          <body>
+            <div id="header">
+              <h1>Nanite Control Tower</h1>
+            </div>
+            <h2>#{@mapper.options[:vhost]}</h2>
+            <div class="section">
+              <form method="post" action="/">
+                <input type="hidden" value="POST" name="_method"/>
+                  <label>Send</label>
+                  <select name="type">
+                    <option #{@selection == 'least_loaded' ? 'selected="true"' : ''} value="least_loaded">the least loaded nanite</option>
+                    <option #{@selection == 'random' ? 'selected="true"' : ''} value="random">a random nanite</option>
+                    <option #{@selection == 'all' ? 'selected="true"' : ''} value="all">all nanites</option>
+                    <option #{@selection == 'rr' ? 'selected="true"' : ''} value="rr">a nanite chosen by round robin</option>
+                    #{@mapper.cluster.nanites.map {|k,v| "<option #{@selection == k ? 'selected="true"' : ''} value='#{k}'>#{k}</option>" }.join}
+                  </select>
+                  <label>providing service</label>
+                  #{services}
+                  <label>the payload</label>
+                  <input type="text" class="text" name="payload" id="payload"/>
+                  <input type="submit" class="submit" value="Go!" name="submit"/>
+              </form>
+              #{"<h3>Responses</h3>" if content}
+              #{content}
+            </div>
+            <h2>Running nanites</h2>
+            <div class="section">
+              #{"No nanites online." if @mapper.cluster.nanites.size == 0}
+              <ul>
+                #{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}<br />tags: #{v[:tags].to_a.inspect}</li>" }.join}
+              </ul>
+            </div>
+            <div id="footer">
+              Nanite #{Nanite::VERSION}
+              <br />
+              &copy; 2009 a bunch of random geeks
+            </div>
+          </body>
+        </html>
+      }
+    end
+  end
+end

data/lib/nanite/agent.rb ADDED

@@ -0,0 +1,264 @@
+module Nanite
+  class Agent
+    include AMQPHelper
+    include FileStreaming
+    include ConsoleHelper
+    include DaemonizeHelper
+    attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags
+    attr_accessor :status_proc
+    DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({
+      :user => 'nanite',
+      :ping_time => 15,
+      :default_services => []
+    }) unless defined?(DEFAULT_OPTIONS)
+    # Initializes a new agent and establishes AMQP connection.
+    # This must be used inside EM.run block or if EventMachine reactor
+    # is already started, for instance, by a Thin server that your Merb/Rails
+    # application runs on.
+    #
+    # Agent options:
+    #
+    # identity    : identity of this agent, may be any string
+    #
+    # status_proc : a callable object that returns agent load as a string,
+    #               defaults to load averages string extracted from `uptime`
+    # format      : format to use for packets serialization. One of the three:
+    #               :marshall, :json, or :yaml. Defaults to
+    #               Ruby's Marshall format. For interoperability with
+    #               AMQP clients implemented in other languages, use JSON.
+    #
+    #               Note that Nanite uses JSON gem,
+    #               and ActiveSupport's JSON encoder may cause clashes
+    #               if ActiveSupport is loaded after JSON gem.
+    #
+    # root        : application root for this agent, defaults to Dir.pwd
+    #
+    # log_dir     : path to directory where agent stores it's log file
+    #               if not given, app_root is used.
+    #
+    # file_root   : path to directory to files this agent provides
+    #               defaults to app_root/files
+    #
+    # ping_time   : time interval in seconds between two subsequent heartbeat messages
+    #               this agent broadcasts. Default value is 15.
+    #
+    # console     : true tells Nanite to start interactive console
+    #
+    # daemonize   : true tells Nanite to daemonize
+    #
+    # pid_dir     : path to the directory where the agent stores its pid file (only if daemonized)
+    #               defaults to the root or the current working directory.
+    #
+    # services    : list of services provided by this agent, by default
+    #               all methods exposed by actors are listed
+    #
+    # single_threaded: Run all operations in one thread
+    #
+    # threadpool_size: Number of threads to run operations in
+    #
+    # Connection options:
+    #
+    # vhost    : AMQP broker vhost that should be used
+    #
+    # user     : AMQP broker user
+    #
+    # pass     : AMQP broker password
+    #
+    # host     : host AMQP broker (or node of interest) runs on,
+    #            defaults to 0.0.0.0
+    #
+    # port     : port AMQP broker (or node of interest) runs on,
+    #            this defaults to 5672, port used by some widely
+    #            used AMQP brokers (RabbitMQ and ZeroMQ)
+    #
+    # On start Nanite reads config.yml, so it is common to specify
+    # options in the YAML file. However, when both Ruby code options
+    # and YAML file specify option, Ruby code options take precedence.
+    def self.start(options = {})
+      agent = new(options)
+      agent.run
+      agent
+    end
+    def initialize(opts)
+      set_configuration(opts)
+      @tags = []
+      @tags << opts[:tag]
+      @tags.flatten!
+      @options.freeze
+    end
+    def run
+      Log.init(@identity, @options[:log_path])
+      Log.level = @options[:log_level] if @options[:log_level]
+      @serializer = Serializer.new(@options[:format])
+      @status_proc = lambda { parse_uptime(`uptime 2> /dev/null`) rescue 'no status' }
+      pid_file = PidFile.new(@identity, @options)
+      pid_file.check
+      if @options[:daemonize]
+        daemonize(@identity, @options)
+        pid_file.write
+        at_exit { pid_file.remove }
+      end
+      @amq = start_amqp(@options)
+      @registry = ActorRegistry.new
+      @dispatcher = Dispatcher.new(@amq, @registry, @serializer, @identity, @options)
+      setup_mapper_proxy
+      load_actors
+      setup_traps
+      setup_queue
+      advertise_services
+      setup_heartbeat
+      at_exit { un_register } unless $TESTING
+      start_console if @options[:console] && !@options[:daemonize]
+    end
+    def register(actor, prefix = nil)
+      registry.register(actor, prefix)
+    end
+    # Can be used in agent's initialization file to register a security module
+    # This security module 'authorize' method will be called back whenever the
+    # agent receives a request and will be given the corresponding deliverable.
+    # It should return 'true' for the request to proceed.
+    # Requests will return 'deny_token' or the string "Denied" by default when
+    # 'authorize' does not return 'true'.
+    def register_security(security, deny_token = "Denied")
+      @security = security
+      @deny_token = deny_token
+    end
+    protected
+    def set_configuration(opts)
+      @options = DEFAULT_OPTIONS.clone
+      root = opts[:root] || @options[:root]
+      custom_config = if root
+        file = File.expand_path(File.join(root, 'config.yml'))
+        File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {}
+      else
+        {}
+      end
+      opts.delete(:identity) unless opts[:identity]
+      @options.update(custom_config.merge(opts))
+      @options[:file_root] ||= File.join(@options[:root], 'files')
+      @options[:log_path] = false
+      if @options[:daemonize]
+        @options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
+      end
+      return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
+      token = Identity.generate
+      @identity = "nanite-#{token}"
+      File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd|
+        fd.write(YAML.dump(custom_config.merge(:identity => token)))
+      end
+    end
+    def load_actors
+      return unless options[:root]
+      actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
+      Nanite::Log.warn("Actors dir #{actors_dir} does not exist or is not reachable") unless File.directory?(actors_dir)
+      actors = @options[:actors]
+      Dir["#{actors_dir}/*.rb"].each do |actor|
+        next if actors && !actors.include?(File.basename(actor, ".rb"))
+        Nanite::Log.info("[setup] loading #{actor}")
+        require actor
+      end
+      init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
+      if File.exist?(init_path)
+        instance_eval(File.read(init_path), init_path)
+      else
+        Nanite::Log.warn("init.rb #{init_path} does not exist or is not reachable") unless File.exists?(init_path)
+      end
+    end
+    def receive(packet)
+      Nanite::Log.debug("RECV #{packet.to_s}")
+      case packet
+      when Advertise
+        Nanite::Log.info("RECV #{packet.to_s}") unless Nanite::Log.level == :debug
+        advertise_services
+      when Request, Push
+        if @security && !@security.authorize(packet)
+          Nanite::Log.warn("RECV NOT AUTHORIZED #{packet.to_s}")
+          if packet.kind_of?(Request)
+            r = Result.new(packet.token, packet.reply_to, @deny_token, identity)
+            amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
+          end
+        else
+          Nanite::Log.info("RECV #{packet.to_s([:from, :tags])}") unless Nanite::Log.level == :debug
+          dispatcher.dispatch(packet)
+        end
+      when Result
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
+        @mapper_proxy.handle_result(packet)
+      when IntermediateMessage
+        Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == :debug
+        @mapper_proxy.handle_intermediate_result(packet)
+      end
+    end
+    def tag(*tags)
+      tags.each {|t| @tags << t}
+      @tags.uniq!
+    end
+    def setup_queue
+      amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
+        begin
+          info.ack
+          receive(serializer.load(msg))
+        rescue Exception => e
+          Nanite::Log.error("RECV #{e.message}")
+        end
+      end
+    end
+    def setup_heartbeat
+      EM.add_periodic_timer(options[:ping_time]) do
+        amq.fanout('heartbeat', :no_declare => options[:secure]).publish(serializer.dump(Ping.new(identity, status_proc.call)))
+      end
+    end
+    def setup_mapper_proxy
+      @mapper_proxy = MapperProxy.new(identity, options)
+    end
+    def setup_traps
+      ['INT', 'TERM'].each do |sig|
+        old = trap(sig) do
+          un_register
+          amq.instance_variable_get('@connection').close do
+            EM.stop
+            old.call if old.is_a? Proc
+          end
+        end
+      end
+    end
+    def un_register
+      unless @unregistered
+        @unregistered = true
+        Nanite::Log.info("SEND [un_register]")
+        amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity)))
+      end
+    end
+    def advertise_services
+      reg = Register.new(identity, registry.services, status_proc.call, self.tags)
+      Nanite::Log.info("SEND #{reg.to_s}")
+      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(reg))
+    end
+    def parse_uptime(up)
+      if up =~ /load averages?: (.*)/
+        a,b,c = $1.split(/\s+|,\s+/)
+        (a.to_f + b.to_f + c.to_f) / 3
+      end
+    end
+  end
+end

data/lib/nanite/amqp.rb ADDED

@@ -0,0 +1,58 @@
+class MQ
+  class Queue
+    # Asks the broker to redeliver all unacknowledged messages on a
+    # specifieid channel. Zero or more messages may be redelivered.
+    #
+    # * requeue (default false)
+    # If this parameter is false, the message will be redelivered to the original recipient.
+    # If this flag is true, the server will attempt to requeue the message, potentially then
+    # delivering it to an alternative subscriber.
+    #
+    def recover requeue = false
+      @mq.callback{
+        @mq.send Protocol::Basic::Recover.new({ :requeue => requeue })
+      }
+      self
+    end
+  end
+end
+# monkey patch to the amqp gem that adds :no_declare => true option for new
+# Exchange objects. This allows us to send messeages to exchanges that are
+# declared by the mappers and that we have no configuration priviledges on.
+# temporary until we get this into amqp proper
+MQ::Exchange.class_eval do
+  def initialize mq, type, name, opts = {}
+    @mq = mq
+    @type, @name, @opts = type, name, opts
+    @mq.exchanges[@name = name] ||= self
+    @key = opts[:key]
+    @mq.callback{
+      @mq.send AMQP::Protocol::Exchange::Declare.new({ :exchange => name,
+                                                 :type => type,
+                                                 :nowait => true }.merge(opts))
+    } unless name == "amq.#{type}" or name == ''  or opts[:no_declare]
+  end
+end
+module Nanite
+  module AMQPHelper
+    def start_amqp(options)
+      connection = AMQP.connect({
+        :user => options[:user],
+        :pass => options[:pass],
+        :vhost => options[:vhost],
+        :host => options[:host],
+        :port => (options[:port] || ::AMQP::PORT).to_i,
+        :insist => options[:insist] || false,
+        :retry => options[:retry] || 5,
+        :connection_status => options[:connection_callback] || proc {|event|
+          Nanite::Log.debug("CONNECTED to MQ") if event == :connected
+          Nanite::Log.debug("DISCONNECTED from MQ") if event == :disconnected
+        }
+      })
+      MQ.new(connection)
+    end
+  end
+end