ezmobius-nanite 0.4.0 → 0.4.1.1

data/README.rdoc

@@ -228,7 +228,7 @@ Second shell
 
 Now run a mapper. Mappers can be run from within your Merb or Rails app, from an interactive irb shell, or from the command line.  For this example we'll run it from the command line so open a third shell window and run the following:
 
-  cd examples
+  cd examples/simpleagent
   ./cli.rb
 
 Which should soon return something like the following.
@@ -296,32 +296,82 @@ Mongrel on the other hand does not use EventMachine and therefore requires to wr
 
 Using nanite with Passenger:
 
-     current = Thread.current
-     Thread.new do
-       AMQP.start(:host => AMQP_HOST) do
-         current.wakeup
-       end
-     end
-     Thread.stop
-
-     # catch these, stop AMQP, stop eventmachine, and re-throw to
-Mongrel/Passenger's signal traps
-     EM.run do
-       ['INT', 'TERM'].each do |sig|
-         old = trap(sig) do
-           AMQP.stop do
-             EM.stop
-             old.call
-           end
-         end
-       end
+if defined?(PhusionPassenger)
+ PhusionPassenger.on_event(:starting_worker_process) do |forked|
+   if forked
+     if EM.reactor_running?
+       EM.stop_event_loop
+       EM.release_machine
+       EM.instance_variable_set( '@reactor_running', false )
      end
+     Thread.current[:mq] = nil
+     AMQP.instance_variable_set('@conn', nil)
+   end
+
+   th = Thread.current
+   Thread.new{
+     Nanite.start_mapper(:host => 'localhost', :user => 'mapper', :pass => 'testing', :vhost => '/nanite', :log_level => 'info')
+   }
+   Thread.stop
+ end
+end
 
 =======
 Where to put the mapper initialization code depends on the framework and our preference. 
 For Rails the canonical place to start our mapper is within nanite.rb (or any other filename you prefer) in config/initalizers.
 In Merb we can use init.rb in config.
 
+== Security
+
+Nanite implements a secure serializer which can be used in place of the other serializers to encrypt the 
+AMQP messages exchanged between the mappers and the agents.
+
+The secure serializer uses X.509 certificates and cryptographic keys to sign and encrypt the messages.
+
+It is important to understand that:
+1. A certificate only includes the public key component of a cryptographic key 
+   pair.
+2. Signing requires the use of a certificate and its private key, checking the
+   signature then only requires the certificate (the idea is that only the
+   signer has the secret private key and thus can sign but anyone can check the 
+   signature).
+3. Encrypting only requires the certificate but decrypting also requires the
+   private key (anyone can encrypt the data but only the intended recipient can
+   decrypt it).
+
+A signing serializer thus needs access to the signer certificate and private 
+key. An encrypting serializer *also* needs access to the intended recipients
+certificates. There needs to be a way to dynamically retrieve the corresponding
+certificates. This is done using certificate stores.
+
+Certificate stores associate identities with certificates. The identity is 
+associated when the data is serialized and can be keyed off to retrieve the 
+right certificate upon deserialization.
+
+Nanite provides a static store implementation which can be used when the 
+certificates used for serialization are always the same and can be kept in 
+memory. Nanite also provides a certificate store proxy cache which can be
+associated with any store implementation and will cache the most used 
+certificates.
+
+The serializer should be initialized prior to being used by calling the 'init'
+method:
+
+  # Initialize serializer, must be called prior to using it.
+  #
+  #  - 'identity':   Identity associated with serialized messages
+  #  - 'cert':       Certificate used to sign serialized messages and
+  #                  decrypt encrypted messages
+  #  - 'key':        Private key corresponding to 'cert'
+  #  - 'store':      Certificate store. Exposes certificates used for
+  #                  encryption and signature validation.
+  #  - 'encrypt':    Whether data should be signed and encrypted ('true')
+  #                  or just signed ('false'), 'true' by default.
+  #
+  def SecureSerializer.init(identity, cert, key, store, encrypt = true)
+
+The 'secure' agent example (examples\secure) shows how the mappers and agents should be
+configured to use the secure serializer.
 
 == Troubleshooting
 

data/Rakefile

@@ -10,7 +10,7 @@ end
 require 'rake/clean'
 
 GEM = "nanite"
-VER = "0.4.0"
+VER = "0.4.1"
 AUTHOR = "Ezra Zygmuntowicz"
 EMAIL = "ezra@engineyard.com"
 HOMEPAGE = "http://github.com/ezmobius/nanite"

data/bin/nanite-agent

@@ -21,25 +21,51 @@ opts = OptionParser.new do |opts|
   opts.on("--ping-time PINGTIME", "Specify how often the agents contacts the mapper") do |ping|
     options[:ping_time] = ping
   end
+  
+  opts.on("--actors-dir DIR", "Path to directory containing actors (NANITE_ROOT/actors by default)") do |dir|
+    options[:actors_dir] = dir
+  end
+  
+  opts.on("--actors ACTORS", "Comma separated list of actors to load (all ruby files in actors directory by default)") do |a|
+    options[:actors] = a.split(',')
+  end
+
+  opts.on("--initrb FILE", "Path to agent initialization file (NANITE_ROOT/init.rb by default)") do |initrb|
+    options[:initrb] = initrb
+  end
+
+  opts.on("--single-threaded", "Run all operations in one thread") do
+    options[:single_threaded] = true
+  end
 end
 
 opts.parse!
 
-if ARGV[0] == 'stop'
+if ARGV[0] == 'stop' || ARGV[0] == 'status'
   agent = Nanite::Agent.new(options)
   pid_file = Nanite::PidFile.new(agent.identity, agent.options)
   unless pid = pid_file.read_pid
     puts "#{pid_file} not found"
     exit
   end
-  puts "Stopping nanite agent #{agent.identity} (pid #{pid})"
-  begin
-    Process.kill('TERM', pid)
-  rescue Errno::ESRCH
-    puts "Process does not exist (pid #{pid})"
-    exit
+  if ARGV[0] == 'stop'
+    puts "Stopping nanite agent #{agent.identity} (pid #{pid})"
+    begin
+      Process.kill('TERM', pid)
+    rescue Errno::ESRCH
+      puts "Process does not exist (pid #{pid})"
+      exit
+    end
+    puts 'Done.'
+  else
+    if Process.getpgid(pid) != -1
+      psdata = `ps up #{pid}`.split("\n").last.split
+      memory = (psdata[5].to_i / 1024)
+      puts "The agent is alive, using #{memory}MB of memory"
+    else
+      puts "The agent is not running but has a stale pid file at #{pid_file}"
+    end
   end
-  puts 'Done.'
   exit
 end
 

data/bin/nanite-mapper

@@ -17,21 +17,31 @@ end
 
 opts.parse!
 
-if ARGV[0] == 'stop'
+if ARGV[0] == 'stop' || ARGV[0] == 'status'
   mapper = Nanite::Mapper.new(options)
   pid_file = Nanite::PidFile.new(mapper.identity, mapper.options)
   unless pid = pid_file.read_pid
     puts "#{pid_file} not found"
     exit
   end
-  puts "Stopping nanite mapper #{mapper.identity} (pid #{pid})"
-  begin
-    Process.kill('TERM', pid)
-  rescue Errno::ESRCH
-    puts "Process does not exist (pid #{pid})"
-    exit
+  if ARGV[0] == 'stop'
+    puts "Stopping nanite mapper #{mapper.identity} (pid #{pid})"
+    begin
+      Process.kill('TERM', pid)
+    rescue Errno::ESRCH
+      puts "Process does not exist (pid #{pid})"
+      exit
+    end
+    puts 'Done.'
+  else
+    if Process.getpgid(pid) != -1
+      psdata = `ps up #{pid}`.split("\n").last.split
+      memory = (psdata[5].to_i / 1024)
+      puts "The mapper is alive, using #{memory}MB of memory"
+    else
+      puts "The mapper is not running but has a stale pid file at #{pid_file}"
+    end
   end
-  puts 'Done.'
   exit
 end
 

data/lib/nanite.rb

@@ -0,0 +1,71 @@
+require 'rubygems'
+require 'amqp'
+require 'mq'
+require 'json'
+require 'logger'
+require 'yaml'
+require 'openssl'
+
+$:.unshift File.dirname(__FILE__)
+require 'nanite/amqp'
+require 'nanite/util'
+require 'nanite/config'
+require 'nanite/packets'
+require 'nanite/identity'
+require 'nanite/console'
+require 'nanite/daemonize'
+require 'nanite/pid_file'
+require 'nanite/job'
+require 'nanite/mapper'
+require 'nanite/actor'
+require 'nanite/actor_registry'
+require 'nanite/streaming'
+require 'nanite/dispatcher'
+require 'nanite/agent'
+require 'nanite/cluster'
+require 'nanite/reaper'
+require 'nanite/log'
+require 'nanite/mapper_proxy'
+require 'nanite/security_provider'
+require 'nanite/security/cached_certificate_store_proxy'
+require 'nanite/security/certificate'
+require 'nanite/security/certificate_cache'
+require 'nanite/security/distinguished_name'
+require 'nanite/security/encrypted_document'
+require 'nanite/security/rsa_key_pair'
+require 'nanite/security/secure_serializer'
+require 'nanite/security/signature'
+require 'nanite/security/static_certificate_store'
+require 'nanite/serializer'
+
+module Nanite
+  VERSION = '0.4.0' unless defined?(Nanite::VERSION)
+
+  class MapperNotRunning < StandardError; end
+
+  class << self
+    attr_reader :mapper, :agent
+
+    def start_agent(options = {})
+      @agent = Nanite::Agent.start(options)
+    end
+
+    def start_mapper(options = {})
+      @mapper = Nanite::Mapper.start(options)
+    end
+
+    def request(*args, &blk)
+      ensure_mapper
+      @mapper.request(*args, &blk)
+    end
+
+    def push(*args)
+      ensure_mapper
+      @mapper.push(*args)
+    end
+
+    def ensure_mapper
+      raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper') unless @mapper
+    end
+  end
+end

data/lib/nanite/actor.rb

@@ -0,0 +1,60 @@
+module Nanite
+  # This mixin provides Nanite actor functionality.
+  #
+  # To use it simply include it your class containing the functionality to be exposed:
+  #
+  #   class Foo
+  #     include Nanite::Actor
+  #     expose :bar
+  #
+  #     def bar(payload)
+  #       # ...
+  #     end
+  #
+  #   end
+  module Actor
+
+    def self.included(base)
+      base.class_eval do 
+        include Nanite::Actor::InstanceMethods
+        extend  Nanite::Actor::ClassMethods
+      end # base.class_eval
+    end # self.included
+    
+    module ClassMethods
+      def default_prefix
+        to_s.to_const_path
+      end
+
+      def expose(*meths)
+        @exposed ||= []
+        meths.each do |meth|
+          @exposed << meth unless @exposed.include?(meth)
+        end
+      end
+
+      def provides_for(prefix)
+        return [] unless @exposed
+        @exposed.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
+      end
+
+      def on_exception(proc = nil, &blk)
+        raise 'No callback provided for on_exception' unless proc || blk
+        @exception_callback = proc || blk
+      end
+
+      def exception_callback
+        @exception_callback
+      end
+      
+    end # ClassMethods     
+    
+    module InstanceMethods
+      # send nanite request to another agent (through the mapper)
+      def request(*args, &blk)
+        MapperProxy.instance.request(*args, &blk)
+      end
+    end # InstanceMethods
+    
+  end # Actor
+end # Nanite

data/lib/nanite/actor_registry.rb

@@ -0,0 +1,24 @@
+module Nanite
+  class ActorRegistry
+    attr_reader :actors
+
+    def initialize
+      @actors = {}
+    end
+
+    def register(actor, prefix)
+      raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
+      Nanite::Log.info("Registering #{actor.inspect} with prefix #{prefix.inspect}")
+      prefix ||= actor.class.default_prefix
+      actors[prefix.to_s] = actor
+    end
+
+    def services
+      actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
+    end
+
+    def actor_for(prefix)
+      actor = actors[prefix]
+    end
+  end # ActorRegistry
+end # Nanite 

data/lib/nanite/admin.rb

@@ -0,0 +1,138 @@
+require 'rack'
+
+module Nanite
+  # This is a Rack app for nanite-admin.  You need to have an async capable
+  # version of Thin installed for this to work.  See bin/nanite-admin for install
+  # instructions.
+  class Admin
+    def initialize(mapper)
+      @mapper = mapper
+    end
+
+    AsyncResponse = [-1, {}, []].freeze
+
+    def call(env)
+      req = Rack::Request.new(env)
+      if cmd = req.params['command']
+        @command = cmd
+        @selection = req.params['type'] if req.params['type']
+        
+        options = {}
+        case @selection
+        when 'least_loaded', 'random', 'all', 'rr'
+          options[:selector] = @selection
+        else
+          options[:target] = @selection
+        end
+
+        @mapper.request(cmd, req.params['payload'], options) do |response, responsejob|
+          env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]]
+        end
+        AsyncResponse
+      else
+        [200, {'Content-Type' => 'text/html'}, layout]
+      end
+    end
+
+    def services
+      buf = "<select name='command'>"
+      @mapper.cluster.nanites.all_services.each do |srv|
+        buf << "<option value='#{srv}' #{@command == srv ? 'selected="true"' : ''}>#{srv}</option>"
+      end
+      buf << "</select>"
+      buf
+    end
+
+    def ul(hash, job)
+      buf = "<ul>"
+      hash.each do |k,v|
+        buf << "<li><div class=\"nanite\">#{k}:</div><div class=\"response\">#{v.inspect}</div>"
+        if job.intermediate_state && job.intermediate_state[k]
+          buf << "<div class=\"intermediatestates\"><span class=\"statenote\">intermediate state:</span> #{job.intermediate_state[k].inspect}</div>"
+        end
+        buf << "</li>"
+      end
+      buf << "</ul>"
+      buf
+    end
+
+    def layout(content=nil)
+      %Q{
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns='http://www.w3.org/1999/xhtml'>
+          <head>
+            <meta content='text/html; charset=utf-8' http-equiv='Content-Type' />
+            <meta content='en' http-equiv='Content-Language' />
+            <meta content='Engineyard' name='author' />
+            <title>Nanite Control Tower</title>
+
+            <style>
+              body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
+              ul {margin: 0; padding: 0; margin-left: 10px}
+              li {list-style-type: none; margin-bottom: 6px}
+              li .nanite {font-weight: bold; font-size: 12px}
+              li .response {padding: 8px}
+              li .intermediatestates {padding: 8px; font-size: 10px;}
+              li .intermediatestates span.statenote {font-style: italic;}
+              h1, h2, h3 {margin-top: none; padding: none; margin-left: 40px;}
+              h1 {font-size: 22px; margin-top: 40px; margin-bottom: 30px; border-bottom: 1px solid #ddd; padding-bottom: 6px;
+                margin-right: 40px}
+              h2 {font-size: 16px;}
+              h3 {margin-left: 0; font-size: 14px}
+              .section {border: 1px solid #ccc; background-color: #fefefe; padding: 10px; margin: 20px 40px; padding: 20px;
+                font-size: 14px}
+              #footer {text-align: center; color: #AAA; font-size: 12px}
+            </style>
+
+          </head>
+
+          <body>
+            <div id="header">
+              <h1>Nanite Control Tower</h1>
+            </div>
+
+            <h2>#{@mapper.options[:vhost]}</h2>
+            <div class="section">
+              <form method="post" action="/">
+                <input type="hidden" value="POST" name="_method"/>
+
+                  <label>Send</label>
+                  <select name="type">
+                    <option #{@selection == 'least_loaded' ? 'selected="true"' : ''} value="least_loaded">the least loaded nanite</option>
+                    <option #{@selection == 'random' ? 'selected="true"' : ''} value="random">a random nanite</option>
+                    <option #{@selection == 'all' ? 'selected="true"' : ''} value="all">all nanites</option>
+                    <option #{@selection == 'rr' ? 'selected="true"' : ''} value="rr">a nanite chosen by round robin</option>
+                    #{@mapper.cluster.nanites.map {|k,v| "<option #{@selection == k ? 'selected="true"' : ''} value='#{k}'>#{k}</option>" }.join}
+                  </select>
+
+                  <label>providing service</label>
+                  #{services}
+
+                  <label>the payload</label>
+                  <input type="text" class="text" name="payload" id="payload"/>
+
+                  <input type="submit" class="submit" value="Go!" name="submit"/>
+              </form>
+
+              #{"<h3>Responses</h3>" if content}
+              #{content}
+            </div>
+
+            <h2>Running nanites</h2>
+            <div class="section">
+              #{"No nanites online." if @mapper.cluster.nanites.size == 0}
+              <ul>
+                #{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}<br />tags: #{v[:tags].to_a.inspect}</li>" }.join}
+              </ul>
+            </div>
+            <div id="footer">
+              Nanite #{Nanite::VERSION}
+              <br />
+              &copy; 2009 a bunch of random geeks
+            </div>
+          </body>
+        </html>
+      }
+    end
+  end
+end