rightscale-nanite-dev 0.4.1.10

data/lib/nanite/security/rsa_key_pair.rb

@@ -0,0 +1,53 @@
+module Nanite
+
+  # Allows generating RSA key pairs and extracting public key component
+  # Note: Creating a RSA key pair can take a fair amount of time (seconds)
+  class RsaKeyPair
+    
+    DEFAULT_LENGTH = 2048
+
+    # Underlying OpenSSL keys
+    attr_reader :raw_key
+
+    # Create new RSA key pair using 'length' bits
+    def initialize(length = DEFAULT_LENGTH)
+      @raw_key = OpenSSL::PKey::RSA.generate(length)
+    end
+
+    # Does key pair include private key?
+    def has_private?
+      raw_key.private?
+    end
+    
+    # New RsaKeyPair instance with identical public key but no private key
+    def to_public
+      RsaKeyPair.from_data(raw_key.public_key.to_pem)
+    end
+    
+    # Key material in PEM format
+    def data
+      raw_key.to_pem
+    end
+    alias :to_s :data
+    
+    # Load key pair previously serialized via 'data'    
+    def self.from_data(data)
+      res = RsaKeyPair.allocate
+      res.instance_variable_set(:@raw_key, OpenSSL::PKey::RSA.new(data)) 
+      res
+    end
+
+    # Load key from file
+    def self.load(file)
+      from_data(File.read(file))
+    end
+    
+    # Save key to file in PEM format
+    def save(file)
+      File.open(file, "w") do |f|
+        f.write(@raw_key.to_pem)
+      end
+    end
+
+  end
+end

data/lib/nanite/security/secure_serializer.rb

@@ -0,0 +1,68 @@
+module Nanite
+  
+  # Serializer implementation which secures messages by using
+  # X.509 certificate sigining.
+  class SecureSerializer
+    
+    # Initialize serializer, must be called prior to using it.
+    #
+    #  - 'identity':   Identity associated with serialized messages
+    #  - 'cert':       Certificate used to sign and decrypt serialized messages
+    #  - 'key':        Private key corresponding to 'cert'
+    #  - 'store':      Certificate store. Exposes certificates used for
+    #                  encryption and signature validation.
+    #  - 'encrypt':    Whether data should be signed and encrypted ('true')
+    #                  or just signed ('false'), 'true' by default.
+    #
+    def self.init(identity, cert, key, store, encrypt = true)
+      @identity = identity
+      @cert = cert
+      @key = key
+      @store = store
+      @encrypt = encrypt
+    end
+    
+    # Was serializer initialized?
+    def self.initialized?
+      @identity && @cert && @key && @store
+    end
+
+    # Serialize message and sign it using X.509 certificate
+    def self.dump(obj)
+      raise "Missing certificate identity" unless @identity
+      raise "Missing certificate" unless @cert
+      raise "Missing certificate key" unless @key
+      raise "Missing certificate store" unless @store || !@encrypt
+      json = obj.to_json
+      if @encrypt
+        certs = @store.get_recipients(obj)
+        json = EncryptedDocument.new(json, certs).encrypted_data if certs
+      end
+      sig = Signature.new(json, @cert, @key)
+      { 'id' => @identity, 'data' => json, 'signature' => sig.data, 'encrypted' => !certs.nil? }.to_json
+    end
+    
+    # Unserialize data using certificate store
+    def self.load(json)
+      begin
+        raise "Missing certificate store" unless @store
+        raise "Missing certificate" unless @cert || !@encrypt
+        raise "Missing certificate key" unless @key || !@encrypt
+        data = JSON.load(json)
+        sig = Signature.from_data(data['signature'])
+        certs = @store.get_signer(data['id'])
+        raise "Could not find a cert for signer #{data['id']}" unless certs
+        certs = [ certs ] unless certs.respond_to?(:each)
+        jsn = data['data'] if certs.any? { |c| sig.match?(c) }
+        if jsn && @encrypt && data['encrypted']
+          jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert)
+        end
+        JSON.load(jsn) if jsn
+      rescue Exception => e
+        Nanite::Log.error("Loading of secure packet failed: #{e.message}\n#{e.backtrace.join("\n")}")
+        raise
+      end
+    end
+       
+  end
+end

data/lib/nanite/security/signature.rb

@@ -0,0 +1,46 @@
+if defined?(OpenSSL::PKCS7::PKCS7)
+  Nanite::PKCS7 = OpenSSL::PKCS7::PKCS7
+else
+  Nanite::PKCS7 = OpenSSL::PKCS7
+end
+
+module Nanite
+
+  # Signature that can be validated against certificates
+  class Signature
+    
+    FLAGS = OpenSSL::PKCS7::NOCERTS || OpenSSL::PKCS7::BINARY || OpenSSL::PKCS7::NOATTR || OpenSSL::PKCS7::NOSMIMECAP || OpenSSL::PKCS7::DETACH
+
+    # Create signature using certificate and key pair.
+    #
+    # Arguments:
+    #  - 'data': Data to be signed
+    #  - 'cert': Certificate used for signature
+    #  - 'key':  RsaKeyPair used for signature
+    #
+    def initialize(data, cert, key)
+      @p7 = OpenSSL::PKCS7.sign(cert.raw_cert, key.raw_key, data, [], FLAGS)
+      @store = OpenSSL::X509::Store.new
+    end
+    
+    # Load signature previously serialized via 'data'
+    def self.from_data(data)
+      sig = Signature.allocate
+      sig.instance_variable_set(:@p7, Nanite::PKCS7.new(data))
+      sig.instance_variable_set(:@store, OpenSSL::X509::Store.new)
+      sig
+    end
+
+    # 'true' if signature was created using given cert, 'false' otherwise
+    def match?(cert)
+      @p7.verify([cert.raw_cert], @store, nil, OpenSSL::PKCS7::NOVERIFY)
+    end
+
+    # Signature in PEM format
+    def data
+      @p7.to_pem
+    end
+    alias :to_s :data
+
+  end
+end

data/lib/nanite/security/static_certificate_store.rb

@@ -0,0 +1,35 @@
+module Nanite
+
+  # Simple certificate store, serves a static set of certificates.
+  class StaticCertificateStore
+    
+    # Initialize store:
+    #
+    #  - Signer certificates are used when loading data to check the digital
+    #    signature. The signature associated with the serialized data needs
+    #    to match with one of the signer certificates for loading to succeed.
+    #
+    #  - Recipient certificates are used when serializing data for encryption.
+    #    Loading the data can only be done through serializers that have been
+    #    initialized with a certificate that's in the recipient certificates if
+    #    encryption is enabled.
+    #
+    def initialize(signer_certs, recipients_certs)
+      signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
+      @signer_certs = signer_certs 
+      recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
+      @recipients_certs = recipients_certs
+    end
+    
+    # Retrieve signer certificate for given id
+    def get_signer(identity)
+      @signer_certs
+    end
+
+    # Recipient certificate(s) that will be able to decrypt the serialized data
+    def get_recipients(obj)
+      @recipients_certs
+    end
+    
+  end  
+end

data/lib/nanite/security_provider.rb

@@ -0,0 +1,47 @@
+module Nanite
+  # This class is used to interface the nanite mapper with an external security
+  # module.
+  # There are two points of integration:
+  #  1. When an agent registers with a mapper
+  #  2. When an agent sends a request to another agent 
+  #
+  # In both these cases the security module is called back and can deny the 
+  # operation.
+  # Note: it's the responsability of the module to do any logging or
+  # notification that is required.
+  class SecurityProvider
+    
+    # Register an external security module
+    # This module should expose the 'authorize_registration' and 
+    # 'authorize_request' methods.
+    def self.register(mod)
+      @security_module = mod
+    end
+    
+    # Used internally by nanite to retrieve the current security module
+    def self.get
+      @security_module || default_security_module
+    end
+    
+    # Default security module, authorizes all operations
+    def self.default_security_module
+      @default_sec_mod ||= DefaultSecurityModule.new
+    end
+    
+  end
+  
+  # Default security module
+  class DefaultSecurityModule
+    
+    # Authorize registration of agent (registration is an instance of Register)
+    def authorize_registration(registration)
+      true
+    end
+    
+    # Authorize given inter-agent request (request is an instance of Request)
+    def authorize_request(request)
+      true
+    end
+    
+  end
+end

data/lib/nanite/serializer.rb

@@ -0,0 +1,52 @@
+module Nanite
+  class Serializer
+
+    class SerializationError < StandardError
+      attr_accessor :action, :packet
+      def initialize(action, packet, serializers, msg = nil)
+        @action, @packet = action, packet
+        msg = ":\n#{msg}" if msg && !msg.empty?
+        super("Could not #{action} #{packet.inspect} using #{serializers.inspect}#{msg}")
+      end
+    end # SerializationError
+
+    # The secure serializer should not be part of the cascading
+    def initialize(preferred_format = :marshal)
+      preferred_format ||= :marshal
+      if preferred_format.to_s == 'secure'
+        @serializers = [ SecureSerializer ]
+      else
+        preferred_serializer = SERIALIZERS[preferred_format.to_sym]
+        @serializers = SERIALIZERS.values.clone
+        @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
+      end
+    end
+
+    def dump(packet)
+      cascade_serializers(:dump, packet)
+    end
+
+    def load(packet)
+      cascade_serializers(:load, packet)
+    end
+
+    private
+
+    SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
+
+    def cascade_serializers(action, packet)
+      errors = []
+      @serializers.map do |serializer|
+        begin
+          o = serializer.send(action, packet)
+        rescue Exception => e
+          o = nil
+          errors << "#{e.message}\n\t#{e.backtrace[0]}"
+        end
+        return o if o
+      end
+      raise SerializationError.new(action, packet, @serializers, errors.join("\n"))
+    end
+
+  end # Serializer
+end # Nanite

data/lib/nanite/state.rb

@@ -0,0 +1,135 @@
+require 'redis'
+require 'redis_tag_store'
+
+module Nanite
+  class State
+    include Enumerable
+    
+    # This class encapsulates the state of a nanite system using redis as the 
+    # data store and a provided tag store. For a nanite with the identity
+    # 'nanite-foobar' we store the following:
+    #
+    # nanite-foobar: 0.72        # load average or 'status'
+    # t-nanite-foobar: 123456789 # unix timestamp of the last state update
+    #
+    # The tag store is used to store the associated services and tags.
+    #
+    # A tag store should provide the following methods:
+    #  - initialize(redis): Initialize tag store, may use provided redis handle
+    #  - services(nanite): Retrieve services implemented by given agent
+    #  - tags(nanite): Retrieve tags implemented by given agent
+    #  - all_services: Retrieve all services implemented by all agents
+    #  - all_tags: Retrieve all tags exposed by all agents
+    #  - store(nanite, services, tags): Store agent's services and tags
+    #  - update(name, new_tags,obsolete_tags): Update agent's tags
+    #  - delete(nanite): Delete all entries associated with given agent
+    #  - nanites_for(service, tags): Retrieve agents implementing given service
+    #                                and exposing given tags
+    #
+    # The default implementation for the tag store reuses Redis.
+  
+    def initialize(redis, tag_store=nil)
+      host, port, tag_store_type = redis.split(':')
+      host ||= '127.0.0.1'
+      port ||= '6379'
+      tag_store||= 'Nanite::RedisTagStore'
+      @redis = Redis.new(:host => host, :port => port)
+      @tag_store = tag_store.to_const.new(@redis)
+      Nanite::Log.info("[setup] Initializing redis state using host '#{host}', port '#{port}' and tag store #{tag_store}")
+    end
+
+    # Retrieve given agent services, tags, status and timestamp
+    def [](nanite)
+      log_redis_error do
+        status    = @redis[nanite]
+        timestamp = @redis["t-#{nanite}"]
+        services  = @tag_store.services(nanite)
+        tags      = @tag_store.tags(nanite)
+        return nil unless status && timestamp && services
+        {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
+      end
+    end
+
+    # Set given attributes for given agent
+    # Attributes may include services, tags and status
+    def []=(nanite, attributes)
+      @tag_store.store(nanite, attributes[:services], attributes[:tags])
+      update_status(nanite, attributes[:status])
+    end
+
+    # Delete all information related to given agent
+    def delete(nanite)
+      @tag_store.delete(nanite)
+      log_redis_error do
+        @redis.delete(nanite)
+        @redis.delete("t-#{nanite}")
+      end
+    end
+
+    # Return all services exposed by all agents
+    def all_services
+      @tag_store.all_services
+    end
+
+    # Return all tags exposed by all agents
+    def all_tags
+      @tag_store.all_tags
+    end
+
+    # Update status and timestamp for given agent
+    def update_status(name, status)
+      log_redis_error do
+        @redis[name] = status
+        @redis["t-#{name}"] = Time.now.utc.to_i
+      end
+    end
+
+    # Update tags for given agent
+    def update_tags(name, new_tags, obsolete_tags)
+      @tag_store.update(name, new_tags, obsolete_tags)
+    end
+
+    # Return all registered agents
+    def list_nanites
+      log_redis_error do
+        @redis.keys("nanite-*")
+      end
+    end
+
+    # Number of registered agents
+    def size
+      list_nanites.size
+    end
+
+    # Iterate through all agents, yielding services, tags
+    # status and timestamp keyed by agent name
+    def each
+      list_nanites.each do |nan|
+        yield nan, self[nan]
+      end
+    end
+
+    # Return agents that implement given service and expose
+    # all given tags
+    def nanites_for(from, service, tags)
+      res = []
+      @tag_store.nanites_for(from, service, tags).each do |nanite_id|
+        if nanite = self[nanite_id]
+          res << [nanite_id, nanite]
+        end
+      end
+      res
+    end
+
+    private
+
+    # Helper method, catch and log errors
+    def log_redis_error(&blk)
+      blk.call
+    rescue Exception => e
+      Nanite::Log.warn("redis error in method: #{caller[0]}")
+      raise e
+    end
+    
+  end
+end  

data/lib/nanite/streaming.rb

@@ -0,0 +1,125 @@
+module Nanite
+  # Nanite actors can transfer files to each other.
+  #
+  # ==== Options
+  #
+  # filename    : you guessed it, name of the file!
+  # domain      : part of the routing key used to locate receiver(s)
+  # destination : is a name of the file as it gonna be stored at the destination
+  # meta        :
+  #
+  # File streaming is done in chunks. When file streaming starts,
+  # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
+  # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
+  # Nanite::FileEnd packet is sent.
+  #
+  # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
+  # more than that in one operation via socket.
+  #
+  # ==== Domains
+  #
+  # Streaming happens using a topic exchange called 'file broadcast', with keys
+  # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
+  # receivers find each other in the cluster. Default domain is 'global'.
+  #
+  # Domains also serve as a way to register a callback Nanite agent executes once file
+  # streaming is completed. If a callback with name of domain is registered, it is called.
+  #
+  # Callbacks are registered by passing a block to subscribe_to_files method.
+  module FileStreaming
+    def broadcast_file(filename, options = {})
+      if File.exist?(filename)
+        File.open(filename, 'rb') do |file|
+           broadcast_data(filename, file, options)
+        end
+      else
+        return "file not found"
+      end
+    end
+
+    def broadcast_data(filename, io, options = {})
+      domain   = options[:domain] || 'global'
+      filename = File.basename(filename)
+      dest     = options[:destination] || filename
+      sent = 0
+
+        begin
+          file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
+          amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
+          res = Nanite::FileChunk.new(file_push.token)
+          while chunk = io.read(16384)
+            res.chunk = chunk
+            amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
+            sent += chunk.length
+          end
+          fend = Nanite::FileEnd.new(file_push.token, options[:meta])
+          amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
+          ""
+        ensure
+          io.close
+        end
+
+        sent
+    end
+
+    # FileState represents a file download in progress.
+    # It incapsulates the following information:
+    #
+    # * unique operation token
+    # * domain (namespace for file streaming operations)
+    # * file IO chunks are written to on receiver's side
+    class FileState
+
+      def initialize(token, dest, domain, write, blk)
+        @token = token
+        @cb = blk
+        @domain = domain
+        @write = write
+        
+        if write
+          @filename = File.join(Nanite.agent.file_root, dest)
+          @dest = File.open(@filename, 'wb')
+        else
+          @dest = dest
+        end
+
+        @data = ""
+      end
+
+      def handle_packet(packet)
+        case packet
+        when Nanite::FileChunk
+          Nanite::Log.debug "written chunk to #{@dest.inspect}"
+          @data << packet.chunk
+        
+          if @write
+            @dest.write(packet.chunk)
+          end
+        when Nanite::FileEnd
+          Nanite::Log.debug "#{@dest.inspect} receiving is completed"
+          if @write
+            @dest.close
+          end
+
+          @cb.call(@data, @dest, packet.meta)
+        end
+      end
+
+    end
+
+    def subscribe_to_files(domain='global', write=false, &blk)
+      Nanite::Log.info "subscribing to file broadcasts for #{domain}"
+      @files ||= {}
+      amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
+        case msg = serializer.load(packet)
+        when FileStart
+          @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
+        when FileChunk, FileEnd
+          if file = @files[msg.token]
+            file.handle_packet(msg)
+          end
+        end
+      end
+    end
+  end
+end