shift-nanite 0.4.1.2

data/lib/nanite/security/signature.rb

@@ -0,0 +1,46 @@
+if defined?(OpenSSL::PKCS7::PKCS7)
+  Nanite::PKCS7 = OpenSSL::PKCS7::PKCS7
+else
+  Nanite::PKCS7 = OpenSSL::PKCS7
+end
+
+module Nanite
+
+  # Signature that can be validated against certificates
+  class Signature
+    
+    FLAGS = OpenSSL::PKCS7::NOCERTS || OpenSSL::PKCS7::BINARY || OpenSSL::PKCS7::NOATTR || OpenSSL::PKCS7::NOSMIMECAP || OpenSSL::PKCS7::DETACH
+
+    # Create signature using certificate and key pair.
+    #
+    # Arguments:
+    #  - 'data': Data to be signed
+    #  - 'cert': Certificate used for signature
+    #  - 'key':  RsaKeyPair used for signature
+    #
+    def initialize(data, cert, key)
+      @p7 = OpenSSL::PKCS7.sign(cert.raw_cert, key.raw_key, data, [], FLAGS)
+      @store = OpenSSL::X509::Store.new
+    end
+    
+    # Load signature previously serialized via 'data'
+    def self.from_data(data)
+      sig = Signature.allocate
+      sig.instance_variable_set(:@p7, Nanite::PKCS7.new(data))
+      sig.instance_variable_set(:@store, OpenSSL::X509::Store.new)
+      sig
+    end
+
+    # 'true' if signature was created using given cert, 'false' otherwise
+    def match?(cert)
+      @p7.verify([cert.raw_cert], @store, nil, OpenSSL::PKCS7::NOVERIFY)
+    end
+
+    # Signature in PEM format
+    def data
+      @p7.to_pem
+    end
+    alias :to_s :data
+
+  end
+end

data/lib/nanite/security/static_certificate_store.rb

@@ -0,0 +1,35 @@
+module Nanite
+
+  # Simple certificate store, serves a static set of certificates.
+  class StaticCertificateStore
+    
+    # Initialize store:
+    #
+    #  - Signer certificates are used when loading data to check the digital
+    #    signature. The signature associated with the serialized data needs
+    #    to match with one of the signer certificates for loading to succeed.
+    #
+    #  - Recipient certificates are used when serializing data for encryption.
+    #    Loading the data can only be done through serializers that have been
+    #    initialized with a certificate that's in the recipient certificates if
+    #    encryption is enabled.
+    #
+    def initialize(signer_certs, recipients_certs)
+      signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
+      @signer_certs = signer_certs 
+      recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
+      @recipients_certs = recipients_certs
+    end
+    
+    # Retrieve signer certificate for given id
+    def get_signer(identity)
+      @signer_certs
+    end
+
+    # Recipient certificate(s) that will be able to decrypt the serialized data
+    def get_recipients(obj)
+      @recipients_certs
+    end
+    
+  end  
+end

data/lib/nanite/security_provider.rb

@@ -0,0 +1,47 @@
+module Nanite
+  # This class is used to interface the nanite mapper with an external security
+  # module.
+  # There are two points of integration:
+  #  1. When an agent registers with a mapper
+  #  2. When an agent sends a request to another agent 
+  #
+  # In both these cases the security module is called back and can deny the 
+  # operation.
+  # Note: it's the responsability of the module to do any logging or
+  # notification that is required.
+  class SecurityProvider
+    
+    # Register an external security module
+    # This module should expose the 'authorize_registration' and 
+    # 'authorize_request' methods.
+    def self.register(mod)
+      @security_module = mod
+    end
+    
+    # Used internally by nanite to retrieve the current security module
+    def self.get
+      @security_module || default_security_module
+    end
+    
+    # Default security module, authorizes all operations
+    def self.default_security_module
+      @default_sec_mod ||= DefaultSecurityModule.new
+    end
+    
+  end
+  
+  # Default security module
+  class DefaultSecurityModule
+    
+    # Authorize registration of agent (registration is an instance of Register)
+    def authorize_registration(registration)
+      true
+    end
+    
+    # Authorize given inter-agent request (request is an instance of Request)
+    def authorize_request(request)
+      true
+    end
+    
+  end
+end

data/lib/nanite/serializer.rb

@@ -0,0 +1,52 @@
+module Nanite
+  class Serializer
+
+    class SerializationError < StandardError
+      attr_accessor :action, :packet
+      def initialize(action, packet, serializers, msg = nil)
+        @action, @packet = action, packet
+        msg = ":\n#{msg}" if msg && !msg.empty?
+        super("Could not #{action} #{packet.inspect} using #{serializers.inspect}#{msg}")
+      end
+    end # SerializationError
+
+    # The secure serializer should not be part of the cascading
+    def initialize(preferred_format = :marshal)
+      preferred_format ||= :marshal
+      if preferred_format.to_s == 'secure'
+        @serializers = [ SecureSerializer ]
+      else
+        preferred_serializer = SERIALIZERS[preferred_format.to_sym]
+        @serializers = SERIALIZERS.values.clone
+        @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
+      end
+    end
+
+    def dump(packet)
+      cascade_serializers(:dump, packet)
+    end
+
+    def load(packet)
+      cascade_serializers(:load, packet)
+    end
+
+    private
+
+    SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
+
+    def cascade_serializers(action, packet)
+      errors = []
+      @serializers.map do |serializer|
+        begin
+          o = serializer.send(action, packet)
+        rescue Exception => e
+          o = nil
+          errors << "#{e.message}\n\t#{e.backtrace[0]}"
+        end
+        return o if o
+      end
+      raise SerializationError.new(action, packet, @serializers, errors.join("\n"))
+    end
+
+  end # Serializer
+end # Nanite

data/lib/nanite/state.rb

@@ -0,0 +1,168 @@
+require 'redis'
+
+module Nanite
+  class State
+    include Enumerable
+    
+    # this class encapsulates the state of a nanite system using redis as the 
+    # data store. here is the schema, for each agent we store a number of items,
+    # for a nanite with the identity:  nanite-foobar we store the following things:
+    #
+    # nanite-foobar: 0.72 # load average or 'status'
+    # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
+    # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
+    # t-nanite-foobar: 123456789 # unix timestamp of the last state update
+    #
+    # also we do an inverted index for quick lookup of agents providing a certain
+    # service, so for each service the agent provides, we add the nanite to a SET 
+    # of all the nanites that provide said service:
+    #
+    # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    #
+    # we do that same thing for tags:
+    # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    #
+    # This way we can do a lookup of what nanites provide a set of services and tags based
+    # on redis SET intersection:
+    #
+    # nanites_for('/gems/list', 'some-tag')
+    # => returns a nested array of nanites and their state that provide the intersection
+    # of these two service tags
+    
+    def initialize(redis)
+      Nanite::Log.info("[setup] initializing redis state: #{redis}")
+      host, port = redis.split(':')
+      host ||= '127.0.0.1'
+      port ||= '6379'
+      @redis = Redis.new :host => host, :port => port
+    end
+    
+    def log_redis_error(meth,&blk)
+      blk.call
+    rescue Exception => e
+      Nanite::Log.info("redis error in method: #{meth}")
+      raise e
+    end
+    
+    def [](nanite)
+      log_redis_error("[]") do
+        status = @redis[nanite]
+        timestamp = @redis["t-#{nanite}"]
+        services = @redis.set_members("s-#{nanite}")
+        tags = @redis.set_members("tg-#{nanite}")
+        return nil unless status && timestamp && services
+        {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
+      end
+    end
+    
+    def []=(nanite, attributes)
+      log_redis_error("[]=") do
+        update_state(nanite, attributes[:status], attributes[:services], attributes[:tags])
+      end
+    end
+    
+    def delete(nanite)
+      log_redis_error("delete") do
+        (@redis.set_members("s-#{nanite}")||[]).each do |srv|
+          @redis.set_delete(srv, nanite)
+          if @redis.set_count(srv) == 0
+            @redis.delete(srv)
+            @redis.set_delete("naniteservices", srv)
+          end
+        end
+        (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
+          @redis.set_delete(tag, nanite)
+          if @redis.set_count(tag) == 0
+            @redis.delete(tag)
+            @redis.set_delete("nanitetags", tag)
+          end
+        end
+        @redis.delete nanite
+        @redis.delete "s-#{nanite}"
+        @redis.delete "t-#{nanite}"
+        @redis.delete "tg-#{nanite}"
+      end
+    end
+    
+    def all_services
+      log_redis_error("all_services") do
+        @redis.set_members("naniteservices")
+      end
+    end
+
+    def all_tags
+      log_redis_error("all_tags") do
+        @redis.set_members("nanitetags")
+      end
+    end
+    
+    def update_state(name, status, services, tags)
+      old_services = @redis.set_members("s-#{name}")
+      if old_services
+        (old_services - services).each do |s|
+          @redis.set_delete(s, name)
+          @redis.set_delete("naniteservices", s)
+        end
+      end
+      old_tags = @redis.set_members("tg-#{name}")
+      if old_tags
+        (old_tags - tags).each do |t|
+          @redis.set_delete(t, name)
+          @redis.set_delete("nanitetags", t)
+        end
+      end
+      @redis.delete("s-#{name}")
+      services.each do |srv|
+        @redis.set_add(srv, name)
+        @redis.set_add("s-#{name}", srv)
+        @redis.set_add("naniteservices", srv)
+      end
+      @redis.delete("tg-#{name}")
+      tags.each do |tag|
+        next if tag.nil?
+        @redis.set_add(tag, name)
+        @redis.set_add("tg-#{name}", tag)
+        @redis.set_add("nanitetags", tag)
+      end
+      update_status(name, status)
+    end
+
+    def update_status(name, status)
+      @redis[name] = status
+      @redis["t-#{name}"] = Time.now.utc.to_i
+    end
+    
+    def list_nanites
+      log_redis_error("list_nanites") do
+        @redis.keys("nanite-*")
+      end
+    end
+    
+    def size
+      list_nanites.size
+    end
+    
+    def clear_state
+      log_redis_error("clear_state") do
+        @redis.keys("*").each {|k| @redis.delete k}
+      end
+    end
+    
+    def each
+      list_nanites.each do |nan|
+        yield nan, self[nan]
+      end
+    end
+    
+    def nanites_for(service, *tags)
+      keys = tags.dup << service
+      log_redis_error("nanites_for") do
+        res = []
+        (@redis.set_intersect(keys)||[]).each do |nan|
+          res << [nan, self[nan]]
+        end
+        res
+      end
+    end
+  end
+end  

data/lib/nanite/streaming.rb

@@ -0,0 +1,125 @@
+module Nanite
+  # Nanite actors can transfer files to each other.
+  #
+  # ==== Options
+  #
+  # filename    : you guessed it, name of the file!
+  # domain      : part of the routing key used to locate receiver(s)
+  # destination : is a name of the file as it gonna be stored at the destination
+  # meta        :
+  #
+  # File streaming is done in chunks. When file streaming starts,
+  # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
+  # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
+  # Nanite::FileEnd packet is sent.
+  #
+  # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
+  # more than that in one operation via socket.
+  #
+  # ==== Domains
+  #
+  # Streaming happens using a topic exchange called 'file broadcast', with keys
+  # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
+  # receivers find each other in the cluster. Default domain is 'global'.
+  #
+  # Domains also serve as a way to register a callback Nanite agent executes once file
+  # streaming is completed. If a callback with name of domain is registered, it is called.
+  #
+  # Callbacks are registered by passing a block to subscribe_to_files method.
+  module FileStreaming
+    def broadcast_file(filename, options = {})
+      if File.exist?(filename)
+        File.open(filename, 'rb') do |file|
+           broadcast_data(filename, file, options)
+        end
+      else
+        return "file not found"
+      end
+    end
+
+    def broadcast_data(filename, io, options = {})
+      domain   = options[:domain] || 'global'
+      filename = File.basename(filename)
+      dest     = options[:destination] || filename
+      sent = 0
+
+        begin
+          file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
+          amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
+          res = Nanite::FileChunk.new(file_push.token)
+          while chunk = io.read(16384)
+            res.chunk = chunk
+            amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
+            sent += chunk.length
+          end
+          fend = Nanite::FileEnd.new(file_push.token, options[:meta])
+          amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
+          ""
+        ensure
+          io.close
+        end
+
+        sent
+    end
+
+    # FileState represents a file download in progress.
+    # It incapsulates the following information:
+    #
+    # * unique operation token
+    # * domain (namespace for file streaming operations)
+    # * file IO chunks are written to on receiver's side
+    class FileState
+
+      def initialize(token, dest, domain, write, blk)
+        @token = token
+        @cb = blk
+        @domain = domain
+        @write = write
+        
+        if write
+          @filename = File.join(Nanite.agent.file_root, dest)
+          @dest = File.open(@filename, 'wb')
+        else
+          @dest = dest
+        end
+
+        @data = ""
+      end
+
+      def handle_packet(packet)
+        case packet
+        when Nanite::FileChunk
+          Nanite::Log.debug "written chunk to #{@dest.inspect}"
+          @data << packet.chunk
+        
+          if @write
+            @dest.write(packet.chunk)
+          end
+        when Nanite::FileEnd
+          Nanite::Log.debug "#{@dest.inspect} receiving is completed"
+          if @write
+            @dest.close
+          end
+
+          @cb.call(@data, @dest, packet.meta)
+        end
+      end
+
+    end
+
+    def subscribe_to_files(domain='global', write=false, &blk)
+      Nanite::Log.info "subscribing to file broadcasts for #{domain}"
+      @files ||= {}
+      amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
+        case msg = serializer.load(packet)
+        when FileStart
+          @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
+        when FileChunk, FileEnd
+          if file = @files[msg.token]
+            file.handle_packet(msg)
+          end
+        end
+      end
+    end
+  end
+end