rightscale-nanite-dev 0.4.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +78 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +274 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +256 -0
  15. data/lib/nanite/config.rb +111 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +315 -0
  24. data/lib/nanite/mapper_proxy.rb +75 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +401 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/redis_tag_store.rb +141 -0
  30. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  31. data/lib/nanite/security/certificate.rb +55 -0
  32. data/lib/nanite/security/certificate_cache.rb +66 -0
  33. data/lib/nanite/security/distinguished_name.rb +34 -0
  34. data/lib/nanite/security/encrypted_document.rb +46 -0
  35. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  36. data/lib/nanite/security/secure_serializer.rb +68 -0
  37. data/lib/nanite/security/signature.rb +46 -0
  38. data/lib/nanite/security/static_certificate_store.rb +35 -0
  39. data/lib/nanite/security_provider.rb +47 -0
  40. data/lib/nanite/serializer.rb +52 -0
  41. data/lib/nanite/state.rb +135 -0
  42. data/lib/nanite/streaming.rb +125 -0
  43. data/lib/nanite/util.rb +78 -0
  44. metadata +111 -0
@@ -0,0 +1,53 @@
1
+ module Nanite
2
+
3
+ # Allows generating RSA key pairs and extracting public key component
4
+ # Note: Creating a RSA key pair can take a fair amount of time (seconds)
5
+ class RsaKeyPair
6
+
7
+ DEFAULT_LENGTH = 2048
8
+
9
+ # Underlying OpenSSL keys
10
+ attr_reader :raw_key
11
+
12
+ # Create new RSA key pair using 'length' bits
13
+ def initialize(length = DEFAULT_LENGTH)
14
+ @raw_key = OpenSSL::PKey::RSA.generate(length)
15
+ end
16
+
17
+ # Does key pair include private key?
18
+ def has_private?
19
+ raw_key.private?
20
+ end
21
+
22
+ # New RsaKeyPair instance with identical public key but no private key
23
+ def to_public
24
+ RsaKeyPair.from_data(raw_key.public_key.to_pem)
25
+ end
26
+
27
+ # Key material in PEM format
28
+ def data
29
+ raw_key.to_pem
30
+ end
31
+ alias :to_s :data
32
+
33
+ # Load key pair previously serialized via 'data'
34
+ def self.from_data(data)
35
+ res = RsaKeyPair.allocate
36
+ res.instance_variable_set(:@raw_key, OpenSSL::PKey::RSA.new(data))
37
+ res
38
+ end
39
+
40
+ # Load key from file
41
+ def self.load(file)
42
+ from_data(File.read(file))
43
+ end
44
+
45
+ # Save key to file in PEM format
46
+ def save(file)
47
+ File.open(file, "w") do |f|
48
+ f.write(@raw_key.to_pem)
49
+ end
50
+ end
51
+
52
+ end
53
+ end
@@ -0,0 +1,68 @@
1
+ module Nanite
2
+
3
+ # Serializer implementation which secures messages by using
4
+ # X.509 certificate sigining.
5
+ class SecureSerializer
6
+
7
+ # Initialize serializer, must be called prior to using it.
8
+ #
9
+ # - 'identity': Identity associated with serialized messages
10
+ # - 'cert': Certificate used to sign and decrypt serialized messages
11
+ # - 'key': Private key corresponding to 'cert'
12
+ # - 'store': Certificate store. Exposes certificates used for
13
+ # encryption and signature validation.
14
+ # - 'encrypt': Whether data should be signed and encrypted ('true')
15
+ # or just signed ('false'), 'true' by default.
16
+ #
17
+ def self.init(identity, cert, key, store, encrypt = true)
18
+ @identity = identity
19
+ @cert = cert
20
+ @key = key
21
+ @store = store
22
+ @encrypt = encrypt
23
+ end
24
+
25
+ # Was serializer initialized?
26
+ def self.initialized?
27
+ @identity && @cert && @key && @store
28
+ end
29
+
30
+ # Serialize message and sign it using X.509 certificate
31
+ def self.dump(obj)
32
+ raise "Missing certificate identity" unless @identity
33
+ raise "Missing certificate" unless @cert
34
+ raise "Missing certificate key" unless @key
35
+ raise "Missing certificate store" unless @store || !@encrypt
36
+ json = obj.to_json
37
+ if @encrypt
38
+ certs = @store.get_recipients(obj)
39
+ json = EncryptedDocument.new(json, certs).encrypted_data if certs
40
+ end
41
+ sig = Signature.new(json, @cert, @key)
42
+ { 'id' => @identity, 'data' => json, 'signature' => sig.data, 'encrypted' => !certs.nil? }.to_json
43
+ end
44
+
45
+ # Unserialize data using certificate store
46
+ def self.load(json)
47
+ begin
48
+ raise "Missing certificate store" unless @store
49
+ raise "Missing certificate" unless @cert || !@encrypt
50
+ raise "Missing certificate key" unless @key || !@encrypt
51
+ data = JSON.load(json)
52
+ sig = Signature.from_data(data['signature'])
53
+ certs = @store.get_signer(data['id'])
54
+ raise "Could not find a cert for signer #{data['id']}" unless certs
55
+ certs = [ certs ] unless certs.respond_to?(:each)
56
+ jsn = data['data'] if certs.any? { |c| sig.match?(c) }
57
+ if jsn && @encrypt && data['encrypted']
58
+ jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert)
59
+ end
60
+ JSON.load(jsn) if jsn
61
+ rescue Exception => e
62
+ Nanite::Log.error("Loading of secure packet failed: #{e.message}\n#{e.backtrace.join("\n")}")
63
+ raise
64
+ end
65
+ end
66
+
67
+ end
68
+ end
@@ -0,0 +1,46 @@
1
+ if defined?(OpenSSL::PKCS7::PKCS7)
2
+ Nanite::PKCS7 = OpenSSL::PKCS7::PKCS7
3
+ else
4
+ Nanite::PKCS7 = OpenSSL::PKCS7
5
+ end
6
+
7
+ module Nanite
8
+
9
+ # Signature that can be validated against certificates
10
+ class Signature
11
+
12
+ FLAGS = OpenSSL::PKCS7::NOCERTS || OpenSSL::PKCS7::BINARY || OpenSSL::PKCS7::NOATTR || OpenSSL::PKCS7::NOSMIMECAP || OpenSSL::PKCS7::DETACH
13
+
14
+ # Create signature using certificate and key pair.
15
+ #
16
+ # Arguments:
17
+ # - 'data': Data to be signed
18
+ # - 'cert': Certificate used for signature
19
+ # - 'key': RsaKeyPair used for signature
20
+ #
21
+ def initialize(data, cert, key)
22
+ @p7 = OpenSSL::PKCS7.sign(cert.raw_cert, key.raw_key, data, [], FLAGS)
23
+ @store = OpenSSL::X509::Store.new
24
+ end
25
+
26
+ # Load signature previously serialized via 'data'
27
+ def self.from_data(data)
28
+ sig = Signature.allocate
29
+ sig.instance_variable_set(:@p7, Nanite::PKCS7.new(data))
30
+ sig.instance_variable_set(:@store, OpenSSL::X509::Store.new)
31
+ sig
32
+ end
33
+
34
+ # 'true' if signature was created using given cert, 'false' otherwise
35
+ def match?(cert)
36
+ @p7.verify([cert.raw_cert], @store, nil, OpenSSL::PKCS7::NOVERIFY)
37
+ end
38
+
39
+ # Signature in PEM format
40
+ def data
41
+ @p7.to_pem
42
+ end
43
+ alias :to_s :data
44
+
45
+ end
46
+ end
@@ -0,0 +1,35 @@
1
+ module Nanite
2
+
3
+ # Simple certificate store, serves a static set of certificates.
4
+ class StaticCertificateStore
5
+
6
+ # Initialize store:
7
+ #
8
+ # - Signer certificates are used when loading data to check the digital
9
+ # signature. The signature associated with the serialized data needs
10
+ # to match with one of the signer certificates for loading to succeed.
11
+ #
12
+ # - Recipient certificates are used when serializing data for encryption.
13
+ # Loading the data can only be done through serializers that have been
14
+ # initialized with a certificate that's in the recipient certificates if
15
+ # encryption is enabled.
16
+ #
17
+ def initialize(signer_certs, recipients_certs)
18
+ signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
19
+ @signer_certs = signer_certs
20
+ recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
21
+ @recipients_certs = recipients_certs
22
+ end
23
+
24
+ # Retrieve signer certificate for given id
25
+ def get_signer(identity)
26
+ @signer_certs
27
+ end
28
+
29
+ # Recipient certificate(s) that will be able to decrypt the serialized data
30
+ def get_recipients(obj)
31
+ @recipients_certs
32
+ end
33
+
34
+ end
35
+ end
@@ -0,0 +1,47 @@
1
+ module Nanite
2
+ # This class is used to interface the nanite mapper with an external security
3
+ # module.
4
+ # There are two points of integration:
5
+ # 1. When an agent registers with a mapper
6
+ # 2. When an agent sends a request to another agent
7
+ #
8
+ # In both these cases the security module is called back and can deny the
9
+ # operation.
10
+ # Note: it's the responsability of the module to do any logging or
11
+ # notification that is required.
12
+ class SecurityProvider
13
+
14
+ # Register an external security module
15
+ # This module should expose the 'authorize_registration' and
16
+ # 'authorize_request' methods.
17
+ def self.register(mod)
18
+ @security_module = mod
19
+ end
20
+
21
+ # Used internally by nanite to retrieve the current security module
22
+ def self.get
23
+ @security_module || default_security_module
24
+ end
25
+
26
+ # Default security module, authorizes all operations
27
+ def self.default_security_module
28
+ @default_sec_mod ||= DefaultSecurityModule.new
29
+ end
30
+
31
+ end
32
+
33
+ # Default security module
34
+ class DefaultSecurityModule
35
+
36
+ # Authorize registration of agent (registration is an instance of Register)
37
+ def authorize_registration(registration)
38
+ true
39
+ end
40
+
41
+ # Authorize given inter-agent request (request is an instance of Request)
42
+ def authorize_request(request)
43
+ true
44
+ end
45
+
46
+ end
47
+ end
@@ -0,0 +1,52 @@
1
+ module Nanite
2
+ class Serializer
3
+
4
+ class SerializationError < StandardError
5
+ attr_accessor :action, :packet
6
+ def initialize(action, packet, serializers, msg = nil)
7
+ @action, @packet = action, packet
8
+ msg = ":\n#{msg}" if msg && !msg.empty?
9
+ super("Could not #{action} #{packet.inspect} using #{serializers.inspect}#{msg}")
10
+ end
11
+ end # SerializationError
12
+
13
+ # The secure serializer should not be part of the cascading
14
+ def initialize(preferred_format = :marshal)
15
+ preferred_format ||= :marshal
16
+ if preferred_format.to_s == 'secure'
17
+ @serializers = [ SecureSerializer ]
18
+ else
19
+ preferred_serializer = SERIALIZERS[preferred_format.to_sym]
20
+ @serializers = SERIALIZERS.values.clone
21
+ @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
22
+ end
23
+ end
24
+
25
+ def dump(packet)
26
+ cascade_serializers(:dump, packet)
27
+ end
28
+
29
+ def load(packet)
30
+ cascade_serializers(:load, packet)
31
+ end
32
+
33
+ private
34
+
35
+ SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
36
+
37
+ def cascade_serializers(action, packet)
38
+ errors = []
39
+ @serializers.map do |serializer|
40
+ begin
41
+ o = serializer.send(action, packet)
42
+ rescue Exception => e
43
+ o = nil
44
+ errors << "#{e.message}\n\t#{e.backtrace[0]}"
45
+ end
46
+ return o if o
47
+ end
48
+ raise SerializationError.new(action, packet, @serializers, errors.join("\n"))
49
+ end
50
+
51
+ end # Serializer
52
+ end # Nanite
@@ -0,0 +1,135 @@
1
+ require 'redis'
2
+ require 'redis_tag_store'
3
+
4
+ module Nanite
5
+ class State
6
+ include Enumerable
7
+
8
+ # This class encapsulates the state of a nanite system using redis as the
9
+ # data store and a provided tag store. For a nanite with the identity
10
+ # 'nanite-foobar' we store the following:
11
+ #
12
+ # nanite-foobar: 0.72 # load average or 'status'
13
+ # t-nanite-foobar: 123456789 # unix timestamp of the last state update
14
+ #
15
+ # The tag store is used to store the associated services and tags.
16
+ #
17
+ # A tag store should provide the following methods:
18
+ # - initialize(redis): Initialize tag store, may use provided redis handle
19
+ # - services(nanite): Retrieve services implemented by given agent
20
+ # - tags(nanite): Retrieve tags implemented by given agent
21
+ # - all_services: Retrieve all services implemented by all agents
22
+ # - all_tags: Retrieve all tags exposed by all agents
23
+ # - store(nanite, services, tags): Store agent's services and tags
24
+ # - update(name, new_tags,obsolete_tags): Update agent's tags
25
+ # - delete(nanite): Delete all entries associated with given agent
26
+ # - nanites_for(service, tags): Retrieve agents implementing given service
27
+ # and exposing given tags
28
+ #
29
+ # The default implementation for the tag store reuses Redis.
30
+
31
+ def initialize(redis, tag_store=nil)
32
+ host, port, tag_store_type = redis.split(':')
33
+ host ||= '127.0.0.1'
34
+ port ||= '6379'
35
+ tag_store||= 'Nanite::RedisTagStore'
36
+ @redis = Redis.new(:host => host, :port => port)
37
+ @tag_store = tag_store.to_const.new(@redis)
38
+ Nanite::Log.info("[setup] Initializing redis state using host '#{host}', port '#{port}' and tag store #{tag_store}")
39
+ end
40
+
41
+ # Retrieve given agent services, tags, status and timestamp
42
+ def [](nanite)
43
+ log_redis_error do
44
+ status = @redis[nanite]
45
+ timestamp = @redis["t-#{nanite}"]
46
+ services = @tag_store.services(nanite)
47
+ tags = @tag_store.tags(nanite)
48
+ return nil unless status && timestamp && services
49
+ {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
50
+ end
51
+ end
52
+
53
+ # Set given attributes for given agent
54
+ # Attributes may include services, tags and status
55
+ def []=(nanite, attributes)
56
+ @tag_store.store(nanite, attributes[:services], attributes[:tags])
57
+ update_status(nanite, attributes[:status])
58
+ end
59
+
60
+ # Delete all information related to given agent
61
+ def delete(nanite)
62
+ @tag_store.delete(nanite)
63
+ log_redis_error do
64
+ @redis.delete(nanite)
65
+ @redis.delete("t-#{nanite}")
66
+ end
67
+ end
68
+
69
+ # Return all services exposed by all agents
70
+ def all_services
71
+ @tag_store.all_services
72
+ end
73
+
74
+ # Return all tags exposed by all agents
75
+ def all_tags
76
+ @tag_store.all_tags
77
+ end
78
+
79
+ # Update status and timestamp for given agent
80
+ def update_status(name, status)
81
+ log_redis_error do
82
+ @redis[name] = status
83
+ @redis["t-#{name}"] = Time.now.utc.to_i
84
+ end
85
+ end
86
+
87
+ # Update tags for given agent
88
+ def update_tags(name, new_tags, obsolete_tags)
89
+ @tag_store.update(name, new_tags, obsolete_tags)
90
+ end
91
+
92
+ # Return all registered agents
93
+ def list_nanites
94
+ log_redis_error do
95
+ @redis.keys("nanite-*")
96
+ end
97
+ end
98
+
99
+ # Number of registered agents
100
+ def size
101
+ list_nanites.size
102
+ end
103
+
104
+ # Iterate through all agents, yielding services, tags
105
+ # status and timestamp keyed by agent name
106
+ def each
107
+ list_nanites.each do |nan|
108
+ yield nan, self[nan]
109
+ end
110
+ end
111
+
112
+ # Return agents that implement given service and expose
113
+ # all given tags
114
+ def nanites_for(from, service, tags)
115
+ res = []
116
+ @tag_store.nanites_for(from, service, tags).each do |nanite_id|
117
+ if nanite = self[nanite_id]
118
+ res << [nanite_id, nanite]
119
+ end
120
+ end
121
+ res
122
+ end
123
+
124
+ private
125
+
126
+ # Helper method, catch and log errors
127
+ def log_redis_error(&blk)
128
+ blk.call
129
+ rescue Exception => e
130
+ Nanite::Log.warn("redis error in method: #{caller[0]}")
131
+ raise e
132
+ end
133
+
134
+ end
135
+ end
@@ -0,0 +1,125 @@
1
+ module Nanite
2
+ # Nanite actors can transfer files to each other.
3
+ #
4
+ # ==== Options
5
+ #
6
+ # filename : you guessed it, name of the file!
7
+ # domain : part of the routing key used to locate receiver(s)
8
+ # destination : is a name of the file as it gonna be stored at the destination
9
+ # meta :
10
+ #
11
+ # File streaming is done in chunks. When file streaming starts,
12
+ # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
13
+ # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
14
+ # Nanite::FileEnd packet is sent.
15
+ #
16
+ # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
17
+ # more than that in one operation via socket.
18
+ #
19
+ # ==== Domains
20
+ #
21
+ # Streaming happens using a topic exchange called 'file broadcast', with keys
22
+ # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
23
+ # receivers find each other in the cluster. Default domain is 'global'.
24
+ #
25
+ # Domains also serve as a way to register a callback Nanite agent executes once file
26
+ # streaming is completed. If a callback with name of domain is registered, it is called.
27
+ #
28
+ # Callbacks are registered by passing a block to subscribe_to_files method.
29
+ module FileStreaming
30
+ def broadcast_file(filename, options = {})
31
+ if File.exist?(filename)
32
+ File.open(filename, 'rb') do |file|
33
+ broadcast_data(filename, file, options)
34
+ end
35
+ else
36
+ return "file not found"
37
+ end
38
+ end
39
+
40
+ def broadcast_data(filename, io, options = {})
41
+ domain = options[:domain] || 'global'
42
+ filename = File.basename(filename)
43
+ dest = options[:destination] || filename
44
+ sent = 0
45
+
46
+ begin
47
+ file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
48
+ amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
49
+ res = Nanite::FileChunk.new(file_push.token)
50
+ while chunk = io.read(16384)
51
+ res.chunk = chunk
52
+ amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
53
+ sent += chunk.length
54
+ end
55
+ fend = Nanite::FileEnd.new(file_push.token, options[:meta])
56
+ amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
57
+ ""
58
+ ensure
59
+ io.close
60
+ end
61
+
62
+ sent
63
+ end
64
+
65
+ # FileState represents a file download in progress.
66
+ # It incapsulates the following information:
67
+ #
68
+ # * unique operation token
69
+ # * domain (namespace for file streaming operations)
70
+ # * file IO chunks are written to on receiver's side
71
+ class FileState
72
+
73
+ def initialize(token, dest, domain, write, blk)
74
+ @token = token
75
+ @cb = blk
76
+ @domain = domain
77
+ @write = write
78
+
79
+ if write
80
+ @filename = File.join(Nanite.agent.file_root, dest)
81
+ @dest = File.open(@filename, 'wb')
82
+ else
83
+ @dest = dest
84
+ end
85
+
86
+ @data = ""
87
+ end
88
+
89
+ def handle_packet(packet)
90
+ case packet
91
+ when Nanite::FileChunk
92
+ Nanite::Log.debug "written chunk to #{@dest.inspect}"
93
+ @data << packet.chunk
94
+
95
+ if @write
96
+ @dest.write(packet.chunk)
97
+ end
98
+ when Nanite::FileEnd
99
+ Nanite::Log.debug "#{@dest.inspect} receiving is completed"
100
+ if @write
101
+ @dest.close
102
+ end
103
+
104
+ @cb.call(@data, @dest, packet.meta)
105
+ end
106
+ end
107
+
108
+ end
109
+
110
+ def subscribe_to_files(domain='global', write=false, &blk)
111
+ Nanite::Log.info "subscribing to file broadcasts for #{domain}"
112
+ @files ||= {}
113
+ amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
114
+ case msg = serializer.load(packet)
115
+ when FileStart
116
+ @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
117
+ when FileChunk, FileEnd
118
+ if file = @files[msg.token]
119
+ file.handle_packet(msg)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end