shift-nanite 0.4.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +76 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +264 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +250 -0
  15. data/lib/nanite/config.rb +112 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +309 -0
  24. data/lib/nanite/mapper_proxy.rb +67 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +365 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  30. data/lib/nanite/security/certificate.rb +55 -0
  31. data/lib/nanite/security/certificate_cache.rb +66 -0
  32. data/lib/nanite/security/distinguished_name.rb +34 -0
  33. data/lib/nanite/security/encrypted_document.rb +46 -0
  34. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  35. data/lib/nanite/security/secure_serializer.rb +68 -0
  36. data/lib/nanite/security/signature.rb +46 -0
  37. data/lib/nanite/security/static_certificate_store.rb +35 -0
  38. data/lib/nanite/security_provider.rb +47 -0
  39. data/lib/nanite/serializer.rb +52 -0
  40. data/lib/nanite/state.rb +168 -0
  41. data/lib/nanite/streaming.rb +125 -0
  42. data/lib/nanite/util.rb +58 -0
  43. data/spec/actor_registry_spec.rb +60 -0
  44. data/spec/actor_spec.rb +77 -0
  45. data/spec/agent_spec.rb +240 -0
  46. data/spec/cached_certificate_store_proxy_spec.rb +34 -0
  47. data/spec/certificate_cache_spec.rb +49 -0
  48. data/spec/certificate_spec.rb +27 -0
  49. data/spec/cluster_spec.rb +622 -0
  50. data/spec/distinguished_name_spec.rb +24 -0
  51. data/spec/encrypted_document_spec.rb +21 -0
  52. data/spec/job_spec.rb +251 -0
  53. data/spec/local_state_spec.rb +130 -0
  54. data/spec/nanite_dispatcher_spec.rb +136 -0
  55. data/spec/packet_spec.rb +220 -0
  56. data/spec/rsa_key_pair_spec.rb +33 -0
  57. data/spec/secure_serializer_spec.rb +41 -0
  58. data/spec/serializer_spec.rb +107 -0
  59. data/spec/signature_spec.rb +30 -0
  60. data/spec/spec_helper.rb +33 -0
  61. data/spec/static_certificate_store_spec.rb +30 -0
  62. data/spec/util_spec.rb +63 -0
  63. metadata +129 -0
@@ -0,0 +1,52 @@
1
+ module Nanite
2
+ class PidFile
3
+ def initialize(identity, options)
4
+ @pid_dir = File.expand_path(options[:pid_dir] || options[:root] || Dir.pwd)
5
+ @pid_file = File.join(@pid_dir, "nanite.#{identity}.pid")
6
+ end
7
+
8
+ def check
9
+ if pid = read_pid
10
+ if process_running? pid
11
+ raise "#{@pid_file} already exists (pid: #{pid})"
12
+ else
13
+ Log.info "removing stale pid file: #{@pid_file}"
14
+ remove
15
+ end
16
+ end
17
+ end
18
+
19
+ def ensure_dir
20
+ FileUtils.mkdir_p @pid_dir
21
+ end
22
+
23
+ def write
24
+ ensure_dir
25
+ open(@pid_file,'w') {|f| f.write(Process.pid) }
26
+ File.chmod(0644, @pid_file)
27
+ end
28
+
29
+ def remove
30
+ File.delete(@pid_file) if exists?
31
+ end
32
+
33
+ def read_pid
34
+ open(@pid_file,'r') {|f| f.read.to_i } if exists?
35
+ end
36
+
37
+ def exists?
38
+ File.exists? @pid_file
39
+ end
40
+
41
+ def to_s
42
+ @pid_file
43
+ end
44
+
45
+ private
46
+ def process_running?(pid)
47
+ Process.getpgid(pid) != -1
48
+ rescue Errno::ESRCH
49
+ false
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,39 @@
1
+ module Nanite
2
+ class Reaper
3
+ attr_reader :timeouts
4
+ def initialize(frequency=2)
5
+ @timeouts = {}
6
+ EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
7
+ end
8
+
9
+ # Add the specified token to the internal timeout hash.
10
+ # The reaper will then check this instance on every reap.
11
+ def register(token, seconds, &blk)
12
+ @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
13
+ end
14
+
15
+ def unregister(token)
16
+ @timeouts.delete(token)
17
+ end
18
+
19
+ # Updates the timeout timestamp for the given token. If the token is
20
+ # unknown to this reaper instance it will be auto-registered, usually
21
+ # happening when you have several mappers and not all of them know
22
+ # this agent yet, but received a ping from it.
23
+ def update(token, seconds, &blk)
24
+ unless @timeouts[token]
25
+ register(token, seconds, &blk)
26
+ end
27
+ @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
28
+ end
29
+
30
+ private
31
+
32
+ def reap
33
+ time = Time.now
34
+ @timeouts.reject! do |token, data|
35
+ time > data[:timestamp] and data[:callback].call
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,24 @@
1
+ module Nanite
2
+
3
+ # Proxy to actual certificate store which caches results in an LRU
4
+ # cache.
5
+ class CachedCertificateStoreProxy
6
+
7
+ # Initialize cache proxy with given certificate store.
8
+ def initialize(store)
9
+ @signer_cache = CertificateCache.new
10
+ @store = store
11
+ end
12
+
13
+ # Results from 'get_recipients' are not cached
14
+ def get_recipients(obj)
15
+ @store.get_recipients(obj)
16
+ end
17
+
18
+ # Check cache for signer certificate
19
+ def get_signer(id)
20
+ @signer_cache.get(id) { @store.get_signer(id) }
21
+ end
22
+
23
+ end
24
+ end
@@ -0,0 +1,55 @@
1
+ module Nanite
2
+
3
+ # X.509 Certificate management
4
+ class Certificate
5
+
6
+ # Underlying OpenSSL cert
7
+ attr_accessor :raw_cert
8
+
9
+ # Generate a signed X.509 certificate
10
+ #
11
+ # Arguments:
12
+ # - key: RsaKeyPair, key pair used to sign certificate
13
+ # - issuer: DistinguishedName, certificate issuer
14
+ # - subject: DistinguishedName, certificate subject
15
+ # - valid_for: Time in seconds before certificate expires (10 years by default)
16
+ def initialize(key, issuer, subject, valid_for = 3600*24*365*10)
17
+ @raw_cert = OpenSSL::X509::Certificate.new
18
+ @raw_cert.version = 2
19
+ @raw_cert.serial = 1
20
+ @raw_cert.subject = subject.to_x509
21
+ @raw_cert.issuer = issuer.to_x509
22
+ @raw_cert.public_key = key.to_public.raw_key
23
+ @raw_cert.not_before = Time.now
24
+ @raw_cert.not_after = Time.now + valid_for
25
+ @raw_cert.sign(key.raw_key, OpenSSL::Digest::SHA1.new)
26
+ end
27
+
28
+ # Load certificate from file
29
+ def self.load(file)
30
+ from_data(File.new(file))
31
+ end
32
+
33
+ # Initialize with raw certificate
34
+ def self.from_data(data)
35
+ cert = OpenSSL::X509::Certificate.new(data)
36
+ res = Certificate.allocate
37
+ res.instance_variable_set(:@raw_cert, cert)
38
+ res
39
+ end
40
+
41
+ # Save certificate to file in PEM format
42
+ def save(file)
43
+ File.open(file, "w") do |f|
44
+ f.write(@raw_cert.to_pem)
45
+ end
46
+ end
47
+
48
+ # Certificate data in PEM format
49
+ def data
50
+ @raw_cert.to_pem
51
+ end
52
+ alias :to_s :data
53
+
54
+ end
55
+ end
@@ -0,0 +1,66 @@
1
+ module Nanite
2
+
3
+ # Implements a simple LRU cache: items that are the least accessed are
4
+ # deleted first.
5
+ class CertificateCache
6
+
7
+ # Max number of items to keep in memory
8
+ DEFAULT_CACHE_MAX_COUNT = 100
9
+
10
+ # Initialize cache
11
+ def initialize(max_count = DEFAULT_CACHE_MAX_COUNT)
12
+ @items = {}
13
+ @list = []
14
+ @max_count = max_count
15
+ end
16
+
17
+ # Add item to cache
18
+ def put(key, item)
19
+ if @items.include?(key)
20
+ delete(key)
21
+ end
22
+ if @list.size == @max_count
23
+ delete(@list.first)
24
+ end
25
+ @items[key] = item
26
+ @list.push(key)
27
+ item
28
+ end
29
+ alias :[]= :put
30
+
31
+ # Retrieve item from cache
32
+ # Store item returned by given block if any
33
+ def get(key)
34
+ if @items.include?(key)
35
+ @list.each_index do |i|
36
+ if @list[i] == key
37
+ @list.delete_at(i)
38
+ break
39
+ end
40
+ end
41
+ @list.push(key)
42
+ @items[key]
43
+ else
44
+ return nil unless block_given?
45
+ self[key] = yield
46
+ end
47
+ end
48
+ alias :[] :get
49
+
50
+ # Delete item from cache
51
+ def delete(key)
52
+ c = @items[key]
53
+ if c
54
+ @items.delete(key)
55
+ @list.each_index do |i|
56
+ if @list[i] == key
57
+ @list.delete_at(i)
58
+ break
59
+ end
60
+ end
61
+ c
62
+ end
63
+ end
64
+
65
+ end
66
+ end
@@ -0,0 +1,34 @@
1
+ module Nanite
2
+
3
+ # Build X.509 compliant distinguished names
4
+ # Distinghuished names are used to desccribe both a certificate issuer and
5
+ # subject.
6
+ class DistinguishedName
7
+
8
+ # Initialize distinguished name from hash
9
+ # e.g.:
10
+ # { 'C' => 'US',
11
+ # 'ST' => 'California',
12
+ # 'L' => 'Santa Barbara',
13
+ # 'O' => 'RightScale',
14
+ # 'OU' => 'Certification Services',
15
+ # 'CN' => 'rightscale.com/emailAddress=cert@rightscale.com' }
16
+ #
17
+ def initialize(hash)
18
+ @value = hash
19
+ end
20
+
21
+ # Conversion to OpenSSL X509 DN
22
+ def to_x509
23
+ if @value
24
+ OpenSSL::X509::Name.new(@value.to_a, OpenSSL::X509::Name::OBJECT_TYPE_TEMPLATE)
25
+ end
26
+ end
27
+
28
+ # Human readable form
29
+ def to_s
30
+ '/' + @value.to_a.collect { |p| p.join('=') }.join('/') if @value
31
+ end
32
+
33
+ end
34
+ end
@@ -0,0 +1,46 @@
1
+ module Nanite
2
+
3
+ # Represents a signed an encrypted document that can be later decrypted using
4
+ # the right private key and whose signature can be verified using the right
5
+ # cert.
6
+ # This class can be used both to encrypt and sign data and to then check the
7
+ # signature and decrypt an encrypted document.
8
+ class EncryptedDocument
9
+
10
+ # Encrypt and sign data using certificate and key pair.
11
+ #
12
+ # Arguments:
13
+ # - 'data': Data to be encrypted
14
+ # - 'certs': Recipient certificates (certificates corresponding to private
15
+ # keys that may be used to decrypt data)
16
+ # - 'cipher': Cipher used for encryption, AES 256 CBC by default
17
+ #
18
+ def initialize(data, certs, cipher = 'AES-256-CBC')
19
+ cipher = OpenSSL::Cipher::Cipher.new(cipher)
20
+ certs = [ certs ] unless certs.respond_to?(:collect)
21
+ raw_certs = certs.collect { |c| c.raw_cert }
22
+ @pkcs7 = OpenSSL::PKCS7.encrypt(raw_certs, data, cipher, OpenSSL::PKCS7::BINARY)
23
+ end
24
+
25
+ # Initialize from encrypted data.
26
+ def self.from_data(encrypted_data)
27
+ doc = EncryptedDocument.allocate
28
+ doc.instance_variable_set(:@pkcs7, Nanite::PKCS7.new(encrypted_data))
29
+ doc
30
+ end
31
+
32
+ # Encrypted data using DER format
33
+ def encrypted_data
34
+ @pkcs7.to_pem
35
+ end
36
+
37
+ # Decrypted data
38
+ #
39
+ # Arguments:
40
+ # - 'key': Key used for decryption
41
+ # - 'cert': Certificate to use for decryption
42
+ def decrypted_data(key, cert)
43
+ @pkcs7.decrypt(key.raw_key, cert.raw_cert)
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,53 @@
1
+ module Nanite
2
+
3
+ # Allows generating RSA key pairs and extracting public key component
4
+ # Note: Creating a RSA key pair can take a fair amount of time (seconds)
5
+ class RsaKeyPair
6
+
7
+ DEFAULT_LENGTH = 2048
8
+
9
+ # Underlying OpenSSL keys
10
+ attr_reader :raw_key
11
+
12
+ # Create new RSA key pair using 'length' bits
13
+ def initialize(length = DEFAULT_LENGTH)
14
+ @raw_key = OpenSSL::PKey::RSA.generate(length)
15
+ end
16
+
17
+ # Does key pair include private key?
18
+ def has_private?
19
+ raw_key.private?
20
+ end
21
+
22
+ # New RsaKeyPair instance with identical public key but no private key
23
+ def to_public
24
+ RsaKeyPair.from_data(raw_key.public_key.to_pem)
25
+ end
26
+
27
+ # Key material in PEM format
28
+ def data
29
+ raw_key.to_pem
30
+ end
31
+ alias :to_s :data
32
+
33
+ # Load key pair previously serialized via 'data'
34
+ def self.from_data(data)
35
+ res = RsaKeyPair.allocate
36
+ res.instance_variable_set(:@raw_key, OpenSSL::PKey::RSA.new(data))
37
+ res
38
+ end
39
+
40
+ # Load key from file
41
+ def self.load(file)
42
+ from_data(File.read(file))
43
+ end
44
+
45
+ # Save key to file in PEM format
46
+ def save(file)
47
+ File.open(file, "w") do |f|
48
+ f.write(@raw_key.to_pem)
49
+ end
50
+ end
51
+
52
+ end
53
+ end
@@ -0,0 +1,68 @@
1
+ module Nanite
2
+
3
+ # Serializer implementation which secures messages by using
4
+ # X.509 certificate sigining.
5
+ class SecureSerializer
6
+
7
+ # Initialize serializer, must be called prior to using it.
8
+ #
9
+ # - 'identity': Identity associated with serialized messages
10
+ # - 'cert': Certificate used to sign and decrypt serialized messages
11
+ # - 'key': Private key corresponding to 'cert'
12
+ # - 'store': Certificate store. Exposes certificates used for
13
+ # encryption and signature validation.
14
+ # - 'encrypt': Whether data should be signed and encrypted ('true')
15
+ # or just signed ('false'), 'true' by default.
16
+ #
17
+ def self.init(identity, cert, key, store, encrypt = true)
18
+ @identity = identity
19
+ @cert = cert
20
+ @key = key
21
+ @store = store
22
+ @encrypt = encrypt
23
+ end
24
+
25
+ # Was serializer initialized?
26
+ def self.initialized?
27
+ @identity && @cert && @key && @store
28
+ end
29
+
30
+ # Serialize message and sign it using X.509 certificate
31
+ def self.dump(obj)
32
+ raise "Missing certificate identity" unless @identity
33
+ raise "Missing certificate" unless @cert
34
+ raise "Missing certificate key" unless @key
35
+ raise "Missing certificate store" unless @store || !@encrypt
36
+ json = obj.to_json
37
+ if @encrypt
38
+ certs = @store.get_recipients(obj)
39
+ json = EncryptedDocument.new(json, certs).encrypted_data if certs
40
+ end
41
+ sig = Signature.new(json, @cert, @key)
42
+ { 'id' => @identity, 'data' => json, 'signature' => sig.data, 'encrypted' => !certs.nil? }.to_json
43
+ end
44
+
45
+ # Unserialize data using certificate store
46
+ def self.load(json)
47
+ begin
48
+ raise "Missing certificate store" unless @store
49
+ raise "Missing certificate" unless @cert || !@encrypt
50
+ raise "Missing certificate key" unless @key || !@encrypt
51
+ data = JSON.load(json)
52
+ sig = Signature.from_data(data['signature'])
53
+ certs = @store.get_signer(data['id'])
54
+ raise "Could not find a cert for signer #{data['id']}" unless certs
55
+ certs = [ certs ] unless certs.respond_to?(:each)
56
+ jsn = data['data'] if certs.any? { |c| sig.match?(c) }
57
+ if jsn && @encrypt && data['encrypted']
58
+ jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert)
59
+ end
60
+ JSON.load(jsn) if jsn
61
+ rescue Exception => e
62
+ Nanite::Log.error("Loading of secure packet failed: #{e.message}\n#{e.backtrace.join("\n")}")
63
+ raise
64
+ end
65
+ end
66
+
67
+ end
68
+ end