rightscale-nanite 0.4.1 → 0.4.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/lib/nanite.rb +71 -0
  2. data/lib/nanite/actor.rb +60 -0
  3. data/lib/nanite/actor_registry.rb +24 -0
  4. data/lib/nanite/admin.rb +153 -0
  5. data/lib/nanite/agent.rb +250 -0
  6. data/lib/nanite/amqp.rb +47 -0
  7. data/lib/nanite/cluster.rb +203 -0
  8. data/lib/nanite/config.rb +102 -0
  9. data/lib/nanite/console.rb +39 -0
  10. data/lib/nanite/daemonize.rb +13 -0
  11. data/lib/nanite/dispatcher.rb +90 -0
  12. data/lib/nanite/identity.rb +16 -0
  13. data/lib/nanite/job.rb +104 -0
  14. data/lib/nanite/local_state.rb +34 -0
  15. data/lib/nanite/log.rb +64 -0
  16. data/lib/nanite/log/formatter.rb +39 -0
  17. data/lib/nanite/mapper.rb +277 -0
  18. data/lib/nanite/mapper_proxy.rb +56 -0
  19. data/lib/nanite/packets.rb +231 -0
  20. data/lib/nanite/pid_file.rb +52 -0
  21. data/lib/nanite/reaper.rb +38 -0
  22. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  23. data/lib/nanite/security/certificate.rb +55 -0
  24. data/lib/nanite/security/certificate_cache.rb +66 -0
  25. data/lib/nanite/security/distinguished_name.rb +34 -0
  26. data/lib/nanite/security/encrypted_document.rb +46 -0
  27. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  28. data/lib/nanite/security/secure_serializer.rb +67 -0
  29. data/lib/nanite/security/signature.rb +40 -0
  30. data/lib/nanite/security/static_certificate_store.rb +35 -0
  31. data/lib/nanite/security_provider.rb +47 -0
  32. data/lib/nanite/serializer.rb +52 -0
  33. data/lib/nanite/state.rb +164 -0
  34. data/lib/nanite/streaming.rb +125 -0
  35. data/lib/nanite/util.rb +51 -0
  36. data/spec/actor_registry_spec.rb +62 -0
  37. data/spec/actor_spec.rb +59 -0
  38. data/spec/agent_spec.rb +235 -0
  39. data/spec/cached_certificate_store_proxy_spec.rb +34 -0
  40. data/spec/certificate_cache_spec.rb +49 -0
  41. data/spec/certificate_spec.rb +27 -0
  42. data/spec/cluster_spec.rb +300 -0
  43. data/spec/dispatcher_spec.rb +136 -0
  44. data/spec/distinguished_name_spec.rb +24 -0
  45. data/spec/encrypted_document_spec.rb +21 -0
  46. data/spec/job_spec.rb +219 -0
  47. data/spec/local_state_spec.rb +112 -0
  48. data/spec/packet_spec.rb +218 -0
  49. data/spec/rsa_key_pair_spec.rb +33 -0
  50. data/spec/secure_serializer_spec.rb +41 -0
  51. data/spec/serializer_spec.rb +107 -0
  52. data/spec/signature_spec.rb +30 -0
  53. data/spec/spec_helper.rb +23 -0
  54. data/spec/static_certificate_store_spec.rb +30 -0
  55. data/spec/util_spec.rb +63 -0
  56. metadata +62 -1
@@ -0,0 +1,56 @@
1
+ module Nanite
2
+
3
+ # This class allows sending requests to nanite agents without having
4
+ # to run a local mapper.
5
+ # It is used by Actor.request which can be used by actors than need
6
+ # to send requests to remote agents.
7
+ # All requests go through the mapper for security purposes.
8
+ class MapperProxy
9
+
10
+ $:.push File.dirname(__FILE__)
11
+ require 'amqp'
12
+
13
+ include AMQPHelper
14
+
15
+ attr_accessor :pending_requests, :identity, :options, :amqp, :serializer
16
+
17
+ # Accessor for actor
18
+ def self.instance
19
+ @@instance
20
+ end
21
+
22
+ def initialize(id, opts)
23
+ @options = opts || {}
24
+ @identity = id
25
+ @pending_requests = {}
26
+ @amqp = start_amqp(options)
27
+ @serializer = Serializer.new(options[:format])
28
+ @@instance = self
29
+ end
30
+
31
+ # Send request to given agent through the mapper
32
+ def request(type, payload = '', opts = {}, &blk)
33
+ raise "Mapper proxy not initialized" unless identity && options
34
+ request = Request.new(type, payload, opts)
35
+ request.from = identity
36
+ request.token = Identity.generate
37
+ request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
38
+ pending_requests[request.token] =
39
+ { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
40
+ amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
41
+ end
42
+
43
+ # Handle intermediary result
44
+ def handle_intermediate_result(res)
45
+ handlers = pending_requests[res.token]
46
+ handlers[:intermediate_handler].call(res) if handlers && handlers[:intermediate_handler]
47
+ end
48
+
49
+ # Handle final result
50
+ def handle_result(res)
51
+ handlers = pending_requests.delete(res.token)
52
+ handlers[:result_handler].call(res) if handlers && handlers[:result_handler]
53
+ end
54
+
55
+ end
56
+ end
@@ -0,0 +1,231 @@
1
+ module Nanite
2
+ # Base class for all Nanite packets,
3
+ # knows how to dump itself to JSON
4
+ class Packet
5
+ def initialize
6
+ raise NotImplementedError.new("#{self.class.name} is an abstract class.")
7
+ end
8
+ def to_json(*a)
9
+ {
10
+ 'json_class' => self.class.name,
11
+ 'data' => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
12
+ }.to_json(*a)
13
+ end
14
+ end
15
+
16
+ # packet that means start of a file transfer
17
+ # operation
18
+ class FileStart < Packet
19
+ attr_accessor :filename, :token, :dest
20
+ def initialize(filename, dest, token)
21
+ @filename = filename
22
+ @dest = dest
23
+ @token = token
24
+ end
25
+
26
+ def self.json_create(o)
27
+ i = o['data']
28
+ new(i['filename'], i['dest'], i['token'])
29
+ end
30
+ end
31
+
32
+ # packet that means end of a file transfer
33
+ # operation
34
+ class FileEnd < Packet
35
+ attr_accessor :token, :meta
36
+ def initialize(token, meta)
37
+ @token = token
38
+ @meta = meta
39
+ end
40
+
41
+ def self.json_create(o)
42
+ i = o['data']
43
+ new(i['token'], i['meta'])
44
+ end
45
+ end
46
+
47
+ # packet that carries data chunks during a file transfer
48
+ class FileChunk < Packet
49
+ attr_accessor :chunk, :token
50
+ def initialize(token, chunk=nil)
51
+ @chunk = chunk
52
+ @token = token
53
+ end
54
+ def self.json_create(o)
55
+ i = o['data']
56
+ new(i['token'], i['chunk'])
57
+ end
58
+ end
59
+
60
+ # packet that means a work request from mapper
61
+ # to actor node
62
+ #
63
+ # type is a service name
64
+ # payload is arbitrary data that is transferred from mapper to actor
65
+ #
66
+ # Options:
67
+ # from is sender identity
68
+ # token is a generated request id that mapper uses to identify replies
69
+ # reply_to is identity of the node actor replies to, usually a mapper itself
70
+ # selector is the selector used to route the request
71
+ # target is the target nanite for the request
72
+ # persistent signifies if this request should be saved to persistent storage by the AMQP broker
73
+ class Request < Packet
74
+ attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
75
+ DEFAULT_OPTIONS = {:selector => :least_loaded}
76
+ def initialize(type, payload, opts={})
77
+ opts = DEFAULT_OPTIONS.merge(opts)
78
+ @type = type
79
+ @payload = payload
80
+ @from = opts[:from]
81
+ @token = opts[:token]
82
+ @reply_to = opts[:reply_to]
83
+ @selector = opts[:selector]
84
+ @target = opts[:target]
85
+ @persistent = opts[:persistent]
86
+ @tags = opts[:tags] || []
87
+ end
88
+ def self.json_create(o)
89
+ i = o['data']
90
+ new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :reply_to => i['reply_to'], :selector => i['selector'],
91
+ :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
92
+ end
93
+ end
94
+
95
+ # packet that means a work push from mapper
96
+ # to actor node
97
+ #
98
+ # type is a service name
99
+ # payload is arbitrary data that is transferred from mapper to actor
100
+ #
101
+ # Options:
102
+ # from is sender identity
103
+ # token is a generated request id that mapper uses to identify replies
104
+ # selector is the selector used to route the request
105
+ # target is the target nanite for the request
106
+ # persistent signifies if this request should be saved to persistent storage by the AMQP broker
107
+ class Push < Packet
108
+ attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
109
+ DEFAULT_OPTIONS = {:selector => :least_loaded}
110
+ def initialize(type, payload, opts={})
111
+ opts = DEFAULT_OPTIONS.merge(opts)
112
+ @type = type
113
+ @payload = payload
114
+ @from = opts[:from]
115
+ @token = opts[:token]
116
+ @selector = opts[:selector]
117
+ @target = opts[:target]
118
+ @persistent = opts[:persistent]
119
+ @tags = opts[:tags] || []
120
+ end
121
+ def self.json_create(o)
122
+ i = o['data']
123
+ new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :selector => i['selector'],
124
+ :target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
125
+ end
126
+ end
127
+
128
+ # packet that means a work result notification sent from actor to mapper
129
+ #
130
+ # from is sender identity
131
+ # results is arbitrary data that is transferred from actor, a result of actor's work
132
+ # token is a generated request id that mapper uses to identify replies
133
+ # to is identity of the node result should be delivered to
134
+ class Result < Packet
135
+ attr_accessor :token, :results, :to, :from
136
+ def initialize(token, to, results, from)
137
+ @token = token
138
+ @to = to
139
+ @from = from
140
+ @results = results
141
+ end
142
+ def self.json_create(o)
143
+ i = o['data']
144
+ new(i['token'], i['to'], i['results'], i['from'])
145
+ end
146
+ end
147
+
148
+ # packet that means an intermediate status notification sent from actor to mapper. is appended to a list of messages matching messagekey.
149
+ #
150
+ # from is sender identity
151
+ # messagekey is a string that can become part of a redis key, which identifies the name under which the message is stored
152
+ # message is arbitrary data that is transferred from actor, an intermediate result of actor's work
153
+ # token is a generated request id that mapper uses to identify replies
154
+ # to is identity of the node result should be delivered to
155
+ class IntermediateMessage < Packet
156
+ attr_accessor :token, :messagekey, :message, :to, :from
157
+ def initialize(token, to, from, messagekey, message)
158
+ @token = token
159
+ @to = to
160
+ @from = from
161
+ @messagekey = messagekey
162
+ @message = message
163
+ end
164
+ def self.json_create(o)
165
+ i = o['data']
166
+ new(i['token'], i['to'], i['from'], i['messagekey'], i['message'])
167
+ end
168
+ end
169
+
170
+ # packet that means an availability notification sent from actor to mapper
171
+ #
172
+ # from is sender identity
173
+ # services is a list of services provided by the node
174
+ # status is a load of the node by default, but may be any criteria
175
+ # agent may use to report it's availability, load, etc
176
+ class Register < Packet
177
+ attr_accessor :identity, :services, :status, :tags
178
+ def initialize(identity, services, status, tags)
179
+ @status = status
180
+ @tags = tags
181
+ @identity = identity
182
+ @services = services
183
+ end
184
+ def self.json_create(o)
185
+ i = o['data']
186
+ new(i['identity'], i['services'], i['status'], i['tags'])
187
+ end
188
+ end
189
+
190
+ # packet that means deregister an agent from the mappers
191
+ #
192
+ # from is sender identity
193
+ class UnRegister < Packet
194
+ attr_accessor :identity
195
+ def initialize(identity)
196
+ @identity = identity
197
+ end
198
+ def self.json_create(o)
199
+ i = o['data']
200
+ new(i['identity'])
201
+ end
202
+ end
203
+
204
+ # heartbeat packet
205
+ #
206
+ # identity is sender's identity
207
+ # status is sender's status (see Register packet documentation)
208
+ class Ping < Packet
209
+ attr_accessor :identity, :status
210
+ def initialize(identity, status)
211
+ @status = status
212
+ @identity = identity
213
+ end
214
+ def self.json_create(o)
215
+ i = o['data']
216
+ new(i['identity'], i['status'])
217
+ end
218
+ end
219
+
220
+ # packet that is sent by workers to the mapper
221
+ # when worker initially comes online to advertise
222
+ # it's services
223
+ class Advertise < Packet
224
+ def initialize
225
+ end
226
+ def self.json_create(o)
227
+ new
228
+ end
229
+ end
230
+ end
231
+
@@ -0,0 +1,52 @@
1
+ module Nanite
2
+ class PidFile
3
+ def initialize(identity, options)
4
+ @pid_dir = File.expand_path(options[:pid_dir] || options[:root] || Dir.pwd)
5
+ @pid_file = File.join(@pid_dir, "nanite.#{identity}.pid")
6
+ end
7
+
8
+ def check
9
+ if pid = read_pid
10
+ if process_running? pid
11
+ raise "#{@pid_file} already exists (pid: #{pid})"
12
+ else
13
+ Log.info "removing stale pid file: #{@pid_file}"
14
+ remove
15
+ end
16
+ end
17
+ end
18
+
19
+ def ensure_dir
20
+ FileUtils.mkdir_p @pid_dir
21
+ end
22
+
23
+ def write
24
+ ensure_dir
25
+ open(@pid_file,'w') {|f| f.write(Process.pid) }
26
+ File.chmod(0644, @pid_file)
27
+ end
28
+
29
+ def remove
30
+ File.delete(@pid_file) if exists?
31
+ end
32
+
33
+ def read_pid
34
+ open(@pid_file,'r') {|f| f.read.to_i } if exists?
35
+ end
36
+
37
+ def exists?
38
+ File.exists? @pid_file
39
+ end
40
+
41
+ def to_s
42
+ @pid_file
43
+ end
44
+
45
+ private
46
+ def process_running?(pid)
47
+ Process.getpgid(pid) != -1
48
+ rescue Errno::ESRCH
49
+ false
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,38 @@
1
+ module Nanite
2
+ class Reaper
3
+
4
+ def initialize(frequency=2)
5
+ @timeouts = {}
6
+ EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
7
+ end
8
+
9
+ def timeout(token, seconds, &blk)
10
+ @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
11
+ end
12
+
13
+ def reset_with_autoregister_hack(token,seconds,&blk)
14
+ unless @timeouts[token]
15
+ timeout(token, seconds, &blk)
16
+ end
17
+ reset(token)
18
+ end
19
+
20
+ def reset(token)
21
+ @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
22
+ end
23
+
24
+ private
25
+
26
+ def reap
27
+ time = Time.now
28
+ @timeouts.reject! do |token, data|
29
+ if time > data[:timestamp]
30
+ data[:callback].call
31
+ true
32
+ else
33
+ false
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,24 @@
1
+ module Nanite
2
+
3
+ # Proxy to actual certificate store which caches results in an LRU
4
+ # cache.
5
+ class CachedCertificateStoreProxy
6
+
7
+ # Initialize cache proxy with given certificate store.
8
+ def initialize(store)
9
+ @signer_cache = CertificateCache.new
10
+ @store = store
11
+ end
12
+
13
+ # Results from 'get_recipients' are not cached
14
+ def get_recipients(obj)
15
+ @store.get_recipients(obj)
16
+ end
17
+
18
+ # Check cache for signer certificate
19
+ def get_signer(id)
20
+ @signer_cache.get(id) { @store.get_signer(id) }
21
+ end
22
+
23
+ end
24
+ end
@@ -0,0 +1,55 @@
1
+ module Nanite
2
+
3
+ # X.509 Certificate management
4
+ class Certificate
5
+
6
+ # Underlying OpenSSL cert
7
+ attr_accessor :raw_cert
8
+
9
+ # Generate a signed X.509 certificate
10
+ #
11
+ # Arguments:
12
+ # - key: RsaKeyPair, key pair used to sign certificate
13
+ # - issuer: DistinguishedName, certificate issuer
14
+ # - subject: DistinguishedName, certificate subject
15
+ # - valid_for: Time in seconds before certificate expires (10 years by default)
16
+ def initialize(key, issuer, subject, valid_for = 3600*24*365*10)
17
+ @raw_cert = OpenSSL::X509::Certificate.new
18
+ @raw_cert.version = 2
19
+ @raw_cert.serial = 1
20
+ @raw_cert.subject = subject.to_x509
21
+ @raw_cert.issuer = issuer.to_x509
22
+ @raw_cert.public_key = key.to_public.raw_key
23
+ @raw_cert.not_before = Time.now
24
+ @raw_cert.not_after = Time.now + valid_for
25
+ @raw_cert.sign(key.raw_key, OpenSSL::Digest::SHA1.new)
26
+ end
27
+
28
+ # Load certificate from file
29
+ def self.load(file)
30
+ from_data(File.new(file))
31
+ end
32
+
33
+ # Initialize with raw certificate
34
+ def self.from_data(data)
35
+ cert = OpenSSL::X509::Certificate.new(data)
36
+ res = Certificate.allocate
37
+ res.instance_variable_set(:@raw_cert, cert)
38
+ res
39
+ end
40
+
41
+ # Save certificate to file in PEM format
42
+ def save(file)
43
+ File.open(file, "w") do |f|
44
+ f.write(@raw_cert.to_pem)
45
+ end
46
+ end
47
+
48
+ # Certificate data in PEM format
49
+ def data
50
+ @raw_cert.to_pem
51
+ end
52
+ alias :to_s :data
53
+
54
+ end
55
+ end