rightscale-nanite-dev 0.4.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +78 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +274 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +256 -0
  15. data/lib/nanite/config.rb +111 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +315 -0
  24. data/lib/nanite/mapper_proxy.rb +75 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +401 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/redis_tag_store.rb +141 -0
  30. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  31. data/lib/nanite/security/certificate.rb +55 -0
  32. data/lib/nanite/security/certificate_cache.rb +66 -0
  33. data/lib/nanite/security/distinguished_name.rb +34 -0
  34. data/lib/nanite/security/encrypted_document.rb +46 -0
  35. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  36. data/lib/nanite/security/secure_serializer.rb +68 -0
  37. data/lib/nanite/security/signature.rb +46 -0
  38. data/lib/nanite/security/static_certificate_store.rb +35 -0
  39. data/lib/nanite/security_provider.rb +47 -0
  40. data/lib/nanite/serializer.rb +52 -0
  41. data/lib/nanite/state.rb +135 -0
  42. data/lib/nanite/streaming.rb +125 -0
  43. data/lib/nanite/util.rb +78 -0
  44. metadata +111 -0
@@ -0,0 +1,75 @@
1
+ module Nanite
2
+
3
+ # This class allows sending requests to nanite agents without having
4
+ # to run a local mapper.
5
+ # It is used by Actor.request which can be used by actors than need
6
+ # to send requests to remote agents.
7
+ # All requests go through the mapper for security purposes.
8
+ class MapperProxy
9
+
10
+ $:.push File.dirname(__FILE__)
11
+ require 'amqp'
12
+
13
+ include AMQPHelper
14
+
15
+ attr_accessor :pending_requests, :identity, :options, :amqp, :serializer
16
+
17
+ # Accessor for actor
18
+ def self.instance
19
+ @@instance if defined?(@@instance)
20
+ end
21
+
22
+ def initialize(id, opts)
23
+ @options = opts || {}
24
+ @identity = id
25
+ @pending_requests = {}
26
+ @amqp = start_amqp(options)
27
+ @serializer = Serializer.new(options[:format])
28
+ @@instance = self
29
+ end
30
+
31
+ # Send request to given agent through the mapper
32
+ def request(type, payload = '', opts = {}, &blk)
33
+ raise "Mapper proxy not initialized" unless identity && options
34
+ request = Request.new(type, payload, opts)
35
+ request.from = identity
36
+ request.token = Identity.generate
37
+ request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
38
+ pending_requests[request.token] =
39
+ { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
40
+ Nanite::Log.info("SEND #{request.to_s([:tags, :target])}")
41
+ amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
42
+ end
43
+
44
+ def push(type, payload = '', opts = {})
45
+ raise "Mapper proxy not initialized" unless identity && options
46
+ push = Push.new(type, payload, opts)
47
+ push.from = identity
48
+ push.token = Identity.generate
49
+ push.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
50
+ Nanite::Log.info("SEND #{push.to_s([:tags, :target])}")
51
+ amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(push))
52
+ end
53
+
54
+ # Update tags registered by mapper for agent
55
+ def update_tags(new_tags, obsolete_tags)
56
+ raise "Mapper proxy not initialized" unless identity && options
57
+ update = TagUpdate.new(identity, new_tags, obsolete_tags)
58
+ Nanite::Log.info("SEND #{update.to_s}")
59
+ amqp.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(update))
60
+ end
61
+
62
+ # Handle intermediary result
63
+ def handle_intermediate_result(res)
64
+ handlers = pending_requests[res.token]
65
+ handlers[:intermediate_handler].call(res) if handlers && handlers[:intermediate_handler]
66
+ end
67
+
68
+ # Handle final result
69
+ def handle_result(res)
70
+ handlers = pending_requests.delete(res.token)
71
+ handlers[:result_handler].call(res) if handlers && handlers[:result_handler]
72
+ end
73
+
74
+ end
75
+ end
@@ -0,0 +1,92 @@
1
+ module Nanite
2
+ class Dispatcher
3
+ attr_reader :registry, :serializer, :identity, :amq, :options
4
+ attr_accessor :evmclass
5
+
6
+ def initialize(amq, registry, serializer, identity, options)
7
+ @amq = amq
8
+ @registry = registry
9
+ @serializer = serializer
10
+ @identity = identity
11
+ @options = options
12
+ @evmclass = EM
13
+ @evmclass.threadpool_size = (@options[:threadpool_size] || 20).to_i
14
+ end
15
+
16
+ def dispatch(deliverable)
17
+ prefix, meth = deliverable.type.split('/')[1..-1]
18
+ meth ||= :index
19
+ actor = registry.actor_for(prefix)
20
+
21
+ operation = lambda do
22
+ begin
23
+ intermediate_results_proc = lambda { |*args| self.handle_intermediate_results(actor, meth, deliverable, *args) }
24
+ args = [ deliverable.payload ]
25
+ args.push(deliverable) if actor.method(meth).arity == 2
26
+ actor.send(meth, *args, &intermediate_results_proc)
27
+ rescue Exception => e
28
+ handle_exception(actor, meth, deliverable, e)
29
+ end
30
+ end
31
+
32
+ callback = lambda do |r|
33
+ if deliverable.kind_of?(Request)
34
+ r = Result.new(deliverable.token, deliverable.reply_to, r, identity)
35
+ Nanite::Log.info("SEND #{r.to_s([])}")
36
+ amq.queue(deliverable.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
37
+ end
38
+ r # For unit tests
39
+ end
40
+
41
+ if @options[:single_threaded] || @options[:thread_poolsize] == 1
42
+ @evmclass.next_tick { callback.call(operation.call) }
43
+ else
44
+ @evmclass.defer(operation, callback)
45
+ end
46
+ end
47
+
48
+ protected
49
+
50
+ def handle_intermediate_results(actor, meth, deliverable, *args)
51
+ messagekey = case args.size
52
+ when 1
53
+ 'defaultkey'
54
+ when 2
55
+ args.first.to_s
56
+ else
57
+ raise ArgumentError, "handle_intermediate_results passed unexpected number of arguments (#{args.size})"
58
+ end
59
+ message = args.last
60
+ @evmclass.defer(lambda {
61
+ [deliverable.reply_to, IntermediateMessage.new(deliverable.token, deliverable.reply_to, identity, messagekey, message)]
62
+ }, lambda { |r|
63
+ amq.queue(r.first, :no_declare => options[:secure]).publish(serializer.dump(r.last))
64
+ })
65
+ end
66
+
67
+ private
68
+
69
+ def describe_error(e)
70
+ "#{e.class.name}: #{e.message}\n #{e.backtrace.join("\n ")}"
71
+ end
72
+
73
+ def handle_exception(actor, meth, deliverable, e)
74
+ error = describe_error(e)
75
+ Nanite::Log.error(error)
76
+ begin
77
+ if actor.class.exception_callback
78
+ case actor.class.exception_callback
79
+ when Symbol, String
80
+ actor.send(actor.class.exception_callback, meth.to_sym, deliverable, e)
81
+ when Proc
82
+ actor.instance_exec(meth.to_sym, deliverable, e, &actor.class.exception_callback)
83
+ end
84
+ end
85
+ rescue Exception => e1
86
+ error = describe_error(e1)
87
+ Nanite::Log.error(error)
88
+ end
89
+ error
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,401 @@
1
+ module Nanite
2
+ # Base class for all Nanite packets,
3
+ # knows how to dump itself to JSON
4
+ class Packet
5
+
6
+ attr_accessor :size
7
+
8
+ def initialize
9
+ raise NotImplementedError.new("#{self.class.name} is an abstract class.")
10
+ end
11
+
12
+ def to_json(*a)
13
+ js = {
14
+ 'json_class' => self.class.name,
15
+ 'data' => instance_variables.inject({}) {|m,ivar| m[ivar.to_s.sub(/@/,'')] = instance_variable_get(ivar); m }
16
+ }.to_json(*a)
17
+ js = js.chop + ",\"size\":#{js.size}}"
18
+ js
19
+ end
20
+
21
+ # Log representation
22
+ def to_s(filter=nil)
23
+ res = "[#{ self.class.to_s.split('::').last.
24
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
25
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
26
+ downcase }]"
27
+ res += " (#{size.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")} bytes)" if size && !size.to_s.empty?
28
+ res
29
+ end
30
+
31
+ # Log friendly name for given agent id
32
+ def id_to_s(id)
33
+ case id
34
+ when /^mapper-/ then 'mapper'
35
+ when /^nanite-(.*)/ then Regexp.last_match(1)
36
+ else id
37
+ end
38
+ end
39
+
40
+ end
41
+
42
+ # packet that means start of a file transfer
43
+ # operation
44
+ class FileStart < Packet
45
+
46
+ attr_accessor :filename, :token, :dest
47
+
48
+ def initialize(filename, dest, token, size=nil)
49
+ @filename = filename
50
+ @dest = dest
51
+ @token = token
52
+ @size = size
53
+ end
54
+
55
+ def self.json_create(o)
56
+ i = o['data']
57
+ new(i['filename'], i['dest'], i['token'], o['size'])
58
+ end
59
+
60
+ def to_s
61
+ "#{super} <#{token}> #{filename} to #{dest}"
62
+ end
63
+ end
64
+
65
+ # packet that means end of a file transfer
66
+ # operation
67
+ class FileEnd < Packet
68
+
69
+ attr_accessor :token, :meta
70
+
71
+ def initialize(token, meta, size=nil)
72
+ @token = token
73
+ @meta = meta
74
+ @size = size
75
+ end
76
+
77
+ def self.json_create(o)
78
+ i = o['data']
79
+ new(i['token'], i['meta'], o['size'])
80
+ end
81
+
82
+ def to_s
83
+ "#{super} <#{token}> meta #{meta}"
84
+ end
85
+ end
86
+
87
+ # packet that carries data chunks during a file transfer
88
+ class FileChunk < Packet
89
+
90
+ attr_accessor :chunk, :token
91
+
92
+ def initialize(token, chunk=nil, size=nil)
93
+ @chunk = chunk
94
+ @token = token
95
+ @size = size
96
+ end
97
+
98
+ def self.json_create(o)
99
+ i = o['data']
100
+ new(i['token'], i['chunk'], o['size'])
101
+ end
102
+
103
+ def to_s
104
+ "#{super} <#{token}>"
105
+ end
106
+ end
107
+
108
+ # packet that means a work request from mapper
109
+ # to actor node
110
+ #
111
+ # type is a service name
112
+ # payload is arbitrary data that is transferred from mapper to actor
113
+ #
114
+ # Options:
115
+ # from is sender identity
116
+ # on_behalf is agent identity that should be used to authorize request
117
+ # token is a generated request id that mapper uses to identify replies
118
+ # reply_to is identity of the node actor replies to, usually a mapper itself
119
+ # selector is the selector used to route the request
120
+ # target is the target nanite for the request
121
+ # persistent signifies if this request should be saved to persistent storage by the AMQP broker
122
+ class Request < Packet
123
+
124
+ attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
125
+
126
+ DEFAULT_OPTIONS = {:selector => :least_loaded}
127
+
128
+ def initialize(type, payload, opts={}, size=nil)
129
+ opts = DEFAULT_OPTIONS.merge(opts)
130
+ @type = type
131
+ @payload = payload
132
+ @size = size
133
+ @from = opts[:from]
134
+ @on_behalf = opts[:on_behalf]
135
+ @token = opts[:token]
136
+ @reply_to = opts[:reply_to]
137
+ @selector = opts[:selector]
138
+ @target = opts[:target]
139
+ @persistent = opts[:persistent]
140
+ @tags = opts[:tags] || []
141
+ end
142
+
143
+ def self.json_create(o)
144
+ i = o['data']
145
+ new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
146
+ :token => i['token'], :reply_to => i['reply_to'],
147
+ :selector => i['selector'], :target => i['target'],
148
+ :persistent => i['persistent'], :tags => i['tags'] },
149
+ o['size'])
150
+ end
151
+
152
+ def to_s(filter=nil)
153
+ log_msg = "#{super} <#{token}> #{type}"
154
+ log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
155
+ log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
156
+ log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
157
+ log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
158
+ log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
159
+ log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
160
+ log_msg
161
+ end
162
+
163
+ end
164
+
165
+ # packet that means a work push from mapper
166
+ # to actor node
167
+ #
168
+ # type is a service name
169
+ # payload is arbitrary data that is transferred from mapper to actor
170
+ #
171
+ # Options:
172
+ # from is sender identity
173
+ # on_behalf is agent identity that should be used to authorize request
174
+ # token is a generated request id that mapper uses to identify replies
175
+ # selector is the selector used to route the request
176
+ # target is the target nanite for the request
177
+ # persistent signifies if this request should be saved to persistent storage by the AMQP broker
178
+ class Push < Packet
179
+
180
+ attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
181
+
182
+ DEFAULT_OPTIONS = {:selector => :least_loaded}
183
+
184
+ def initialize(type, payload, opts={}, size=nil)
185
+ opts = DEFAULT_OPTIONS.merge(opts)
186
+ @type = type
187
+ @payload = payload
188
+ @size = size
189
+ @from = opts[:from]
190
+ @on_behalf = opts[:on_behalf]
191
+ @token = opts[:token]
192
+ @selector = opts[:selector]
193
+ @target = opts[:target]
194
+ @persistent = opts[:persistent]
195
+ @tags = opts[:tags] || []
196
+ end
197
+
198
+ def self.json_create(o)
199
+ i = o['data']
200
+ new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
201
+ :token => i['token'], :selector => i['selector'],
202
+ :target => i['target'], :persistent => i['persistent'],
203
+ :tags => i['tags'] },
204
+ o['size'])
205
+ end
206
+
207
+ def to_s(filter=nil)
208
+ log_msg = "#{super} <#{token}> #{type}"
209
+ log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
210
+ log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
211
+ log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
212
+ log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
213
+ log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
214
+ log_msg
215
+ end
216
+ end
217
+
218
+ # packet that means a work result notification sent from actor to mapper
219
+ #
220
+ # from is sender identity
221
+ # results is arbitrary data that is transferred from actor, a result of actor's work
222
+ # token is a generated request id that mapper uses to identify replies
223
+ # to is identity of the node result should be delivered to
224
+ class Result < Packet
225
+
226
+ attr_accessor :token, :results, :to, :from
227
+
228
+ def initialize(token, to, results, from, size=nil)
229
+ @token = token
230
+ @to = to
231
+ @from = from
232
+ @results = results
233
+ @size = size
234
+ end
235
+
236
+ def self.json_create(o)
237
+ i = o['data']
238
+ new(i['token'], i['to'], i['results'], i['from'], o['size'])
239
+ end
240
+
241
+ def to_s(filter=nil)
242
+ log_msg = "#{super} <#{token}>"
243
+ log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
244
+ log_msg += " to #{id_to_s(to)}" if filter.nil? || filter.include?(:to)
245
+ log_msg += " results: #{results.inspect}" if filter.nil? || filter.include?(:results)
246
+ log_msg
247
+ end
248
+ end
249
+
250
+ # packet that means an intermediate status notification sent from actor to mapper. is appended to a list of messages matching messagekey.
251
+ #
252
+ # from is sender identity
253
+ # messagekey is a string that can become part of a redis key, which identifies the name under which the message is stored
254
+ # message is arbitrary data that is transferred from actor, an intermediate result of actor's work
255
+ # token is a generated request id that mapper uses to identify replies
256
+ # to is identity of the node result should be delivered to
257
+ class IntermediateMessage < Packet
258
+
259
+ attr_accessor :token, :messagekey, :message, :to, :from
260
+
261
+ def initialize(token, to, from, messagekey, message, size=nil)
262
+ @token = token
263
+ @to = to
264
+ @from = from
265
+ @messagekey = messagekey
266
+ @message = message
267
+ @size = size
268
+ end
269
+
270
+ def self.json_create(o)
271
+ i = o['data']
272
+ new(i['token'], i['to'], i['from'], i['messagekey'], i['message'], o['size'])
273
+ end
274
+
275
+ def to_s
276
+ "#{super} <#{token}> from #{id_to_s(from)}, key #{messagekey}"
277
+ end
278
+ end
279
+
280
+ # packet that means an availability notification sent from actor to mapper
281
+ #
282
+ # from is sender identity
283
+ # services is a list of services provided by the node
284
+ # status is a load of the node by default, but may be any criteria
285
+ # agent may use to report it's availability, load, etc
286
+ class Register < Packet
287
+
288
+ attr_accessor :identity, :services, :status, :tags
289
+
290
+ def initialize(identity, services, status, tags, size=nil)
291
+ @status = status
292
+ @tags = tags
293
+ @identity = identity
294
+ @services = services
295
+ @size = size
296
+ end
297
+
298
+ def self.json_create(o)
299
+ i = o['data']
300
+ new(i['identity'], i['services'], i['status'], i['tags'], o['size'])
301
+ end
302
+
303
+ def to_s
304
+ log_msg = "#{super} #{id_to_s(identity)}"
305
+ log_msg += ", services: #{services.join(', ')}" if services && !services.empty?
306
+ log_msg += ", tags: #{tags.join(', ')}" if tags && !tags.empty?
307
+ log_msg
308
+ end
309
+ end
310
+
311
+ # packet that means deregister an agent from the mappers
312
+ #
313
+ # from is sender identity
314
+ class UnRegister < Packet
315
+
316
+ attr_accessor :identity
317
+
318
+ def initialize(identity, size=nil)
319
+ @identity = identity
320
+ @size = size
321
+ end
322
+
323
+ def self.json_create(o)
324
+ i = o['data']
325
+ new(i['identity'], o['size'])
326
+ end
327
+
328
+ def to_s
329
+ "#{super} #{id_to_s(identity)}"
330
+ end
331
+ end
332
+
333
+ # heartbeat packet
334
+ #
335
+ # identity is sender's identity
336
+ # status is sender's status (see Register packet documentation)
337
+ class Ping < Packet
338
+
339
+ attr_accessor :identity, :status
340
+
341
+ def initialize(identity, status, size=nil)
342
+ @status = status
343
+ @identity = identity
344
+ @size = size
345
+ end
346
+
347
+ def self.json_create(o)
348
+ i = o['data']
349
+ new(i['identity'], i['status'], o['size'])
350
+ end
351
+
352
+ def to_s
353
+ "#{super} #{id_to_s(identity)} status #{status}"
354
+ end
355
+
356
+ end
357
+
358
+ # packet that is sent by workers to the mapper
359
+ # when worker initially comes online to advertise
360
+ # it's services
361
+ class Advertise < Packet
362
+
363
+ def initialize(size=nil)
364
+ @size = size
365
+ end
366
+
367
+ def self.json_create(o)
368
+ new(o['size'])
369
+ end
370
+
371
+ end
372
+
373
+ # packet that is sent by agents to the mapper
374
+ # to update their tags
375
+ class TagUpdate < Packet
376
+
377
+ attr_accessor :identity, :new_tags, :obsolete_tags
378
+
379
+ def initialize(identity, new_tags, obsolete_tags, size=nil)
380
+ @identity = identity
381
+ @new_tags = new_tags
382
+ @obsolete_tags = obsolete_tags
383
+ @size = size
384
+ end
385
+
386
+ def self.json_create(o)
387
+ i = o['data']
388
+ new(i['identity'], i['new_tags'], i['obsolete_tags'], o['size'])
389
+ end
390
+
391
+ def to_s
392
+ log_msg = "#{super} #{id_to_s(identity)}"
393
+ log_msg += ", new tags: #{new_tags.join(', ')}" if new_tags && !new_tags.empty?
394
+ log_msg += ", obsolete tags: #{obsolete_tags.join(', ')}" if obsolete_tags && !obsolete_tags.empty?
395
+ log_msg
396
+ end
397
+
398
+ end
399
+
400
+ end
401
+