rightscale-nanite-dev 0.4.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +78 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +274 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +256 -0
  15. data/lib/nanite/config.rb +111 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +315 -0
  24. data/lib/nanite/mapper_proxy.rb +75 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +401 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/redis_tag_store.rb +141 -0
  30. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  31. data/lib/nanite/security/certificate.rb +55 -0
  32. data/lib/nanite/security/certificate_cache.rb +66 -0
  33. data/lib/nanite/security/distinguished_name.rb +34 -0
  34. data/lib/nanite/security/encrypted_document.rb +46 -0
  35. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  36. data/lib/nanite/security/secure_serializer.rb +68 -0
  37. data/lib/nanite/security/signature.rb +46 -0
  38. data/lib/nanite/security/static_certificate_store.rb +35 -0
  39. data/lib/nanite/security_provider.rb +47 -0
  40. data/lib/nanite/serializer.rb +52 -0
  41. data/lib/nanite/state.rb +135 -0
  42. data/lib/nanite/streaming.rb +125 -0
  43. data/lib/nanite/util.rb +78 -0
  44. metadata +111 -0
@@ -0,0 +1,38 @@
1
+ module Nanite
2
+ class LocalState < ::Hash
3
+ def initialize(hsh={})
4
+ hsh.each do |k,v|
5
+ self[k] = v
6
+ end
7
+ end
8
+
9
+ def all_services
10
+ all(:services)
11
+ end
12
+
13
+ def all_tags
14
+ all(:tags)
15
+ end
16
+
17
+ def nanites_for(from, service, tags)
18
+ tags = tags.dup.flatten
19
+ nanites = select { |name, state| state[:services].include?(service) }
20
+ unless tags.empty?
21
+ nanites.select { |a, b| !(b[:tags] & tags).empty? }
22
+ else
23
+ nanites
24
+ end.to_a
25
+ end
26
+
27
+ def update_status(name, status)
28
+ self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
29
+ end
30
+
31
+ private
32
+
33
+ def all(key)
34
+ map { |n,s| s[key] }.flatten.uniq.compact
35
+ end
36
+
37
+ end # LocalState
38
+ end # Nanite
data/lib/nanite/log.rb ADDED
@@ -0,0 +1,66 @@
1
+ require 'nanite/config'
2
+ require 'nanite/log/formatter'
3
+ require 'logger'
4
+
5
+ module Nanite
6
+ class Log
7
+
8
+ @logger = nil
9
+
10
+ # Map log levels symbols to values
11
+ LEVELS = { :debug => Logger::DEBUG,
12
+ :info => Logger::INFO,
13
+ :warn => Logger::WARN,
14
+ :error => Logger::ERROR,
15
+ :fatal => Logger::FATAL }
16
+
17
+ class << self
18
+ attr_accessor :logger, :level, :file #:nodoc
19
+
20
+ # Use Nanite::Logger.init when you want to set up the logger manually.
21
+ # If this method is called with no arguments, it will log to STDOUT at the :info level.
22
+ # It also configures the Logger instance it creates to use the custom Nanite::Log::Formatter class.
23
+ def init(identity = nil, path = false)
24
+ if path
25
+ @file = File.join(path, "nanite.#{identity}.log")
26
+ else
27
+ @file = STDOUT
28
+ end
29
+ @logger = Logger.new(file)
30
+ @logger.formatter = Nanite::Log::Formatter.new
31
+ Log.level = :info
32
+ end
33
+
34
+ # Sets the level for the Logger by symbol or by command line argument.
35
+ # Throws an ArgumentError if you feed it a bogus log level (that is not
36
+ # one of :debug, :info, :warn, :error, :fatal or the corresponding strings or a valid Logger level)
37
+ def level=(loglevel)
38
+ init unless @logger
39
+ lvl = case loglevel
40
+ when String then loglevel.intern
41
+ when Integer then LEVELS.invert[loglevel]
42
+ else loglevel
43
+ end
44
+ unless LEVELS.include?(lvl)
45
+ raise(ArgumentError, 'Log level must be one of :debug, :info, :warn, :error, or :fatal')
46
+ end
47
+ @logger.info("[setup] setting log level to #{lvl.to_s.upcase}")
48
+ @level = lvl
49
+ @logger.level = LEVELS[lvl]
50
+ end
51
+
52
+ # Passes any other method calls on directly to the underlying Logger object created with init. If
53
+ # this method gets hit before a call to Nanite::Logger.init has been made, it will call
54
+ # Nanite::Logger.init() with no arguments.
55
+ def method_missing(method_symbol, *args)
56
+ init unless @logger
57
+ if args.length > 0
58
+ @logger.send(method_symbol, *args)
59
+ else
60
+ @logger.send(method_symbol)
61
+ end
62
+ end
63
+
64
+ end # class << self
65
+ end
66
+ end
@@ -0,0 +1,39 @@
1
+ require 'logger'
2
+ require 'time'
3
+
4
+ module Nanite
5
+ class Log
6
+ class Formatter < Logger::Formatter
7
+ @@show_time = true
8
+
9
+ def self.show_time=(show=false)
10
+ @@show_time = show
11
+ end
12
+
13
+ # Prints a log message as '[time] severity: message' if Nanite::Log::Formatter.show_time == true.
14
+ # Otherwise, doesn't print the time.
15
+ def call(severity, time, progname, msg)
16
+ if @@show_time
17
+ sprintf("[%s] %s: %s\n", time.rfc2822(), severity, msg2str(msg))
18
+ else
19
+ sprintf("%s: %s\n", severity, msg2str(msg))
20
+ end
21
+ end
22
+
23
+ # Converts some argument to a Logger.severity() call to a string. Regular strings pass through like
24
+ # normal, Exceptions get formatted as "message (class)\nbacktrace", and other random stuff gets
25
+ # put through "object.inspect"
26
+ def msg2str(msg)
27
+ case msg
28
+ when ::String
29
+ msg
30
+ when ::Exception
31
+ "#{ msg.message } (#{ msg.class })\n" <<
32
+ (msg.backtrace || []).join("\n")
33
+ else
34
+ msg.inspect
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,315 @@
1
+ module Nanite
2
+ # Mappers are control nodes in nanite clusters. Nanite clusters
3
+ # can follow peer-to-peer model of communication as well as client-server,
4
+ # and mappers are nodes that know who to send work requests to agents.
5
+ #
6
+ # Mappers can reside inside a front end web application written in Merb/Rails
7
+ # and distribute heavy lifting to actors that register with the mapper as soon
8
+ # as they go online.
9
+ #
10
+ # Each mapper tracks nanites registered with it. It periodically checks
11
+ # when the last time a certain nanite sent a heartbeat notification,
12
+ # and removes those that have timed out from the list of available workers.
13
+ # As soon as a worker goes back online again it re-registers itself
14
+ # and the mapper adds it to the list and makes it available to
15
+ # be called again.
16
+ #
17
+ # This makes Nanite clusters self-healing and immune to individual node
18
+ # failures.
19
+ class Mapper
20
+ include AMQPHelper
21
+ include ConsoleHelper
22
+ include DaemonizeHelper
23
+
24
+ attr_reader :cluster, :identity, :job_warden, :options, :serializer, :amq
25
+
26
+ DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({
27
+ :user => 'mapper',
28
+ :identity => Identity.generate,
29
+ :agent_timeout => 15,
30
+ :offline_redelivery_frequency => 10,
31
+ :persistent => false,
32
+ :offline_failsafe => false,
33
+ :callbacks => {}
34
+ }) unless defined?(DEFAULT_OPTIONS)
35
+
36
+ # Initializes a new mapper and establishes
37
+ # AMQP connection. This must be used inside EM.run block or if EventMachine reactor
38
+ # is already started, for instance, by a Thin server that your Merb/Rails
39
+ # application runs on.
40
+ #
41
+ # Mapper options:
42
+ #
43
+ # identity : identity of this mapper, may be any string
44
+ #
45
+ # format : format to use for packets serialization. Can be :marshal, :json or :yaml or :secure.
46
+ # Defaults to Ruby's Marshall format. For interoperability with
47
+ # AMQP clients implemented in other languages, use JSON.
48
+ #
49
+ # Note that Nanite uses JSON gem,
50
+ # and ActiveSupport's JSON encoder may cause clashes
51
+ # if ActiveSupport is loaded after JSON gem.
52
+ #
53
+ # Also using the secure format requires prior initialization of the serializer, see
54
+ # SecureSerializer.init
55
+ #
56
+ # log_level : the verbosity of logging, can be debug, info, warn, error or fatal.
57
+ #
58
+ # agent_timeout : how long to wait before an agent is considered to be offline
59
+ # and thus removed from the list of available agents.
60
+ #
61
+ # log_dir : log file path, defaults to the current working directory.
62
+ #
63
+ # console : true tells mapper to start interactive console
64
+ #
65
+ # daemonize : true tells mapper to daemonize
66
+ #
67
+ # pid_dir : path to the directory where the agent stores its pid file (only if daemonized)
68
+ # defaults to the root or the current working directory.
69
+ #
70
+ # offline_redelivery_frequency : The frequency in seconds that messages stored in the offline queue will be retrieved
71
+ # for attempted redelivery to the nanites. Default is 10 seconds.
72
+ #
73
+ # persistent : true instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the
74
+ # broker is restarted. Default is false. Can be overriden on a per-message basis using the request and push methods.
75
+ #
76
+ # secure : use Security features of rabbitmq to restrict nanites to themselves
77
+ #
78
+ # prefetch : Sets prefetch (only supported in RabbitMQ >= 1.6)
79
+ #
80
+ # callbacks : A set of callbacks to be executed on specific events. Supported events are :register,
81
+ # :unregister, :timeout and :exception. This option must be a hash with event names as
82
+ # as keys and Procs as values. The Proc's arity (number of parameters) depends on the
83
+ # type of callback:
84
+ # exception -- the exception, the message being processed, a reference to the mapper
85
+ # all others -- the corresponding nanite's identity, a reference to the mapper
86
+ #
87
+ # tag_store : Name of class which implements tag store backend interface, RedisTagStore by default
88
+ #
89
+ # Connection options:
90
+ #
91
+ # vhost : AMQP broker vhost that should be used
92
+ #
93
+ # user : AMQP broker user
94
+ #
95
+ # pass : AMQP broker password
96
+ #
97
+ # host : host AMQP broker (or node of interest) runs on,
98
+ # defaults to 0.0.0.0
99
+ #
100
+ # port : port AMQP broker (or node of interest) runs on,
101
+ # this defaults to 5672, port used by some widely
102
+ # used AMQP brokers (RabbitMQ and ZeroMQ)
103
+ #
104
+ # @api :public:
105
+ def self.start(options = {})
106
+ mapper = new(options)
107
+ mapper.run
108
+ mapper
109
+ end
110
+
111
+ def initialize(options)
112
+ @options = DEFAULT_OPTIONS.clone.merge(options)
113
+ root = options[:root] || @options[:root]
114
+ custom_config = if root
115
+ file = File.expand_path(File.join(root, 'config.yml'))
116
+ File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {}
117
+ else
118
+ {}
119
+ end
120
+ options.delete(:identity) unless options[:identity]
121
+ @options.update(custom_config.merge(options))
122
+ @identity = "mapper-#{@options[:identity]}"
123
+ @options[:file_root] ||= File.join(@options[:root], 'files')
124
+ @options[:log_path] = false
125
+ if @options[:daemonize]
126
+ @options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
127
+ end
128
+ @options.freeze
129
+ end
130
+
131
+ def run
132
+ setup_logging
133
+ @serializer = Serializer.new(@options[:format])
134
+ pid_file = PidFile.new(@identity, @options)
135
+ pid_file.check
136
+ if @options[:daemonize]
137
+ daemonize(@identity, @options)
138
+ pid_file.write
139
+ at_exit { pid_file.remove }
140
+ else
141
+ trap("INT") {exit}
142
+ end
143
+ @amq = start_amqp(@options)
144
+ @job_warden = JobWarden.new(@serializer)
145
+ setup_cluster
146
+ Nanite::Log.info('[setup] starting mapper')
147
+ setup_queues
148
+ start_console if @options[:console] && !@options[:daemonize]
149
+ end
150
+
151
+ # Make a nanite request which expects a response.
152
+ #
153
+ # ==== Parameters
154
+ # type<String>:: The dispatch route for the request
155
+ # payload<Object>:: Payload to send. This will get marshalled en route
156
+ #
157
+ # ==== Options
158
+ # :selector<Symbol>:: Method for selecting an actor. Default is :least_loaded.
159
+ # :least_loaded:: Pick the nanite which has the lowest load.
160
+ # :all:: Send the request to all nanites which respond to the service.
161
+ # :random:: Randomly pick a nanite.
162
+ # :rr: Select a nanite according to round robin ordering.
163
+ # :target<String>:: Select a specific nanite via identity, rather than using
164
+ # a selector.
165
+ # :offline_failsafe<Boolean>:: Store messages in an offline queue when all
166
+ # the nanites are offline. Messages will be redelivered when nanites come online.
167
+ # Default is false unless the mapper was started with the --offline-failsafe flag.
168
+ # :persistent<Boolean>:: Instructs the AMQP broker to save the message to persistent
169
+ # storage so that it isnt lost when the broker is restarted.
170
+ # Default is false unless the mapper was started with the --persistent flag.
171
+ # :intermediate_handler:: Takes a lambda to call when an IntermediateMessage
172
+ # event arrives from a nanite. If passed a Hash, hash keys should correspond to
173
+ # the IntermediateMessage keys provided by the nanite, and each should have a value
174
+ # that is a lambda/proc taking the parameters specified here. Can supply a key '*'
175
+ # as a catch-all for unmatched keys.
176
+ #
177
+ # ==== Block Parameters for intermediate_handler
178
+ # key<String>:: array of unique keys for which intermediate state has been received
179
+ # since the last call to this block.
180
+ # nanite<String>:: nanite which sent the message.
181
+ # state:: most recently delivered intermediate state for the key provided.
182
+ # job:: (optional) -- if provided, this parameter gets the whole job object, if there's
183
+ # a reason to do more complex work with the job.
184
+ #
185
+ # ==== Block Parameters
186
+ # :results<Object>:: The returned value from the nanite actor.
187
+ #
188
+ # @api :public:
189
+ def request(type, payload = '', opts = {}, &blk)
190
+ request = build_deliverable(Request, type, payload, opts)
191
+ send_request(request, opts, &blk)
192
+ end
193
+
194
+ # Send request with pre-built request instance
195
+ def send_request(request, opts = {}, &blk)
196
+ request.reply_to = identity
197
+ intm_handler = opts.delete(:intermediate_handler)
198
+ targets = cluster.targets_for(request)
199
+ if !targets.empty?
200
+ job = job_warden.new_job(request, targets, intm_handler, blk)
201
+ cluster.route(request, job.targets)
202
+ job
203
+ elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
204
+ job_warden.new_job(request, [], intm_handler, blk)
205
+ cluster.publish(request, 'mapper-offline')
206
+ :offline
207
+ else
208
+ false
209
+ end
210
+ end
211
+
212
+ # Make a nanite request which does not expect a response.
213
+ #
214
+ # ==== Parameters
215
+ # type<String>:: The dispatch route for the request
216
+ # payload<Object>:: Payload to send. This will get marshalled en route
217
+ #
218
+ # ==== Options
219
+ # :selector<Symbol>:: Method for selecting an actor. Default is :least_loaded.
220
+ # :least_loaded:: Pick the nanite which has the lowest load.
221
+ # :all:: Send the request to all nanites which respond to the service.
222
+ # :random:: Randomly pick a nanite.
223
+ # :rr: Select a nanite according to round robin ordering.
224
+ # :offline_failsafe<Boolean>:: Store messages in an offline queue when all
225
+ # the nanites are offline. Messages will be redelivered when nanites come online.
226
+ # Default is false unless the mapper was started with the --offline-failsafe flag.
227
+ # :persistent<Boolean>:: Instructs the AMQP broker to save the message to persistent
228
+ # storage so that it isnt lost when the broker is restarted.
229
+ # Default is false unless the mapper was started with the --persistent flag.
230
+ #
231
+ # @api :public:
232
+ def push(type, payload = '', opts = {})
233
+ push = build_deliverable(Push, type, payload, opts)
234
+ send_push(push, opts)
235
+ end
236
+
237
+ def send_push(push, opts = {})
238
+ targets = cluster.targets_for(push)
239
+ if !targets.empty?
240
+ cluster.route(push, targets)
241
+ true
242
+ elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe]
243
+ cluster.publish(push, 'mapper-offline')
244
+ :offline
245
+ else
246
+ false
247
+ end
248
+ end
249
+
250
+ private
251
+
252
+ def build_deliverable(deliverable_type, type, payload, opts)
253
+ deliverable = deliverable_type.new(type, payload, opts)
254
+ deliverable.from = identity
255
+ deliverable.token = Identity.generate
256
+ deliverable.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
257
+ deliverable
258
+ end
259
+
260
+ def setup_queues
261
+ if amq.respond_to?(:prefetch) && @options.has_key?(:prefetch)
262
+ amq.prefetch(@options[:prefetch])
263
+ end
264
+
265
+ setup_offline_queue
266
+ setup_message_queue
267
+ end
268
+
269
+ def setup_offline_queue
270
+ offline_queue = amq.queue('mapper-offline', :durable => true)
271
+ offline_queue.subscribe(:ack => true) do |info, deliverable|
272
+ deliverable = serializer.load(deliverable)
273
+ targets = cluster.targets_for(deliverable)
274
+ unless targets.empty?
275
+ info.ack
276
+ if deliverable.kind_of?(Request)
277
+ if job = job_warden.jobs[deliverable.token]
278
+ job.targets = targets
279
+ else
280
+ deliverable.reply_to = identity
281
+ job_warden.new_job(deliverable, targets)
282
+ end
283
+ end
284
+ cluster.route(deliverable, targets)
285
+ end
286
+ end
287
+
288
+ EM.add_periodic_timer(options[:offline_redelivery_frequency]) { offline_queue.recover }
289
+ end
290
+
291
+ def setup_message_queue
292
+ amq.queue(identity, :exclusive => true).bind(amq.fanout(identity)).subscribe do |msg|
293
+ begin
294
+ msg = serializer.load(msg)
295
+ Nanite::Log.debug("RECV #{msg.to_s}")
296
+ Nanite::Log.info("RECV #{msg.to_s([:from])}") unless Nanite::Log.level == :debug
297
+ job_warden.process(msg)
298
+ rescue Exception => e
299
+ Nanite::Log.error("RECV [result] #{e.message}")
300
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
301
+ end
302
+ end
303
+ end
304
+
305
+ def setup_logging
306
+ Nanite::Log.init(@identity, @options[:log_path])
307
+ Nanite::Log.level = @options[:log_level] if @options[:log_level]
308
+ end
309
+
310
+ def setup_cluster
311
+ @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:tag_store], @options[:callbacks])
312
+ end
313
+ end
314
+ end
315
+