rightscale-nanite-dev 0.4.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +78 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +274 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +256 -0
  15. data/lib/nanite/config.rb +111 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +315 -0
  24. data/lib/nanite/mapper_proxy.rb +75 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +401 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/redis_tag_store.rb +141 -0
  30. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  31. data/lib/nanite/security/certificate.rb +55 -0
  32. data/lib/nanite/security/certificate_cache.rb +66 -0
  33. data/lib/nanite/security/distinguished_name.rb +34 -0
  34. data/lib/nanite/security/encrypted_document.rb +46 -0
  35. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  36. data/lib/nanite/security/secure_serializer.rb +68 -0
  37. data/lib/nanite/security/signature.rb +46 -0
  38. data/lib/nanite/security/static_certificate_store.rb +35 -0
  39. data/lib/nanite/security_provider.rb +47 -0
  40. data/lib/nanite/serializer.rb +52 -0
  41. data/lib/nanite/state.rb +135 -0
  42. data/lib/nanite/streaming.rb +125 -0
  43. data/lib/nanite/util.rb +78 -0
  44. metadata +111 -0
@@ -0,0 +1,256 @@
1
+ module Nanite
2
+ class Cluster
3
+ attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
4
+
5
+ def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, tag_store=nil, callbacks = {})
6
+ @amq = amq
7
+ @agent_timeout = agent_timeout
8
+ @identity = identity
9
+ @serializer = serializer
10
+ @mapper = mapper
11
+ @state = state_configuration
12
+ @tag_store = tag_store
13
+ @security = SecurityProvider.get
14
+ @callbacks = callbacks
15
+ setup_state
16
+ @reaper = Reaper.new(agent_timeout)
17
+ setup_queues
18
+ end
19
+
20
+ # determine which nanites should receive the given request
21
+ def targets_for(request)
22
+ return [request.target] if request.target
23
+ __send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
24
+ end
25
+
26
+ # adds nanite to nanites map: key is nanite's identity
27
+ # and value is a services/status pair implemented
28
+ # as a hash
29
+ def register(reg)
30
+ case reg
31
+ when Register
32
+ if @security.authorize_registration(reg)
33
+ Nanite::Log.info("RECV #{reg.to_s}")
34
+ nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
35
+ reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
36
+ callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
37
+ else
38
+ Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
39
+ end
40
+ when UnRegister
41
+ Nanite::Log.info("RECV #{reg.to_s}")
42
+ reaper.unregister(reg.identity)
43
+ nanites.delete(reg.identity)
44
+ callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
45
+ when TagUpdate
46
+ Nanite::Log.info("RECV #{reg.to_s}")
47
+ nanites.update_tags(reg.identity, reg.new_tags, reg.obsolete_tags)
48
+ else
49
+ Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
50
+ end
51
+ end
52
+
53
+ def nanite_timed_out(token)
54
+ nanite = nanites[token]
55
+ if nanite && timed_out?(nanite)
56
+ Nanite::Log.info("Nanite #{token} timed out")
57
+ nanite = nanites.delete(token)
58
+ callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
59
+ true
60
+ end
61
+ end
62
+
63
+ def route(request, targets)
64
+ EM.next_tick { targets.map { |target| publish(request, target) } }
65
+ end
66
+
67
+ def publish(request, target)
68
+ # We need to initialize the 'target' field of the request object so that the serializer and
69
+ # the security provider have access to it.
70
+ begin
71
+ old_target = request.target
72
+ request.target = target unless target == 'mapper-offline'
73
+ if @security.authorize_request(request)
74
+ Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
75
+ amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
76
+ else
77
+ Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
78
+ end
79
+ ensure
80
+ request.target = old_target
81
+ end
82
+ end
83
+
84
+ protected
85
+
86
+ # updates nanite information (last ping timestamps, status)
87
+ # when heartbeat message is received
88
+ def handle_ping(ping)
89
+ begin
90
+ if nanite = nanites[ping.identity]
91
+ nanites.update_status(ping.identity, ping.status)
92
+ reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
93
+ else
94
+ packet = Advertise.new
95
+ Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
96
+ amq.queue(ping.identity).publish(serializer.dump(packet))
97
+ end
98
+ end
99
+ end
100
+
101
+ # forward request coming from agent
102
+ def handle_request(request)
103
+ Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
104
+ Nanite::Log.debug("RECV #{request.to_s}")
105
+ case request
106
+ when Push
107
+ mapper.send_push(request)
108
+ else
109
+ intm_handler = lambda do |result, job|
110
+ result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
111
+ forward_response(result, request.persistent)
112
+ end
113
+
114
+ result = Result.new(request.token, request.from, nil, mapper.identity)
115
+ ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
116
+ result.results = res
117
+ forward_response(result, request.persistent)
118
+ end
119
+
120
+ if ok == false
121
+ forward_response(result, request.persistent)
122
+ end
123
+ end
124
+ end
125
+
126
+ # forward response back to agent that originally made the request
127
+ def forward_response(res, persistent)
128
+ Nanite::Log.info("SEND #{res.to_s([:to])}")
129
+ amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
130
+ end
131
+
132
+ # returns least loaded nanite that provides given service
133
+ def least_loaded(from, service, tags=[])
134
+ candidates = nanites_providing(from, service, tags)
135
+ return [] if candidates.empty?
136
+
137
+ [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
138
+ end
139
+
140
+ # returns all nanites that provide given service
141
+ def all(from, service, tags=[])
142
+ nanites_providing(from, service,tags)
143
+ end
144
+
145
+ # returns a random nanite
146
+ def random(from, service, tags=[])
147
+ candidates = nanites_providing(from, service,tags)
148
+ return [] if candidates.empty?
149
+
150
+ [candidates[rand(candidates.size)]]
151
+ end
152
+
153
+ # selects next nanite that provides given service
154
+ # using round robin rotation
155
+ def rr(from, service, tags=[])
156
+ @last ||= {}
157
+ @last[service] ||= 0
158
+ candidates = nanites_providing(from, service,tags)
159
+ return [] if candidates.empty?
160
+ @last[service] = 0 if @last[service] >= candidates.size
161
+ candidate = candidates[@last[service]]
162
+ @last[service] += 1
163
+ [candidate]
164
+ end
165
+
166
+ def timed_out?(nanite)
167
+ nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
168
+ end
169
+
170
+ # returns all nanites that provide the given service
171
+ def nanites_providing(from, service, tags)
172
+ nanites.nanites_for(from, service, tags).delete_if do |nanite|
173
+ if res = timed_out?(nanite[1])
174
+ Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
175
+ end
176
+ res
177
+ end
178
+ end
179
+
180
+ def setup_queues
181
+ setup_heartbeat_queue
182
+ setup_registration_queue
183
+ setup_request_queue
184
+ end
185
+
186
+ def setup_heartbeat_queue
187
+ handler = lambda do |ping|
188
+ begin
189
+ ping = serializer.load(ping)
190
+ Nanite::Log.debug("RECV #{ping.to_s}") if ping.respond_to?(:to_s)
191
+ handle_ping(ping)
192
+ rescue Exception => e
193
+ Nanite::Log.error("RECV [ping] #{e.message}")
194
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
195
+ end
196
+ end
197
+ hb_fanout = amq.fanout('heartbeat', :durable => true)
198
+ if shared_state?
199
+ amq.queue("heartbeat").bind(hb_fanout).subscribe &handler
200
+ else
201
+ amq.queue("heartbeat-#{identity}", :exclusive => true).bind(hb_fanout).subscribe &handler
202
+ end
203
+ end
204
+
205
+ def setup_registration_queue
206
+ handler = lambda do |msg|
207
+ begin
208
+ register(serializer.load(msg))
209
+ rescue Exception => e
210
+ Nanite::Log.error("RECV [register] #{e.message}")
211
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
212
+ end
213
+ end
214
+ reg_fanout = amq.fanout('registration', :durable => true)
215
+ if shared_state?
216
+ amq.queue("registration").bind(reg_fanout).subscribe &handler
217
+ else
218
+ amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
219
+ end
220
+ end
221
+
222
+ def setup_request_queue
223
+ handler = lambda do |msg|
224
+ begin
225
+ handle_request(serializer.load(msg))
226
+ rescue Exception => e
227
+ Nanite::Log.error("RECV [request] #{e.message}")
228
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
229
+ end
230
+ end
231
+ req_fanout = amq.fanout('request', :durable => true)
232
+ if shared_state?
233
+ amq.queue("request").bind(req_fanout).subscribe &handler
234
+ else
235
+ amq.queue("request-#{identity}", :exclusive => true).bind(req_fanout).subscribe &handler
236
+ end
237
+ end
238
+
239
+ def setup_state
240
+ case @state
241
+ when String
242
+ # backwards compatibility, we assume redis if the configuration option
243
+ # was a string
244
+ require 'nanite/state'
245
+ @nanites = Nanite::State.new(@state, @tag_store)
246
+ else
247
+ require 'nanite/local_state'
248
+ @nanites = Nanite::LocalState.new
249
+ end
250
+ end
251
+
252
+ def shared_state?
253
+ !@state.nil?
254
+ end
255
+ end
256
+ end
@@ -0,0 +1,111 @@
1
+ module Nanite
2
+
3
+ COMMON_DEFAULT_OPTIONS = {
4
+ :pass => 'testing',
5
+ :vhost => '/nanite',
6
+ :secure => false,
7
+ :host => '0.0.0.0',
8
+ :log_level => :info,
9
+ :format => :marshal,
10
+ :daemonize => false,
11
+ :console => false,
12
+ :root => Dir.pwd
13
+ }
14
+
15
+ module CommonConfig
16
+ def setup_mapper_options(opts, options)
17
+ setup_common_options(opts, options, 'mapper')
18
+
19
+ opts.on("-a", "--agent-timeout TIMEOUT", "How long to wait before an agent is considered to be offline and thus removed from the list of available agents.") do |timeout|
20
+ options[:agent_timeout] = timeout
21
+ end
22
+
23
+ opts.on("-r", "--offline-redelivery-frequency", "The frequency in seconds that messages stored in the offline queue will be retrieved for attempted redelivery to the nanites. Default is 10 seconds.") do |frequency|
24
+ options[:offline_redelivery_frequency] = frequency
25
+ end
26
+
27
+ opts.on("--persistent", "Instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the broker is restarted. Can be overriden on a per-message basis using the request and push methods.") do
28
+ options[:persistent] = true
29
+ end
30
+
31
+ opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do
32
+ options[:offline_failsafe] = true
33
+ end
34
+
35
+ opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis|
36
+ redishost, redisport = redis.split(':')
37
+ redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?)
38
+ redisport = '6379' if (redishost.nil? || redishost.empty?)
39
+ redis = "#{redishost}:#{redisport}"
40
+ options[:redis] = redis
41
+ end
42
+
43
+ end
44
+
45
+ def setup_common_options(opts, options, type)
46
+ opts.version = Nanite::VERSION
47
+
48
+ opts.on("-i", "--irb-console", "Start #{type} in irb console mode.") do |console|
49
+ options[:console] = 'irb'
50
+ end
51
+
52
+ opts.on("-u", "--user USER", "Specify the rabbitmq username.") do |user|
53
+ options[:user] = user
54
+ end
55
+
56
+ opts.on("-h", "--host HOST", "Specify the rabbitmq hostname.") do |host|
57
+ options[:host] = host
58
+ end
59
+
60
+ opts.on("-P", "--port PORT", "Specify the rabbitmq PORT, default 5672.") do |port|
61
+ options[:port] = port
62
+ end
63
+
64
+ opts.on("-p", "--pass PASSWORD", "Specify the rabbitmq password") do |pass|
65
+ options[:pass] = pass
66
+ end
67
+
68
+ opts.on("-t", "--token IDENITY", "Specify the #{type} identity.") do |ident|
69
+ options[:identity] = ident
70
+ end
71
+
72
+ opts.on("-v", "--vhost VHOST", "Specify the rabbitmq vhost") do |vhost|
73
+ options[:vhost] = vhost
74
+ end
75
+
76
+ opts.on("-s", "--secure", "Use Security features of rabbitmq to restrict nanites to themselves") do
77
+ options[:secure] = true
78
+ end
79
+
80
+ opts.on("-f", "--format FORMAT", "The serialization type to use for transfering data. Can be marshal, json or yaml. Default is marshal") do |fmt|
81
+ options[:format] = fmt.to_sym
82
+ end
83
+
84
+ opts.on("-d", "--daemonize", "Run #{type} as a daemon") do |d|
85
+ options[:daemonize] = true
86
+ end
87
+
88
+ opts.on("--pid-dir PATH", "Specify the pid path, only used with daemonize") do |dir|
89
+ options[:pid_dir] = dir
90
+ end
91
+
92
+ opts.on("-l", "--log-level LEVEL", "Specify the log level (fatal, error, warn, info, debug). Default is info") do |level|
93
+ options[:log_level] = level
94
+ end
95
+
96
+ opts.on("--log-dir PATH", "Specify the log path") do |dir|
97
+ options[:log_dir] = dir
98
+ end
99
+
100
+ opts.on("--tag TAG", "Specify a tag. Can issue multiple times.") do |tag|
101
+ options[:tag] ||= []
102
+ options[:tag] << tag
103
+ end
104
+
105
+ opts.on("--version", "Show the nanite version number") do |res|
106
+ puts "Nanite Version #{opts.version}"
107
+ exit
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,39 @@
1
+ module Nanite
2
+ module ConsoleHelper
3
+ def self.included(base)
4
+ @@base = base
5
+ end
6
+
7
+ def start_console
8
+ puts "Starting #{@@base.name.split(":").last.downcase} console (#{self.identity}) (Nanite #{Nanite::VERSION})"
9
+ Thread.new do
10
+ Console.start(self)
11
+ end
12
+ end
13
+ end
14
+
15
+ module Console
16
+ class << self; attr_accessor :instance; end
17
+
18
+ def self.start(binding)
19
+ require 'irb'
20
+ old_args = ARGV.dup
21
+ ARGV.replace ["--simple-prompt"]
22
+
23
+ IRB.setup(nil)
24
+ self.instance = IRB::Irb.new(IRB::WorkSpace.new(binding))
25
+
26
+ @CONF = IRB.instance_variable_get(:@CONF)
27
+ @CONF[:IRB_RC].call self.instance.context if @CONF[:IRB_RC]
28
+ @CONF[:MAIN_CONTEXT] = self.instance.context
29
+
30
+ catch(:IRB_EXIT) { self.instance.eval_input }
31
+ ensure
32
+ ARGV.replace old_args
33
+ # Clean up tty settings in some evil, evil cases
34
+ begin; catch(:IRB_EXIT) { irb_exit }; rescue Exception; end
35
+ # Make nanite exit when irb does
36
+ EM.stop if EM.reactor_running?
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,13 @@
1
+ module Nanite
2
+ module DaemonizeHelper
3
+ def daemonize(identity, options = {})
4
+ exit if fork
5
+ Process.setsid
6
+ exit if fork
7
+ STDIN.reopen "/dev/null"
8
+ STDOUT.reopen "#{options[:log_path]}/nanite.#{identity}.out", "a"
9
+ STDERR.reopen "#{options[:log_path]}/nanite.#{identity}.err", "a"
10
+ File.umask 0000
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,16 @@
1
+ module Nanite
2
+ module Identity
3
+ def self.generate
4
+ values = [
5
+ rand(0x0010000),
6
+ rand(0x0010000),
7
+ rand(0x0010000),
8
+ rand(0x0010000),
9
+ rand(0x0010000),
10
+ rand(0x1000000),
11
+ rand(0x1000000),
12
+ ]
13
+ "%04x%04x%04x%04x%04x%06x%06x" % values
14
+ end
15
+ end
16
+ end
data/lib/nanite/job.rb ADDED
@@ -0,0 +1,104 @@
1
+ module Nanite
2
+ class JobWarden
3
+ attr_reader :serializer, :jobs
4
+
5
+ def initialize(serializer)
6
+ @serializer = serializer
7
+ @jobs = {}
8
+ end
9
+
10
+ def new_job(request, targets, inthandler = nil, blk = nil)
11
+ job = Job.new(request, targets, inthandler, blk)
12
+ jobs[job.token] = job
13
+ job
14
+ end
15
+
16
+ def process(msg)
17
+ if job = jobs[msg.token]
18
+ job.process(msg)
19
+
20
+ if job.intermediate_handler && (job.pending_keys.size > 0)
21
+
22
+ unless job.pending_keys.size == 1
23
+ raise "IntermediateMessages are currently dispatched as they arrive, shouldn't have more than one key in pending_keys: #{job.pending_keys.inspect}"
24
+ end
25
+
26
+ key = job.pending_keys.first
27
+ handler = job.intermediate_handler_for_key(key)
28
+ if handler
29
+ case handler.arity
30
+ when 2
31
+ handler.call(job.intermediate_state[msg.from][key].last, job)
32
+ when 3
33
+ handler.call(key, msg.from, job.intermediate_state[msg.from][key].last)
34
+ when 4
35
+ handler.call(key, msg.from, job.intermediate_state[msg.from][key].last, job)
36
+ end
37
+ end
38
+
39
+ job.reset_pending_intermediate_state_keys
40
+ end
41
+
42
+ if job.completed?
43
+ jobs.delete(job.token)
44
+ if job.completed
45
+ case job.completed.arity
46
+ when 1
47
+ job.completed.call(job.results)
48
+ when 2
49
+ job.completed.call(job.results, job)
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end # JobWarden
56
+
57
+ class Job
58
+ attr_reader :results, :request, :token, :completed, :intermediate_state, :pending_keys, :intermediate_handler
59
+ attr_accessor :targets # This can be updated when a request gets picked up from the offline queue
60
+
61
+ def initialize(request, targets, inthandler = nil, blk = nil)
62
+ @request = request
63
+ @targets = targets
64
+ @token = @request.token
65
+ @results = {}
66
+ @intermediate_handler = inthandler
67
+ @pending_keys = []
68
+ @completed = blk
69
+ @intermediate_state = {}
70
+ end
71
+
72
+ def process(msg)
73
+ case msg
74
+ when Result
75
+ results[msg.from] = msg.results
76
+ targets.delete(msg.from)
77
+ when IntermediateMessage
78
+ intermediate_state[msg.from] ||= {}
79
+ intermediate_state[msg.from][msg.messagekey] ||= []
80
+ intermediate_state[msg.from][msg.messagekey] << msg.message
81
+ @pending_keys << msg.messagekey
82
+ end
83
+ end
84
+
85
+ def intermediate_handler_for_key(key)
86
+ return nil unless @intermediate_handler
87
+ case @intermediate_handler
88
+ when Proc
89
+ @intermediate_handler
90
+ when Hash
91
+ @intermediate_handler[key] || @intermediate_handler['*']
92
+ end
93
+ end
94
+
95
+ def reset_pending_intermediate_state_keys
96
+ @pending_keys = []
97
+ end
98
+
99
+ def completed?
100
+ targets.empty?
101
+ end
102
+ end # Job
103
+
104
+ end # Nanite