rightscale-nanite 0.4.1 → 0.4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/nanite.rb +71 -0
- data/lib/nanite/actor.rb +60 -0
- data/lib/nanite/actor_registry.rb +24 -0
- data/lib/nanite/admin.rb +153 -0
- data/lib/nanite/agent.rb +250 -0
- data/lib/nanite/amqp.rb +47 -0
- data/lib/nanite/cluster.rb +203 -0
- data/lib/nanite/config.rb +102 -0
- data/lib/nanite/console.rb +39 -0
- data/lib/nanite/daemonize.rb +13 -0
- data/lib/nanite/dispatcher.rb +90 -0
- data/lib/nanite/identity.rb +16 -0
- data/lib/nanite/job.rb +104 -0
- data/lib/nanite/local_state.rb +34 -0
- data/lib/nanite/log.rb +64 -0
- data/lib/nanite/log/formatter.rb +39 -0
- data/lib/nanite/mapper.rb +277 -0
- data/lib/nanite/mapper_proxy.rb +56 -0
- data/lib/nanite/packets.rb +231 -0
- data/lib/nanite/pid_file.rb +52 -0
- data/lib/nanite/reaper.rb +38 -0
- data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
- data/lib/nanite/security/certificate.rb +55 -0
- data/lib/nanite/security/certificate_cache.rb +66 -0
- data/lib/nanite/security/distinguished_name.rb +34 -0
- data/lib/nanite/security/encrypted_document.rb +46 -0
- data/lib/nanite/security/rsa_key_pair.rb +53 -0
- data/lib/nanite/security/secure_serializer.rb +67 -0
- data/lib/nanite/security/signature.rb +40 -0
- data/lib/nanite/security/static_certificate_store.rb +35 -0
- data/lib/nanite/security_provider.rb +47 -0
- data/lib/nanite/serializer.rb +52 -0
- data/lib/nanite/state.rb +164 -0
- data/lib/nanite/streaming.rb +125 -0
- data/lib/nanite/util.rb +51 -0
- data/spec/actor_registry_spec.rb +62 -0
- data/spec/actor_spec.rb +59 -0
- data/spec/agent_spec.rb +235 -0
- data/spec/cached_certificate_store_proxy_spec.rb +34 -0
- data/spec/certificate_cache_spec.rb +49 -0
- data/spec/certificate_spec.rb +27 -0
- data/spec/cluster_spec.rb +300 -0
- data/spec/dispatcher_spec.rb +136 -0
- data/spec/distinguished_name_spec.rb +24 -0
- data/spec/encrypted_document_spec.rb +21 -0
- data/spec/job_spec.rb +219 -0
- data/spec/local_state_spec.rb +112 -0
- data/spec/packet_spec.rb +218 -0
- data/spec/rsa_key_pair_spec.rb +33 -0
- data/spec/secure_serializer_spec.rb +41 -0
- data/spec/serializer_spec.rb +107 -0
- data/spec/signature_spec.rb +30 -0
- data/spec/spec_helper.rb +23 -0
- data/spec/static_certificate_store_spec.rb +30 -0
- data/spec/util_spec.rb +63 -0
- metadata +62 -1
@@ -0,0 +1,56 @@
|
|
1
|
+
module Nanite
|
2
|
+
|
3
|
+
# This class allows sending requests to nanite agents without having
|
4
|
+
# to run a local mapper.
|
5
|
+
# It is used by Actor.request which can be used by actors than need
|
6
|
+
# to send requests to remote agents.
|
7
|
+
# All requests go through the mapper for security purposes.
|
8
|
+
class MapperProxy
|
9
|
+
|
10
|
+
$:.push File.dirname(__FILE__)
|
11
|
+
require 'amqp'
|
12
|
+
|
13
|
+
include AMQPHelper
|
14
|
+
|
15
|
+
attr_accessor :pending_requests, :identity, :options, :amqp, :serializer
|
16
|
+
|
17
|
+
# Accessor for actor
|
18
|
+
def self.instance
|
19
|
+
@@instance
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(id, opts)
|
23
|
+
@options = opts || {}
|
24
|
+
@identity = id
|
25
|
+
@pending_requests = {}
|
26
|
+
@amqp = start_amqp(options)
|
27
|
+
@serializer = Serializer.new(options[:format])
|
28
|
+
@@instance = self
|
29
|
+
end
|
30
|
+
|
31
|
+
# Send request to given agent through the mapper
|
32
|
+
def request(type, payload = '', opts = {}, &blk)
|
33
|
+
raise "Mapper proxy not initialized" unless identity && options
|
34
|
+
request = Request.new(type, payload, opts)
|
35
|
+
request.from = identity
|
36
|
+
request.token = Identity.generate
|
37
|
+
request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
|
38
|
+
pending_requests[request.token] =
|
39
|
+
{ :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
|
40
|
+
amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
|
41
|
+
end
|
42
|
+
|
43
|
+
# Handle intermediary result
|
44
|
+
def handle_intermediate_result(res)
|
45
|
+
handlers = pending_requests[res.token]
|
46
|
+
handlers[:intermediate_handler].call(res) if handlers && handlers[:intermediate_handler]
|
47
|
+
end
|
48
|
+
|
49
|
+
# Handle final result
|
50
|
+
def handle_result(res)
|
51
|
+
handlers = pending_requests.delete(res.token)
|
52
|
+
handlers[:result_handler].call(res) if handlers && handlers[:result_handler]
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,231 @@
|
|
1
|
+
module Nanite
|
2
|
+
# Base class for all Nanite packets,
|
3
|
+
# knows how to dump itself to JSON
|
4
|
+
class Packet
|
5
|
+
def initialize
|
6
|
+
raise NotImplementedError.new("#{self.class.name} is an abstract class.")
|
7
|
+
end
|
8
|
+
def to_json(*a)
|
9
|
+
{
|
10
|
+
'json_class' => self.class.name,
|
11
|
+
'data' => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
|
12
|
+
}.to_json(*a)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# packet that means start of a file transfer
|
17
|
+
# operation
|
18
|
+
class FileStart < Packet
|
19
|
+
attr_accessor :filename, :token, :dest
|
20
|
+
def initialize(filename, dest, token)
|
21
|
+
@filename = filename
|
22
|
+
@dest = dest
|
23
|
+
@token = token
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.json_create(o)
|
27
|
+
i = o['data']
|
28
|
+
new(i['filename'], i['dest'], i['token'])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# packet that means end of a file transfer
|
33
|
+
# operation
|
34
|
+
class FileEnd < Packet
|
35
|
+
attr_accessor :token, :meta
|
36
|
+
def initialize(token, meta)
|
37
|
+
@token = token
|
38
|
+
@meta = meta
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.json_create(o)
|
42
|
+
i = o['data']
|
43
|
+
new(i['token'], i['meta'])
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# packet that carries data chunks during a file transfer
|
48
|
+
class FileChunk < Packet
|
49
|
+
attr_accessor :chunk, :token
|
50
|
+
def initialize(token, chunk=nil)
|
51
|
+
@chunk = chunk
|
52
|
+
@token = token
|
53
|
+
end
|
54
|
+
def self.json_create(o)
|
55
|
+
i = o['data']
|
56
|
+
new(i['token'], i['chunk'])
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# packet that means a work request from mapper
|
61
|
+
# to actor node
|
62
|
+
#
|
63
|
+
# type is a service name
|
64
|
+
# payload is arbitrary data that is transferred from mapper to actor
|
65
|
+
#
|
66
|
+
# Options:
|
67
|
+
# from is sender identity
|
68
|
+
# token is a generated request id that mapper uses to identify replies
|
69
|
+
# reply_to is identity of the node actor replies to, usually a mapper itself
|
70
|
+
# selector is the selector used to route the request
|
71
|
+
# target is the target nanite for the request
|
72
|
+
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
73
|
+
class Request < Packet
|
74
|
+
attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
75
|
+
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
76
|
+
def initialize(type, payload, opts={})
|
77
|
+
opts = DEFAULT_OPTIONS.merge(opts)
|
78
|
+
@type = type
|
79
|
+
@payload = payload
|
80
|
+
@from = opts[:from]
|
81
|
+
@token = opts[:token]
|
82
|
+
@reply_to = opts[:reply_to]
|
83
|
+
@selector = opts[:selector]
|
84
|
+
@target = opts[:target]
|
85
|
+
@persistent = opts[:persistent]
|
86
|
+
@tags = opts[:tags] || []
|
87
|
+
end
|
88
|
+
def self.json_create(o)
|
89
|
+
i = o['data']
|
90
|
+
new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :reply_to => i['reply_to'], :selector => i['selector'],
|
91
|
+
:target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# packet that means a work push from mapper
|
96
|
+
# to actor node
|
97
|
+
#
|
98
|
+
# type is a service name
|
99
|
+
# payload is arbitrary data that is transferred from mapper to actor
|
100
|
+
#
|
101
|
+
# Options:
|
102
|
+
# from is sender identity
|
103
|
+
# token is a generated request id that mapper uses to identify replies
|
104
|
+
# selector is the selector used to route the request
|
105
|
+
# target is the target nanite for the request
|
106
|
+
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
107
|
+
class Push < Packet
|
108
|
+
attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
|
109
|
+
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
110
|
+
def initialize(type, payload, opts={})
|
111
|
+
opts = DEFAULT_OPTIONS.merge(opts)
|
112
|
+
@type = type
|
113
|
+
@payload = payload
|
114
|
+
@from = opts[:from]
|
115
|
+
@token = opts[:token]
|
116
|
+
@selector = opts[:selector]
|
117
|
+
@target = opts[:target]
|
118
|
+
@persistent = opts[:persistent]
|
119
|
+
@tags = opts[:tags] || []
|
120
|
+
end
|
121
|
+
def self.json_create(o)
|
122
|
+
i = o['data']
|
123
|
+
new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :selector => i['selector'],
|
124
|
+
:target => i['target'], :persistent => i['persistent'], :tags => i['tags']})
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# packet that means a work result notification sent from actor to mapper
|
129
|
+
#
|
130
|
+
# from is sender identity
|
131
|
+
# results is arbitrary data that is transferred from actor, a result of actor's work
|
132
|
+
# token is a generated request id that mapper uses to identify replies
|
133
|
+
# to is identity of the node result should be delivered to
|
134
|
+
class Result < Packet
|
135
|
+
attr_accessor :token, :results, :to, :from
|
136
|
+
def initialize(token, to, results, from)
|
137
|
+
@token = token
|
138
|
+
@to = to
|
139
|
+
@from = from
|
140
|
+
@results = results
|
141
|
+
end
|
142
|
+
def self.json_create(o)
|
143
|
+
i = o['data']
|
144
|
+
new(i['token'], i['to'], i['results'], i['from'])
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
# packet that means an intermediate status notification sent from actor to mapper. is appended to a list of messages matching messagekey.
|
149
|
+
#
|
150
|
+
# from is sender identity
|
151
|
+
# messagekey is a string that can become part of a redis key, which identifies the name under which the message is stored
|
152
|
+
# message is arbitrary data that is transferred from actor, an intermediate result of actor's work
|
153
|
+
# token is a generated request id that mapper uses to identify replies
|
154
|
+
# to is identity of the node result should be delivered to
|
155
|
+
class IntermediateMessage < Packet
|
156
|
+
attr_accessor :token, :messagekey, :message, :to, :from
|
157
|
+
def initialize(token, to, from, messagekey, message)
|
158
|
+
@token = token
|
159
|
+
@to = to
|
160
|
+
@from = from
|
161
|
+
@messagekey = messagekey
|
162
|
+
@message = message
|
163
|
+
end
|
164
|
+
def self.json_create(o)
|
165
|
+
i = o['data']
|
166
|
+
new(i['token'], i['to'], i['from'], i['messagekey'], i['message'])
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
# packet that means an availability notification sent from actor to mapper
|
171
|
+
#
|
172
|
+
# from is sender identity
|
173
|
+
# services is a list of services provided by the node
|
174
|
+
# status is a load of the node by default, but may be any criteria
|
175
|
+
# agent may use to report it's availability, load, etc
|
176
|
+
class Register < Packet
|
177
|
+
attr_accessor :identity, :services, :status, :tags
|
178
|
+
def initialize(identity, services, status, tags)
|
179
|
+
@status = status
|
180
|
+
@tags = tags
|
181
|
+
@identity = identity
|
182
|
+
@services = services
|
183
|
+
end
|
184
|
+
def self.json_create(o)
|
185
|
+
i = o['data']
|
186
|
+
new(i['identity'], i['services'], i['status'], i['tags'])
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
# packet that means deregister an agent from the mappers
|
191
|
+
#
|
192
|
+
# from is sender identity
|
193
|
+
class UnRegister < Packet
|
194
|
+
attr_accessor :identity
|
195
|
+
def initialize(identity)
|
196
|
+
@identity = identity
|
197
|
+
end
|
198
|
+
def self.json_create(o)
|
199
|
+
i = o['data']
|
200
|
+
new(i['identity'])
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# heartbeat packet
|
205
|
+
#
|
206
|
+
# identity is sender's identity
|
207
|
+
# status is sender's status (see Register packet documentation)
|
208
|
+
class Ping < Packet
|
209
|
+
attr_accessor :identity, :status
|
210
|
+
def initialize(identity, status)
|
211
|
+
@status = status
|
212
|
+
@identity = identity
|
213
|
+
end
|
214
|
+
def self.json_create(o)
|
215
|
+
i = o['data']
|
216
|
+
new(i['identity'], i['status'])
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
# packet that is sent by workers to the mapper
|
221
|
+
# when worker initially comes online to advertise
|
222
|
+
# it's services
|
223
|
+
class Advertise < Packet
|
224
|
+
def initialize
|
225
|
+
end
|
226
|
+
def self.json_create(o)
|
227
|
+
new
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module Nanite
|
2
|
+
class PidFile
|
3
|
+
def initialize(identity, options)
|
4
|
+
@pid_dir = File.expand_path(options[:pid_dir] || options[:root] || Dir.pwd)
|
5
|
+
@pid_file = File.join(@pid_dir, "nanite.#{identity}.pid")
|
6
|
+
end
|
7
|
+
|
8
|
+
def check
|
9
|
+
if pid = read_pid
|
10
|
+
if process_running? pid
|
11
|
+
raise "#{@pid_file} already exists (pid: #{pid})"
|
12
|
+
else
|
13
|
+
Log.info "removing stale pid file: #{@pid_file}"
|
14
|
+
remove
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def ensure_dir
|
20
|
+
FileUtils.mkdir_p @pid_dir
|
21
|
+
end
|
22
|
+
|
23
|
+
def write
|
24
|
+
ensure_dir
|
25
|
+
open(@pid_file,'w') {|f| f.write(Process.pid) }
|
26
|
+
File.chmod(0644, @pid_file)
|
27
|
+
end
|
28
|
+
|
29
|
+
def remove
|
30
|
+
File.delete(@pid_file) if exists?
|
31
|
+
end
|
32
|
+
|
33
|
+
def read_pid
|
34
|
+
open(@pid_file,'r') {|f| f.read.to_i } if exists?
|
35
|
+
end
|
36
|
+
|
37
|
+
def exists?
|
38
|
+
File.exists? @pid_file
|
39
|
+
end
|
40
|
+
|
41
|
+
def to_s
|
42
|
+
@pid_file
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
def process_running?(pid)
|
47
|
+
Process.getpgid(pid) != -1
|
48
|
+
rescue Errno::ESRCH
|
49
|
+
false
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Nanite
|
2
|
+
class Reaper
|
3
|
+
|
4
|
+
def initialize(frequency=2)
|
5
|
+
@timeouts = {}
|
6
|
+
EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
|
7
|
+
end
|
8
|
+
|
9
|
+
def timeout(token, seconds, &blk)
|
10
|
+
@timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
|
11
|
+
end
|
12
|
+
|
13
|
+
def reset_with_autoregister_hack(token,seconds,&blk)
|
14
|
+
unless @timeouts[token]
|
15
|
+
timeout(token, seconds, &blk)
|
16
|
+
end
|
17
|
+
reset(token)
|
18
|
+
end
|
19
|
+
|
20
|
+
def reset(token)
|
21
|
+
@timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def reap
|
27
|
+
time = Time.now
|
28
|
+
@timeouts.reject! do |token, data|
|
29
|
+
if time > data[:timestamp]
|
30
|
+
data[:callback].call
|
31
|
+
true
|
32
|
+
else
|
33
|
+
false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Nanite
|
2
|
+
|
3
|
+
# Proxy to actual certificate store which caches results in an LRU
|
4
|
+
# cache.
|
5
|
+
class CachedCertificateStoreProxy
|
6
|
+
|
7
|
+
# Initialize cache proxy with given certificate store.
|
8
|
+
def initialize(store)
|
9
|
+
@signer_cache = CertificateCache.new
|
10
|
+
@store = store
|
11
|
+
end
|
12
|
+
|
13
|
+
# Results from 'get_recipients' are not cached
|
14
|
+
def get_recipients(obj)
|
15
|
+
@store.get_recipients(obj)
|
16
|
+
end
|
17
|
+
|
18
|
+
# Check cache for signer certificate
|
19
|
+
def get_signer(id)
|
20
|
+
@signer_cache.get(id) { @store.get_signer(id) }
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Nanite
|
2
|
+
|
3
|
+
# X.509 Certificate management
|
4
|
+
class Certificate
|
5
|
+
|
6
|
+
# Underlying OpenSSL cert
|
7
|
+
attr_accessor :raw_cert
|
8
|
+
|
9
|
+
# Generate a signed X.509 certificate
|
10
|
+
#
|
11
|
+
# Arguments:
|
12
|
+
# - key: RsaKeyPair, key pair used to sign certificate
|
13
|
+
# - issuer: DistinguishedName, certificate issuer
|
14
|
+
# - subject: DistinguishedName, certificate subject
|
15
|
+
# - valid_for: Time in seconds before certificate expires (10 years by default)
|
16
|
+
def initialize(key, issuer, subject, valid_for = 3600*24*365*10)
|
17
|
+
@raw_cert = OpenSSL::X509::Certificate.new
|
18
|
+
@raw_cert.version = 2
|
19
|
+
@raw_cert.serial = 1
|
20
|
+
@raw_cert.subject = subject.to_x509
|
21
|
+
@raw_cert.issuer = issuer.to_x509
|
22
|
+
@raw_cert.public_key = key.to_public.raw_key
|
23
|
+
@raw_cert.not_before = Time.now
|
24
|
+
@raw_cert.not_after = Time.now + valid_for
|
25
|
+
@raw_cert.sign(key.raw_key, OpenSSL::Digest::SHA1.new)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Load certificate from file
|
29
|
+
def self.load(file)
|
30
|
+
from_data(File.new(file))
|
31
|
+
end
|
32
|
+
|
33
|
+
# Initialize with raw certificate
|
34
|
+
def self.from_data(data)
|
35
|
+
cert = OpenSSL::X509::Certificate.new(data)
|
36
|
+
res = Certificate.allocate
|
37
|
+
res.instance_variable_set(:@raw_cert, cert)
|
38
|
+
res
|
39
|
+
end
|
40
|
+
|
41
|
+
# Save certificate to file in PEM format
|
42
|
+
def save(file)
|
43
|
+
File.open(file, "w") do |f|
|
44
|
+
f.write(@raw_cert.to_pem)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Certificate data in PEM format
|
49
|
+
def data
|
50
|
+
@raw_cert.to_pem
|
51
|
+
end
|
52
|
+
alias :to_s :data
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|