rightscale-nanite 0.4.1 → 0.4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/nanite.rb +71 -0
- data/lib/nanite/actor.rb +60 -0
- data/lib/nanite/actor_registry.rb +24 -0
- data/lib/nanite/admin.rb +153 -0
- data/lib/nanite/agent.rb +250 -0
- data/lib/nanite/amqp.rb +47 -0
- data/lib/nanite/cluster.rb +203 -0
- data/lib/nanite/config.rb +102 -0
- data/lib/nanite/console.rb +39 -0
- data/lib/nanite/daemonize.rb +13 -0
- data/lib/nanite/dispatcher.rb +90 -0
- data/lib/nanite/identity.rb +16 -0
- data/lib/nanite/job.rb +104 -0
- data/lib/nanite/local_state.rb +34 -0
- data/lib/nanite/log.rb +64 -0
- data/lib/nanite/log/formatter.rb +39 -0
- data/lib/nanite/mapper.rb +277 -0
- data/lib/nanite/mapper_proxy.rb +56 -0
- data/lib/nanite/packets.rb +231 -0
- data/lib/nanite/pid_file.rb +52 -0
- data/lib/nanite/reaper.rb +38 -0
- data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
- data/lib/nanite/security/certificate.rb +55 -0
- data/lib/nanite/security/certificate_cache.rb +66 -0
- data/lib/nanite/security/distinguished_name.rb +34 -0
- data/lib/nanite/security/encrypted_document.rb +46 -0
- data/lib/nanite/security/rsa_key_pair.rb +53 -0
- data/lib/nanite/security/secure_serializer.rb +67 -0
- data/lib/nanite/security/signature.rb +40 -0
- data/lib/nanite/security/static_certificate_store.rb +35 -0
- data/lib/nanite/security_provider.rb +47 -0
- data/lib/nanite/serializer.rb +52 -0
- data/lib/nanite/state.rb +164 -0
- data/lib/nanite/streaming.rb +125 -0
- data/lib/nanite/util.rb +51 -0
- data/spec/actor_registry_spec.rb +62 -0
- data/spec/actor_spec.rb +59 -0
- data/spec/agent_spec.rb +235 -0
- data/spec/cached_certificate_store_proxy_spec.rb +34 -0
- data/spec/certificate_cache_spec.rb +49 -0
- data/spec/certificate_spec.rb +27 -0
- data/spec/cluster_spec.rb +300 -0
- data/spec/dispatcher_spec.rb +136 -0
- data/spec/distinguished_name_spec.rb +24 -0
- data/spec/encrypted_document_spec.rb +21 -0
- data/spec/job_spec.rb +219 -0
- data/spec/local_state_spec.rb +112 -0
- data/spec/packet_spec.rb +218 -0
- data/spec/rsa_key_pair_spec.rb +33 -0
- data/spec/secure_serializer_spec.rb +41 -0
- data/spec/serializer_spec.rb +107 -0
- data/spec/signature_spec.rb +30 -0
- data/spec/spec_helper.rb +23 -0
- data/spec/static_certificate_store_spec.rb +30 -0
- data/spec/util_spec.rb +63 -0
- metadata +62 -1
data/lib/nanite/state.rb
ADDED
@@ -0,0 +1,164 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
module Nanite
|
4
|
+
class State
|
5
|
+
include Enumerable
|
6
|
+
|
7
|
+
# this class encapsulates the state of a nanite system using redis as the
|
8
|
+
# data store. here is the schema, for each agent we store a number of items,
|
9
|
+
# for a nanite with the identity: nanite-foobar we store the following things:
|
10
|
+
#
|
11
|
+
# nanite-foobar: 0.72 # load average or 'status'
|
12
|
+
# s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
|
13
|
+
# tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
|
14
|
+
# t-nanite-foobar: 123456789 # unix timestamp of the last state update
|
15
|
+
#
|
16
|
+
# also we do an inverted index for quick lookup of agents providing a certain
|
17
|
+
# service, so for each service the agent provides, we add the nanite to a SET
|
18
|
+
# of all the nanites that provide said service:
|
19
|
+
#
|
20
|
+
# /gems/list: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
|
21
|
+
#
|
22
|
+
# we do that same thing for tags:
|
23
|
+
# some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
|
24
|
+
#
|
25
|
+
# This way we can do a lookup of what nanites provide a set of services and tags based
|
26
|
+
# on redis SET intersection:
|
27
|
+
#
|
28
|
+
# nanites_for('/gems/list', 'some-tag')
|
29
|
+
# => returns a nested array of nanites and their state that provide the intersection
|
30
|
+
# of these two service tags
|
31
|
+
|
32
|
+
def initialize(redis)
|
33
|
+
Nanite::Log.info("initializing redis state: #{redis}")
|
34
|
+
host, port = redis.split(':')
|
35
|
+
host ||= '127.0.0.1'
|
36
|
+
port ||= '6379'
|
37
|
+
@redis = Redis.new :host => host, :port => port
|
38
|
+
end
|
39
|
+
|
40
|
+
def log_redis_error(meth,&blk)
|
41
|
+
blk.call
|
42
|
+
rescue RedisError => e
|
43
|
+
Nanite::Log.info("redis error in method: #{meth}")
|
44
|
+
raise e
|
45
|
+
end
|
46
|
+
|
47
|
+
def [](nanite)
|
48
|
+
log_redis_error("[]") do
|
49
|
+
status = @redis[nanite]
|
50
|
+
timestamp = @redis["t-#{nanite}"]
|
51
|
+
services = @redis.set_members("s-#{nanite}")
|
52
|
+
tags = @redis.set_members("tg-#{nanite}")
|
53
|
+
return nil unless status && timestamp && services
|
54
|
+
{:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def []=(nanite, hsh)
|
59
|
+
log_redis_error("[]=") do
|
60
|
+
update_state(nanite, hsh[:status], hsh[:services], hsh[:tags])
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def delete(nanite)
|
65
|
+
log_redis_error("delete") do
|
66
|
+
(@redis.set_members("s-#{nanite}")||[]).each do |srv|
|
67
|
+
@redis.set_delete(srv, nanite)
|
68
|
+
if @redis.set_count(srv) == 0
|
69
|
+
@redis.delete(srv)
|
70
|
+
@redis.set_delete("naniteservices", srv)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
(@redis.set_members("tg-#{nanite}")||[]).each do |tag|
|
74
|
+
@redis.set_delete(tag, nanite)
|
75
|
+
if @redis.set_count(tag) == 0
|
76
|
+
@redis.delete(tag)
|
77
|
+
@redis.set_delete("nanitetags", tag)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
@redis.delete nanite
|
81
|
+
@redis.delete "s-#{nanite}"
|
82
|
+
@redis.delete "t-#{nanite}"
|
83
|
+
@redis.delete "tg-#{nanite}"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def all_services
|
88
|
+
log_redis_error("all_services") do
|
89
|
+
@redis.set_members("naniteservices")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def all_tags
|
94
|
+
log_redis_error("all_tags") do
|
95
|
+
@redis.set_members("nanitetags")
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def update_state(name, status, services, tags)
|
100
|
+
old_services = @redis.set_members("s-#{name}")
|
101
|
+
if old_services
|
102
|
+
(old_services - services).each do |s|
|
103
|
+
@redis.set_delete(s, name)
|
104
|
+
@redis.set_delete("naniteservices", s)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
old_tags = @redis.set_members("tg-#{name}")
|
108
|
+
if old_tags
|
109
|
+
(old_tags - tags).each do |t|
|
110
|
+
@redis.set_delete(t, name)
|
111
|
+
@redis.set_delete("nanitetags", t)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
@redis.delete("s-#{name}")
|
115
|
+
services.each do |srv|
|
116
|
+
@redis.set_add(srv, name)
|
117
|
+
@redis.set_add("s-#{name}", srv)
|
118
|
+
@redis.set_add("naniteservices", srv)
|
119
|
+
end
|
120
|
+
@redis.delete("tg-#{name}")
|
121
|
+
tags.each do |tag|
|
122
|
+
next if tag.nil?
|
123
|
+
@redis.set_add(tag, name)
|
124
|
+
@redis.set_add("tg-#{name}", tag)
|
125
|
+
@redis.set_add("nanitetags", tag)
|
126
|
+
end
|
127
|
+
@redis[name] = status
|
128
|
+
@redis["t-#{name}"] = Time.now.to_i
|
129
|
+
end
|
130
|
+
|
131
|
+
def list_nanites
|
132
|
+
log_redis_error("list_nanites") do
|
133
|
+
@redis.keys("nanite-*")
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def size
|
138
|
+
list_nanites.size
|
139
|
+
end
|
140
|
+
|
141
|
+
def clear_state
|
142
|
+
log_redis_error("clear_state") do
|
143
|
+
@redis.keys("*").each {|k| @redis.delete k}
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def each
|
148
|
+
list_nanites.each do |nan|
|
149
|
+
yield nan, self[nan]
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def nanites_for(service, *tags)
|
154
|
+
keys = tags.dup << service
|
155
|
+
log_redis_error("nanites_for") do
|
156
|
+
res = []
|
157
|
+
(@redis.set_intersect(keys)||[]).each do |nan|
|
158
|
+
res << [nan, self[nan]]
|
159
|
+
end
|
160
|
+
res
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
module Nanite
|
2
|
+
# Nanite actors can transfer files to each other.
|
3
|
+
#
|
4
|
+
# ==== Options
|
5
|
+
#
|
6
|
+
# filename : you guessed it, name of the file!
|
7
|
+
# domain : part of the routing key used to locate receiver(s)
|
8
|
+
# destination : is a name of the file as it gonna be stored at the destination
|
9
|
+
# meta :
|
10
|
+
#
|
11
|
+
# File streaming is done in chunks. When file streaming starts,
|
12
|
+
# Nanite::FileStart packet is sent, followed by one or more (usually more ;))
|
13
|
+
# Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
|
14
|
+
# Nanite::FileEnd packet is sent.
|
15
|
+
#
|
16
|
+
# 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
|
17
|
+
# more than that in one operation via socket.
|
18
|
+
#
|
19
|
+
# ==== Domains
|
20
|
+
#
|
21
|
+
# Streaming happens using a topic exchange called 'file broadcast', with keys
|
22
|
+
# formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
|
23
|
+
# receivers find each other in the cluster. Default domain is 'global'.
|
24
|
+
#
|
25
|
+
# Domains also serve as a way to register a callback Nanite agent executes once file
|
26
|
+
# streaming is completed. If a callback with name of domain is registered, it is called.
|
27
|
+
#
|
28
|
+
# Callbacks are registered by passing a block to subscribe_to_files method.
|
29
|
+
module FileStreaming
|
30
|
+
def broadcast_file(filename, options = {})
|
31
|
+
if File.exist?(filename)
|
32
|
+
File.open(filename, 'rb') do |file|
|
33
|
+
broadcast_data(filename, file, options)
|
34
|
+
end
|
35
|
+
else
|
36
|
+
return "file not found"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def broadcast_data(filename, io, options = {})
|
41
|
+
domain = options[:domain] || 'global'
|
42
|
+
filename = File.basename(filename)
|
43
|
+
dest = options[:destination] || filename
|
44
|
+
sent = 0
|
45
|
+
|
46
|
+
begin
|
47
|
+
file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
|
48
|
+
amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
|
49
|
+
res = Nanite::FileChunk.new(file_push.token)
|
50
|
+
while chunk = io.read(16384)
|
51
|
+
res.chunk = chunk
|
52
|
+
amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
|
53
|
+
sent += chunk.length
|
54
|
+
end
|
55
|
+
fend = Nanite::FileEnd.new(file_push.token, options[:meta])
|
56
|
+
amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
|
57
|
+
""
|
58
|
+
ensure
|
59
|
+
io.close
|
60
|
+
end
|
61
|
+
|
62
|
+
sent
|
63
|
+
end
|
64
|
+
|
65
|
+
# FileState represents a file download in progress.
|
66
|
+
# It incapsulates the following information:
|
67
|
+
#
|
68
|
+
# * unique operation token
|
69
|
+
# * domain (namespace for file streaming operations)
|
70
|
+
# * file IO chunks are written to on receiver's side
|
71
|
+
class FileState
|
72
|
+
|
73
|
+
def initialize(token, dest, domain, write, blk)
|
74
|
+
@token = token
|
75
|
+
@cb = blk
|
76
|
+
@domain = domain
|
77
|
+
@write = write
|
78
|
+
|
79
|
+
if write
|
80
|
+
@filename = File.join(Nanite.agent.file_root, dest)
|
81
|
+
@dest = File.open(@filename, 'wb')
|
82
|
+
else
|
83
|
+
@dest = dest
|
84
|
+
end
|
85
|
+
|
86
|
+
@data = ""
|
87
|
+
end
|
88
|
+
|
89
|
+
def handle_packet(packet)
|
90
|
+
case packet
|
91
|
+
when Nanite::FileChunk
|
92
|
+
Nanite::Log.debug "written chunk to #{@dest.inspect}"
|
93
|
+
@data << packet.chunk
|
94
|
+
|
95
|
+
if @write
|
96
|
+
@dest.write(packet.chunk)
|
97
|
+
end
|
98
|
+
when Nanite::FileEnd
|
99
|
+
Nanite::Log.debug "#{@dest.inspect} receiving is completed"
|
100
|
+
if @write
|
101
|
+
@dest.close
|
102
|
+
end
|
103
|
+
|
104
|
+
@cb.call(@data, @dest, packet.meta)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
def subscribe_to_files(domain='global', write=false, &blk)
|
111
|
+
Nanite::Log.info "subscribing to file broadcasts for #{domain}"
|
112
|
+
@files ||= {}
|
113
|
+
amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
|
114
|
+
case msg = serializer.load(packet)
|
115
|
+
when FileStart
|
116
|
+
@files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
|
117
|
+
when FileChunk, FileEnd
|
118
|
+
if file = @files[msg.token]
|
119
|
+
file.handle_packet(msg)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
data/lib/nanite/util.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
class String
|
2
|
+
##
|
3
|
+
# Convert to snake case.
|
4
|
+
#
|
5
|
+
# "FooBar".snake_case #=> "foo_bar"
|
6
|
+
# "HeadlineCNNNews".snake_case #=> "headline_cnn_news"
|
7
|
+
# "CNN".snake_case #=> "cnn"
|
8
|
+
#
|
9
|
+
# @return [String] Receiver converted to snake case.
|
10
|
+
#
|
11
|
+
# @api public
|
12
|
+
def snake_case
|
13
|
+
return self.downcase if self =~ /^[A-Z]+$/
|
14
|
+
self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
|
15
|
+
return $+.downcase
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# Convert a constant name to a path, assuming a conventional structure.
|
20
|
+
#
|
21
|
+
# "FooBar::Baz".to_const_path # => "foo_bar/baz"
|
22
|
+
#
|
23
|
+
# @return [String] Path to the file containing the constant named by receiver
|
24
|
+
# (constantized string), assuming a conventional structure.
|
25
|
+
#
|
26
|
+
# @api public
|
27
|
+
def to_const_path
|
28
|
+
snake_case.gsub(/::/, "/")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class Object
|
33
|
+
module InstanceExecHelper; end
|
34
|
+
include InstanceExecHelper
|
35
|
+
def instance_exec(*args, &block)
|
36
|
+
begin
|
37
|
+
old_critical, Thread.critical = Thread.critical, true
|
38
|
+
n = 0
|
39
|
+
n += 1 while respond_to?(mname="__instance_exec#{n}")
|
40
|
+
InstanceExecHelper.module_eval{ define_method(mname, &block) }
|
41
|
+
ensure
|
42
|
+
Thread.critical = old_critical
|
43
|
+
end
|
44
|
+
begin
|
45
|
+
ret = send(mname, *args)
|
46
|
+
ensure
|
47
|
+
InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
|
48
|
+
end
|
49
|
+
ret
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'spec_helper')
|
2
|
+
|
3
|
+
describe Nanite::ActorRegistry do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
class WebDocumentImporter
|
7
|
+
include Nanite::Actor
|
8
|
+
expose :import, :cancel
|
9
|
+
|
10
|
+
def import
|
11
|
+
1
|
12
|
+
end
|
13
|
+
def cancel
|
14
|
+
0
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module Actors
|
19
|
+
class ComedyActor
|
20
|
+
include Nanite::Actor
|
21
|
+
expose :fun_tricks
|
22
|
+
def fun_tricks
|
23
|
+
:rabbit_in_the_hat
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
before(:each) do
|
30
|
+
Nanite::Log.stub!(:info)
|
31
|
+
@registry = Nanite::ActorRegistry.new
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should know about all services" do
|
35
|
+
@registry.register(WebDocumentImporter.new, nil)
|
36
|
+
@registry.register(Actors::ComedyActor.new, nil)
|
37
|
+
@registry.services.sort.should == ["/actors/comedy_actor/fun_tricks", "/web_document_importer/cancel", "/web_document_importer/import"]
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should not register anything except Nanite::Actor" do
|
41
|
+
lambda { @registry.register(String.new, nil) }.should raise_error(ArgumentError)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should register an actor" do
|
45
|
+
importer = WebDocumentImporter.new
|
46
|
+
@registry.register(importer, nil)
|
47
|
+
@registry.actors['web_document_importer'].should == importer
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should log info message that actor was registered" do
|
51
|
+
importer = WebDocumentImporter.new
|
52
|
+
Nanite::Log.should_receive(:info).with("Registering #{importer.inspect} with prefix nil")
|
53
|
+
@registry.register(importer, nil)
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should handle actors registered with a custom prefix" do
|
57
|
+
importer = WebDocumentImporter.new
|
58
|
+
@registry.register(importer, 'monkey')
|
59
|
+
@registry.actors['monkey'].should == importer
|
60
|
+
end
|
61
|
+
|
62
|
+
end # Nanite::ActorRegistry
|
data/spec/actor_spec.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'spec_helper')
|
2
|
+
|
3
|
+
class WebDocumentImporter
|
4
|
+
include Nanite::Actor
|
5
|
+
expose :import, :cancel
|
6
|
+
|
7
|
+
def import
|
8
|
+
1
|
9
|
+
end
|
10
|
+
def cancel
|
11
|
+
0
|
12
|
+
end
|
13
|
+
def continue
|
14
|
+
1
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module Actors
|
19
|
+
class ComedyActor
|
20
|
+
include Nanite::Actor
|
21
|
+
expose :fun_tricks
|
22
|
+
def fun_tricks
|
23
|
+
:rabbit_in_the_hat
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
describe Nanite::Actor do
|
29
|
+
|
30
|
+
describe ".expose" do
|
31
|
+
it "should single expose method only once" do
|
32
|
+
3.times { WebDocumentImporter.expose(:continue) }
|
33
|
+
WebDocumentImporter.provides_for("webfiles").should == ["/webfiles/import", "/webfiles/cancel", "/webfiles/continue"]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe ".default_prefix" do
|
38
|
+
it "is calculated as default prefix as const path of class name" do
|
39
|
+
Actors::ComedyActor.default_prefix.should == "actors/comedy_actor"
|
40
|
+
WebDocumentImporter.default_prefix.should == "web_document_importer"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe ".provides_for(prefix)" do
|
45
|
+
before :each do
|
46
|
+
@provides = Actors::ComedyActor.provides_for("money")
|
47
|
+
end
|
48
|
+
it "returns an array" do
|
49
|
+
@provides.should be_kind_of(Array)
|
50
|
+
end
|
51
|
+
|
52
|
+
it "maps exposed service methods to prefix" do
|
53
|
+
@provides.should == ["/money/fun_tricks"]
|
54
|
+
wdi_provides = WebDocumentImporter.provides_for("webfiles")
|
55
|
+
wdi_provides.should include("/webfiles/import")
|
56
|
+
wdi_provides.should include("/webfiles/cancel")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|