rightscale-nanite 0.4.1 → 0.4.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/lib/nanite.rb +71 -0
  2. data/lib/nanite/actor.rb +60 -0
  3. data/lib/nanite/actor_registry.rb +24 -0
  4. data/lib/nanite/admin.rb +153 -0
  5. data/lib/nanite/agent.rb +250 -0
  6. data/lib/nanite/amqp.rb +47 -0
  7. data/lib/nanite/cluster.rb +203 -0
  8. data/lib/nanite/config.rb +102 -0
  9. data/lib/nanite/console.rb +39 -0
  10. data/lib/nanite/daemonize.rb +13 -0
  11. data/lib/nanite/dispatcher.rb +90 -0
  12. data/lib/nanite/identity.rb +16 -0
  13. data/lib/nanite/job.rb +104 -0
  14. data/lib/nanite/local_state.rb +34 -0
  15. data/lib/nanite/log.rb +64 -0
  16. data/lib/nanite/log/formatter.rb +39 -0
  17. data/lib/nanite/mapper.rb +277 -0
  18. data/lib/nanite/mapper_proxy.rb +56 -0
  19. data/lib/nanite/packets.rb +231 -0
  20. data/lib/nanite/pid_file.rb +52 -0
  21. data/lib/nanite/reaper.rb +38 -0
  22. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  23. data/lib/nanite/security/certificate.rb +55 -0
  24. data/lib/nanite/security/certificate_cache.rb +66 -0
  25. data/lib/nanite/security/distinguished_name.rb +34 -0
  26. data/lib/nanite/security/encrypted_document.rb +46 -0
  27. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  28. data/lib/nanite/security/secure_serializer.rb +67 -0
  29. data/lib/nanite/security/signature.rb +40 -0
  30. data/lib/nanite/security/static_certificate_store.rb +35 -0
  31. data/lib/nanite/security_provider.rb +47 -0
  32. data/lib/nanite/serializer.rb +52 -0
  33. data/lib/nanite/state.rb +164 -0
  34. data/lib/nanite/streaming.rb +125 -0
  35. data/lib/nanite/util.rb +51 -0
  36. data/spec/actor_registry_spec.rb +62 -0
  37. data/spec/actor_spec.rb +59 -0
  38. data/spec/agent_spec.rb +235 -0
  39. data/spec/cached_certificate_store_proxy_spec.rb +34 -0
  40. data/spec/certificate_cache_spec.rb +49 -0
  41. data/spec/certificate_spec.rb +27 -0
  42. data/spec/cluster_spec.rb +300 -0
  43. data/spec/dispatcher_spec.rb +136 -0
  44. data/spec/distinguished_name_spec.rb +24 -0
  45. data/spec/encrypted_document_spec.rb +21 -0
  46. data/spec/job_spec.rb +219 -0
  47. data/spec/local_state_spec.rb +112 -0
  48. data/spec/packet_spec.rb +218 -0
  49. data/spec/rsa_key_pair_spec.rb +33 -0
  50. data/spec/secure_serializer_spec.rb +41 -0
  51. data/spec/serializer_spec.rb +107 -0
  52. data/spec/signature_spec.rb +30 -0
  53. data/spec/spec_helper.rb +23 -0
  54. data/spec/static_certificate_store_spec.rb +30 -0
  55. data/spec/util_spec.rb +63 -0
  56. metadata +62 -1
@@ -0,0 +1,164 @@
1
+ require 'redis'
2
+
3
+ module Nanite
4
+ class State
5
+ include Enumerable
6
+
7
+ # this class encapsulates the state of a nanite system using redis as the
8
+ # data store. here is the schema, for each agent we store a number of items,
9
+ # for a nanite with the identity: nanite-foobar we store the following things:
10
+ #
11
+ # nanite-foobar: 0.72 # load average or 'status'
12
+ # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
13
+ # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
14
+ # t-nanite-foobar: 123456789 # unix timestamp of the last state update
15
+ #
16
+ # also we do an inverted index for quick lookup of agents providing a certain
17
+ # service, so for each service the agent provides, we add the nanite to a SET
18
+ # of all the nanites that provide said service:
19
+ #
20
+ # /gems/list: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
21
+ #
22
+ # we do that same thing for tags:
23
+ # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
24
+ #
25
+ # This way we can do a lookup of what nanites provide a set of services and tags based
26
+ # on redis SET intersection:
27
+ #
28
+ # nanites_for('/gems/list', 'some-tag')
29
+ # => returns a nested array of nanites and their state that provide the intersection
30
+ # of these two service tags
31
+
32
+ def initialize(redis)
33
+ Nanite::Log.info("initializing redis state: #{redis}")
34
+ host, port = redis.split(':')
35
+ host ||= '127.0.0.1'
36
+ port ||= '6379'
37
+ @redis = Redis.new :host => host, :port => port
38
+ end
39
+
40
+ def log_redis_error(meth,&blk)
41
+ blk.call
42
+ rescue RedisError => e
43
+ Nanite::Log.info("redis error in method: #{meth}")
44
+ raise e
45
+ end
46
+
47
+ def [](nanite)
48
+ log_redis_error("[]") do
49
+ status = @redis[nanite]
50
+ timestamp = @redis["t-#{nanite}"]
51
+ services = @redis.set_members("s-#{nanite}")
52
+ tags = @redis.set_members("tg-#{nanite}")
53
+ return nil unless status && timestamp && services
54
+ {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
55
+ end
56
+ end
57
+
58
+ def []=(nanite, hsh)
59
+ log_redis_error("[]=") do
60
+ update_state(nanite, hsh[:status], hsh[:services], hsh[:tags])
61
+ end
62
+ end
63
+
64
+ def delete(nanite)
65
+ log_redis_error("delete") do
66
+ (@redis.set_members("s-#{nanite}")||[]).each do |srv|
67
+ @redis.set_delete(srv, nanite)
68
+ if @redis.set_count(srv) == 0
69
+ @redis.delete(srv)
70
+ @redis.set_delete("naniteservices", srv)
71
+ end
72
+ end
73
+ (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
74
+ @redis.set_delete(tag, nanite)
75
+ if @redis.set_count(tag) == 0
76
+ @redis.delete(tag)
77
+ @redis.set_delete("nanitetags", tag)
78
+ end
79
+ end
80
+ @redis.delete nanite
81
+ @redis.delete "s-#{nanite}"
82
+ @redis.delete "t-#{nanite}"
83
+ @redis.delete "tg-#{nanite}"
84
+ end
85
+ end
86
+
87
+ def all_services
88
+ log_redis_error("all_services") do
89
+ @redis.set_members("naniteservices")
90
+ end
91
+ end
92
+
93
+ def all_tags
94
+ log_redis_error("all_tags") do
95
+ @redis.set_members("nanitetags")
96
+ end
97
+ end
98
+
99
+ def update_state(name, status, services, tags)
100
+ old_services = @redis.set_members("s-#{name}")
101
+ if old_services
102
+ (old_services - services).each do |s|
103
+ @redis.set_delete(s, name)
104
+ @redis.set_delete("naniteservices", s)
105
+ end
106
+ end
107
+ old_tags = @redis.set_members("tg-#{name}")
108
+ if old_tags
109
+ (old_tags - tags).each do |t|
110
+ @redis.set_delete(t, name)
111
+ @redis.set_delete("nanitetags", t)
112
+ end
113
+ end
114
+ @redis.delete("s-#{name}")
115
+ services.each do |srv|
116
+ @redis.set_add(srv, name)
117
+ @redis.set_add("s-#{name}", srv)
118
+ @redis.set_add("naniteservices", srv)
119
+ end
120
+ @redis.delete("tg-#{name}")
121
+ tags.each do |tag|
122
+ next if tag.nil?
123
+ @redis.set_add(tag, name)
124
+ @redis.set_add("tg-#{name}", tag)
125
+ @redis.set_add("nanitetags", tag)
126
+ end
127
+ @redis[name] = status
128
+ @redis["t-#{name}"] = Time.now.to_i
129
+ end
130
+
131
+ def list_nanites
132
+ log_redis_error("list_nanites") do
133
+ @redis.keys("nanite-*")
134
+ end
135
+ end
136
+
137
+ def size
138
+ list_nanites.size
139
+ end
140
+
141
+ def clear_state
142
+ log_redis_error("clear_state") do
143
+ @redis.keys("*").each {|k| @redis.delete k}
144
+ end
145
+ end
146
+
147
+ def each
148
+ list_nanites.each do |nan|
149
+ yield nan, self[nan]
150
+ end
151
+ end
152
+
153
+ def nanites_for(service, *tags)
154
+ keys = tags.dup << service
155
+ log_redis_error("nanites_for") do
156
+ res = []
157
+ (@redis.set_intersect(keys)||[]).each do |nan|
158
+ res << [nan, self[nan]]
159
+ end
160
+ res
161
+ end
162
+ end
163
+ end
164
+ end
@@ -0,0 +1,125 @@
1
+ module Nanite
2
+ # Nanite actors can transfer files to each other.
3
+ #
4
+ # ==== Options
5
+ #
6
+ # filename : you guessed it, name of the file!
7
+ # domain : part of the routing key used to locate receiver(s)
8
+ # destination : is a name of the file as it gonna be stored at the destination
9
+ # meta :
10
+ #
11
+ # File streaming is done in chunks. When file streaming starts,
12
+ # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
13
+ # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
14
+ # Nanite::FileEnd packet is sent.
15
+ #
16
+ # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
17
+ # more than that in one operation via socket.
18
+ #
19
+ # ==== Domains
20
+ #
21
+ # Streaming happens using a topic exchange called 'file broadcast', with keys
22
+ # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
23
+ # receivers find each other in the cluster. Default domain is 'global'.
24
+ #
25
+ # Domains also serve as a way to register a callback Nanite agent executes once file
26
+ # streaming is completed. If a callback with name of domain is registered, it is called.
27
+ #
28
+ # Callbacks are registered by passing a block to subscribe_to_files method.
29
+ module FileStreaming
30
+ def broadcast_file(filename, options = {})
31
+ if File.exist?(filename)
32
+ File.open(filename, 'rb') do |file|
33
+ broadcast_data(filename, file, options)
34
+ end
35
+ else
36
+ return "file not found"
37
+ end
38
+ end
39
+
40
+ def broadcast_data(filename, io, options = {})
41
+ domain = options[:domain] || 'global'
42
+ filename = File.basename(filename)
43
+ dest = options[:destination] || filename
44
+ sent = 0
45
+
46
+ begin
47
+ file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
48
+ amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
49
+ res = Nanite::FileChunk.new(file_push.token)
50
+ while chunk = io.read(16384)
51
+ res.chunk = chunk
52
+ amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
53
+ sent += chunk.length
54
+ end
55
+ fend = Nanite::FileEnd.new(file_push.token, options[:meta])
56
+ amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
57
+ ""
58
+ ensure
59
+ io.close
60
+ end
61
+
62
+ sent
63
+ end
64
+
65
+ # FileState represents a file download in progress.
66
+ # It incapsulates the following information:
67
+ #
68
+ # * unique operation token
69
+ # * domain (namespace for file streaming operations)
70
+ # * file IO chunks are written to on receiver's side
71
+ class FileState
72
+
73
+ def initialize(token, dest, domain, write, blk)
74
+ @token = token
75
+ @cb = blk
76
+ @domain = domain
77
+ @write = write
78
+
79
+ if write
80
+ @filename = File.join(Nanite.agent.file_root, dest)
81
+ @dest = File.open(@filename, 'wb')
82
+ else
83
+ @dest = dest
84
+ end
85
+
86
+ @data = ""
87
+ end
88
+
89
+ def handle_packet(packet)
90
+ case packet
91
+ when Nanite::FileChunk
92
+ Nanite::Log.debug "written chunk to #{@dest.inspect}"
93
+ @data << packet.chunk
94
+
95
+ if @write
96
+ @dest.write(packet.chunk)
97
+ end
98
+ when Nanite::FileEnd
99
+ Nanite::Log.debug "#{@dest.inspect} receiving is completed"
100
+ if @write
101
+ @dest.close
102
+ end
103
+
104
+ @cb.call(@data, @dest, packet.meta)
105
+ end
106
+ end
107
+
108
+ end
109
+
110
+ def subscribe_to_files(domain='global', write=false, &blk)
111
+ Nanite::Log.info "subscribing to file broadcasts for #{domain}"
112
+ @files ||= {}
113
+ amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
114
+ case msg = serializer.load(packet)
115
+ when FileStart
116
+ @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
117
+ when FileChunk, FileEnd
118
+ if file = @files[msg.token]
119
+ file.handle_packet(msg)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,51 @@
1
+ class String
2
+ ##
3
+ # Convert to snake case.
4
+ #
5
+ # "FooBar".snake_case #=> "foo_bar"
6
+ # "HeadlineCNNNews".snake_case #=> "headline_cnn_news"
7
+ # "CNN".snake_case #=> "cnn"
8
+ #
9
+ # @return [String] Receiver converted to snake case.
10
+ #
11
+ # @api public
12
+ def snake_case
13
+ return self.downcase if self =~ /^[A-Z]+$/
14
+ self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
15
+ return $+.downcase
16
+ end
17
+
18
+ ##
19
+ # Convert a constant name to a path, assuming a conventional structure.
20
+ #
21
+ # "FooBar::Baz".to_const_path # => "foo_bar/baz"
22
+ #
23
+ # @return [String] Path to the file containing the constant named by receiver
24
+ # (constantized string), assuming a conventional structure.
25
+ #
26
+ # @api public
27
+ def to_const_path
28
+ snake_case.gsub(/::/, "/")
29
+ end
30
+ end
31
+
32
+ class Object
33
+ module InstanceExecHelper; end
34
+ include InstanceExecHelper
35
+ def instance_exec(*args, &block)
36
+ begin
37
+ old_critical, Thread.critical = Thread.critical, true
38
+ n = 0
39
+ n += 1 while respond_to?(mname="__instance_exec#{n}")
40
+ InstanceExecHelper.module_eval{ define_method(mname, &block) }
41
+ ensure
42
+ Thread.critical = old_critical
43
+ end
44
+ begin
45
+ ret = send(mname, *args)
46
+ ensure
47
+ InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
48
+ end
49
+ ret
50
+ end
51
+ end
@@ -0,0 +1,62 @@
1
+ require File.join(File.dirname(__FILE__), 'spec_helper')
2
+
3
+ describe Nanite::ActorRegistry do
4
+
5
+ before(:all) do
6
+ class WebDocumentImporter
7
+ include Nanite::Actor
8
+ expose :import, :cancel
9
+
10
+ def import
11
+ 1
12
+ end
13
+ def cancel
14
+ 0
15
+ end
16
+ end
17
+
18
+ module Actors
19
+ class ComedyActor
20
+ include Nanite::Actor
21
+ expose :fun_tricks
22
+ def fun_tricks
23
+ :rabbit_in_the_hat
24
+ end
25
+ end
26
+ end
27
+ end
28
+
29
+ before(:each) do
30
+ Nanite::Log.stub!(:info)
31
+ @registry = Nanite::ActorRegistry.new
32
+ end
33
+
34
+ it "should know about all services" do
35
+ @registry.register(WebDocumentImporter.new, nil)
36
+ @registry.register(Actors::ComedyActor.new, nil)
37
+ @registry.services.sort.should == ["/actors/comedy_actor/fun_tricks", "/web_document_importer/cancel", "/web_document_importer/import"]
38
+ end
39
+
40
+ it "should not register anything except Nanite::Actor" do
41
+ lambda { @registry.register(String.new, nil) }.should raise_error(ArgumentError)
42
+ end
43
+
44
+ it "should register an actor" do
45
+ importer = WebDocumentImporter.new
46
+ @registry.register(importer, nil)
47
+ @registry.actors['web_document_importer'].should == importer
48
+ end
49
+
50
+ it "should log info message that actor was registered" do
51
+ importer = WebDocumentImporter.new
52
+ Nanite::Log.should_receive(:info).with("Registering #{importer.inspect} with prefix nil")
53
+ @registry.register(importer, nil)
54
+ end
55
+
56
+ it "should handle actors registered with a custom prefix" do
57
+ importer = WebDocumentImporter.new
58
+ @registry.register(importer, 'monkey')
59
+ @registry.actors['monkey'].should == importer
60
+ end
61
+
62
+ end # Nanite::ActorRegistry
@@ -0,0 +1,59 @@
1
+ require File.join(File.dirname(__FILE__), 'spec_helper')
2
+
3
+ class WebDocumentImporter
4
+ include Nanite::Actor
5
+ expose :import, :cancel
6
+
7
+ def import
8
+ 1
9
+ end
10
+ def cancel
11
+ 0
12
+ end
13
+ def continue
14
+ 1
15
+ end
16
+ end
17
+
18
+ module Actors
19
+ class ComedyActor
20
+ include Nanite::Actor
21
+ expose :fun_tricks
22
+ def fun_tricks
23
+ :rabbit_in_the_hat
24
+ end
25
+ end
26
+ end
27
+
28
+ describe Nanite::Actor do
29
+
30
+ describe ".expose" do
31
+ it "should single expose method only once" do
32
+ 3.times { WebDocumentImporter.expose(:continue) }
33
+ WebDocumentImporter.provides_for("webfiles").should == ["/webfiles/import", "/webfiles/cancel", "/webfiles/continue"]
34
+ end
35
+ end
36
+
37
+ describe ".default_prefix" do
38
+ it "is calculated as default prefix as const path of class name" do
39
+ Actors::ComedyActor.default_prefix.should == "actors/comedy_actor"
40
+ WebDocumentImporter.default_prefix.should == "web_document_importer"
41
+ end
42
+ end
43
+
44
+ describe ".provides_for(prefix)" do
45
+ before :each do
46
+ @provides = Actors::ComedyActor.provides_for("money")
47
+ end
48
+ it "returns an array" do
49
+ @provides.should be_kind_of(Array)
50
+ end
51
+
52
+ it "maps exposed service methods to prefix" do
53
+ @provides.should == ["/money/fun_tricks"]
54
+ wdi_provides = WebDocumentImporter.provides_for("webfiles")
55
+ wdi_provides.should include("/webfiles/import")
56
+ wdi_provides.should include("/webfiles/cancel")
57
+ end
58
+ end
59
+ end