rightscale-nanite-dev 0.4.1.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. data/LICENSE +201 -0
  2. data/README.rdoc +430 -0
  3. data/Rakefile +78 -0
  4. data/TODO +24 -0
  5. data/bin/nanite-admin +65 -0
  6. data/bin/nanite-agent +79 -0
  7. data/bin/nanite-mapper +50 -0
  8. data/lib/nanite.rb +74 -0
  9. data/lib/nanite/actor.rb +71 -0
  10. data/lib/nanite/actor_registry.rb +26 -0
  11. data/lib/nanite/admin.rb +138 -0
  12. data/lib/nanite/agent.rb +274 -0
  13. data/lib/nanite/amqp.rb +58 -0
  14. data/lib/nanite/cluster.rb +256 -0
  15. data/lib/nanite/config.rb +111 -0
  16. data/lib/nanite/console.rb +39 -0
  17. data/lib/nanite/daemonize.rb +13 -0
  18. data/lib/nanite/identity.rb +16 -0
  19. data/lib/nanite/job.rb +104 -0
  20. data/lib/nanite/local_state.rb +38 -0
  21. data/lib/nanite/log.rb +66 -0
  22. data/lib/nanite/log/formatter.rb +39 -0
  23. data/lib/nanite/mapper.rb +315 -0
  24. data/lib/nanite/mapper_proxy.rb +75 -0
  25. data/lib/nanite/nanite_dispatcher.rb +92 -0
  26. data/lib/nanite/packets.rb +401 -0
  27. data/lib/nanite/pid_file.rb +52 -0
  28. data/lib/nanite/reaper.rb +39 -0
  29. data/lib/nanite/redis_tag_store.rb +141 -0
  30. data/lib/nanite/security/cached_certificate_store_proxy.rb +24 -0
  31. data/lib/nanite/security/certificate.rb +55 -0
  32. data/lib/nanite/security/certificate_cache.rb +66 -0
  33. data/lib/nanite/security/distinguished_name.rb +34 -0
  34. data/lib/nanite/security/encrypted_document.rb +46 -0
  35. data/lib/nanite/security/rsa_key_pair.rb +53 -0
  36. data/lib/nanite/security/secure_serializer.rb +68 -0
  37. data/lib/nanite/security/signature.rb +46 -0
  38. data/lib/nanite/security/static_certificate_store.rb +35 -0
  39. data/lib/nanite/security_provider.rb +47 -0
  40. data/lib/nanite/serializer.rb +52 -0
  41. data/lib/nanite/state.rb +135 -0
  42. data/lib/nanite/streaming.rb +125 -0
  43. data/lib/nanite/util.rb +78 -0
  44. metadata +111 -0
data/Rakefile ADDED
@@ -0,0 +1,78 @@
1
+ require 'rubygems'
2
+ require 'rake/gempackagetask'
3
+ require "spec/rake/spectask"
4
+ begin; require 'rubygems'; rescue LoadError; end
5
+ begin
6
+ require 'hanna/rdoctask'
7
+ rescue LoadError
8
+ require 'rake/rdoctask'
9
+ end
10
+ require 'rake/clean'
11
+ require 'lib/nanite'
12
+
13
+ GEM = "rightscale-nanite-dev"
14
+ AUTHOR = "Ezra Zygmuntowicz"
15
+ EMAIL = "ezra@engineyard.com"
16
+ HOMEPAGE = "http://github.com/ezmobius/nanite"
17
+ SUMMARY = "self assembling fabric of ruby daemons"
18
+
19
+ Dir.glob('tasks/*.rake').each { |r| Rake.application.add_import r }
20
+
21
+ spec = Gem::Specification.new do |s|
22
+
23
+ s.name = GEM
24
+ s.version = Nanite::VERSION
25
+ s.platform = Gem::Platform::RUBY
26
+ s.has_rdoc = true
27
+ s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
28
+ s.summary = SUMMARY
29
+ s.description = s.summary
30
+ s.author = AUTHOR
31
+ s.email = EMAIL
32
+ s.homepage = HOMEPAGE
33
+
34
+ s.bindir = "bin"
35
+ s.executables = %w( nanite-agent nanite-mapper nanite-admin )
36
+
37
+ s.add_dependency('amqp', '>= 0.6.0')
38
+
39
+ s.require_path = 'lib'
40
+ s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*")
41
+ end
42
+
43
+ Rake::GemPackageTask.new(spec) do |pkg|
44
+ pkg.gem_spec = spec
45
+ end
46
+
47
+ task :default => :spec
48
+
49
+ task :install => [:package] do
50
+ sh %{sudo gem install pkg/#{GEM}-#{Nanite::VERSION}}
51
+ end
52
+
53
+ desc "Run unit specs"
54
+ Spec::Rake::SpecTask.new do |t|
55
+ t.spec_opts = ["--format", "specdoc", "--colour"]
56
+ t.spec_files = FileList["spec/**/*_spec.rb"]
57
+ end
58
+
59
+ desc 'Generate RDoc documentation'
60
+ Rake::RDocTask.new do |rd|
61
+ rd.title = spec.name
62
+ rd.rdoc_dir = 'rdoc'
63
+ rd.main = "README.rdoc"
64
+ rd.rdoc_files.include("lib/**/*.rb", *spec.extra_rdoc_files)
65
+ end
66
+ CLOBBER.include(:clobber_rdoc)
67
+
68
+ desc 'Generate and open documentation'
69
+ task :docs => :rdoc do
70
+ case RUBY_PLATFORM
71
+ when /darwin/ ; sh 'open rdoc/index.html'
72
+ when /mswin|mingw/ ; sh 'start rdoc\index.html'
73
+ else
74
+ sh 'firefox rdoc/index.html'
75
+ end
76
+ end
77
+
78
+ require 'rightscale-nanite.rb'
data/TODO ADDED
@@ -0,0 +1,24 @@
1
+ TODO:
2
+
3
+ - The examples/crew.rb file is pointing towards a hard coded user dir. Needs to be
4
+ documented as part of a working example.
5
+ - examples/async_rack_front/async_rack_front.ru needs to be documented and verified working.
6
+
7
+ Ian:
8
+ - Sync Mapper/Agent#start with nanite-mapper/agent
9
+ - Update docs for Agent#start and Mapper#start
10
+ - Update docs in nanite-agent and nanite-mapper
11
+ - Ensure file transfer works
12
+ - Check secure stuff still works
13
+ - Check custom status_proc works
14
+ - Check documentation, only document public methods
15
+ - ensure the removal of threaded_actors option doesn't cause shit to block
16
+
17
+ - Look into using EM deferables for actors dispatch.
18
+ - Integration specs that spawn a small cluster of nanites
19
+ - Rename Ping to Status
20
+ - request/push should take *args for payload?
21
+
22
+ Maybe:
23
+ - Make mapper queue durable and Results respect :persistent flag on the request
24
+ - Add a global result received callback
data/bin/nanite-admin ADDED
@@ -0,0 +1,65 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # To work without being installed as a gem:
4
+ libdir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
5
+ $:.unshift libdir unless $:.include? libdir
6
+
7
+ require 'nanite'
8
+ require 'nanite/admin'
9
+ require 'eventmachine'
10
+ require 'thin'
11
+ # IMPORTANT!
12
+ # You need raggi's patched async version of Thin at the moment to use
13
+ # the nanite-admin tool.
14
+ #
15
+ # raggi's Git repo contains a branch called 'async_for_rack' which contains the
16
+ # version of Thin you want to install. raggi has apparently removed the 'master'
17
+ # branch of his Git repo so you may see a warning like that shown below.
18
+ #
19
+ # git clone git://github.com/raggi/thin.git thin-raggi-async
20
+ # ...
21
+ # warning: remote HEAD refers to nonexistent ref, unable to checkout. <<== IGNORE THIS
22
+ #
23
+ # cd thin-raggi-async/
24
+ # git checkout --track -b async_for_rack origin/async_for_rack
25
+ # warning: You appear to be on a branch yet to be born. <<== IGNORE THIS
26
+ # warning: Forcing checkout of origin/async_for_rack. <<== IGNORE THIS
27
+ # Branch async_for_rack set up to track remote branch refs/remotes/origin/async_for_rack.
28
+ # Switched to a new branch "async_for_rack"
29
+
30
+ # run : 'rake install' to build and install the Thin gem
31
+ # cd <NANITE>
32
+ # ./bin/nanite-admin
33
+
34
+ # When you need to update this Thin install you should be able to do a 'git pull' on the
35
+ # "async_for_rack" branch.
36
+
37
+ require File.dirname(__FILE__) + '/../lib/nanite'
38
+ require 'yaml'
39
+ require "optparse"
40
+
41
+ include Nanite::CommonConfig
42
+
43
+ options = {}
44
+
45
+ opts = OptionParser.new do |opts|
46
+ opts.banner = "Usage: nanite-admin [-flags] [argument]"
47
+ opts.define_head "Nanite Admin: a basic control interface for your nanite cluster."
48
+ opts.separator '*'*80
49
+
50
+ setup_mapper_options(opts, options)
51
+
52
+ opts.on("--thin-debug", "Set the equivalent of the '--debug' flag on the Thin webserver.") do
53
+ options[:thin_debug] = true
54
+ end
55
+ end
56
+
57
+ opts.parse!
58
+
59
+ EM.run do
60
+ Nanite.start_mapper(options)
61
+ Nanite::Log.info "starting nanite-admin"
62
+ Rack::Handler::Thin.run(Nanite::Admin.new(Nanite.mapper), :Port => 4000) do
63
+ Thin::Logging.debug = options[:thin_debug]
64
+ end
65
+ end
data/bin/nanite-agent ADDED
@@ -0,0 +1,79 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.dirname(__FILE__) + '/../lib/nanite'
4
+ require 'optparse'
5
+
6
+ include Nanite::CommonConfig
7
+
8
+ options = {}
9
+
10
+ opts = OptionParser.new do |opts|
11
+ opts.banner = "Usage: nanite-agent [-flag] [argument]"
12
+ opts.define_head "Nanite Agent: ruby process that acts upon messages passed to it by a mapper."
13
+ opts.separator '*'*80
14
+
15
+ setup_common_options(opts, options, 'agent')
16
+
17
+ opts.on("-n", "--nanite NANITE_ROOT", "Specify the root of your nanite agent project.") do |nanite|
18
+ options[:root] = nanite
19
+ end
20
+
21
+ opts.on("--ping-time PINGTIME", "Specify how often the agents contacts the mapper") do |ping|
22
+ options[:ping_time] = ping
23
+ end
24
+
25
+ opts.on("--actors-dir DIR", "Path to directory containing actors (NANITE_ROOT/actors by default)") do |dir|
26
+ options[:actors_dir] = dir
27
+ end
28
+
29
+ opts.on("--actors ACTORS", "Comma separated list of actors to load (all ruby files in actors directory by default)") do |a|
30
+ options[:actors] = a.split(',')
31
+ end
32
+
33
+ opts.on("--initrb FILE", "Path to agent initialization file (NANITE_ROOT/init.rb by default)") do |initrb|
34
+ options[:initrb] = initrb
35
+ end
36
+
37
+ opts.on("--single-threaded", "Run all operations in one thread") do
38
+ options[:single_threaded] = true
39
+ end
40
+
41
+ opts.on("--threadpool COUNT", Integer, "Number of threads to run all operations in") do |tps|
42
+ options[:threadpool_size] = tps
43
+ end
44
+
45
+ end
46
+
47
+ opts.parse!
48
+
49
+ if ARGV[0] == 'stop' || ARGV[0] == 'status'
50
+ agent = Nanite::Agent.new(options)
51
+ pid_file = Nanite::PidFile.new(agent.identity, agent.options)
52
+ unless pid = pid_file.read_pid
53
+ puts "#{pid_file} not found"
54
+ exit
55
+ end
56
+ if ARGV[0] == 'stop'
57
+ puts "Stopping nanite agent #{agent.identity} (pid #{pid})"
58
+ begin
59
+ Process.kill('TERM', pid)
60
+ rescue Errno::ESRCH
61
+ puts "Process does not exist (pid #{pid})"
62
+ exit
63
+ end
64
+ puts 'Done.'
65
+ else
66
+ if Process.getpgid(pid) != -1
67
+ psdata = `ps up #{pid}`.split("\n").last.split
68
+ memory = (psdata[5].to_i / 1024)
69
+ puts "The agent is alive, using #{memory}MB of memory"
70
+ else
71
+ puts "The agent is not running but has a stale pid file at #{pid_file}"
72
+ end
73
+ end
74
+ exit
75
+ end
76
+
77
+ EM.run do
78
+ Nanite.start_agent(options)
79
+ end
data/bin/nanite-mapper ADDED
@@ -0,0 +1,50 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.dirname(__FILE__) + '/../lib/nanite'
4
+ require 'optparse'
5
+
6
+ include Nanite::CommonConfig
7
+
8
+ options = {}
9
+
10
+ opts = OptionParser.new do |opts|
11
+ opts.banner = "Usage: nanite-mapper [-flags] [argument]"
12
+ opts.define_head "Nanite Mapper: clustered head unit for self assembling cluster of ruby processes."
13
+ opts.separator '*'*80
14
+
15
+ setup_mapper_options(opts, options)
16
+ end
17
+
18
+ opts.parse!
19
+
20
+ if ARGV[0] == 'stop' || ARGV[0] == 'status'
21
+ mapper = Nanite::Mapper.new(options)
22
+ pid_file = Nanite::PidFile.new(mapper.identity, mapper.options)
23
+ unless pid = pid_file.read_pid
24
+ puts "#{pid_file} not found"
25
+ exit
26
+ end
27
+ if ARGV[0] == 'stop'
28
+ puts "Stopping nanite mapper #{mapper.identity} (pid #{pid})"
29
+ begin
30
+ Process.kill('TERM', pid)
31
+ rescue Errno::ESRCH
32
+ puts "Process does not exist (pid #{pid})"
33
+ exit
34
+ end
35
+ puts 'Done.'
36
+ else
37
+ if Process.getpgid(pid) != -1
38
+ psdata = `ps up #{pid}`.split("\n").last.split
39
+ memory = (psdata[5].to_i / 1024)
40
+ puts "The mapper is alive, using #{memory}MB of memory"
41
+ else
42
+ puts "The mapper is not running but has a stale pid file at #{pid_file}"
43
+ end
44
+ end
45
+ exit
46
+ end
47
+
48
+ EM.run do
49
+ Nanite.start_mapper(options)
50
+ end
data/lib/nanite.rb ADDED
@@ -0,0 +1,74 @@
1
+ require 'rubygems'
2
+ require 'amqp'
3
+ require 'mq'
4
+ require 'json'
5
+ require 'logger'
6
+ require 'yaml'
7
+ require 'openssl'
8
+
9
+ $:.unshift File.dirname(__FILE__)
10
+ require 'nanite/amqp'
11
+ require 'nanite/util'
12
+ require 'nanite/config'
13
+ require 'nanite/packets'
14
+ require 'nanite/identity'
15
+ require 'nanite/console'
16
+ require 'nanite/daemonize'
17
+ require 'nanite/pid_file'
18
+ require 'nanite/job'
19
+ require 'nanite/mapper'
20
+ require 'nanite/actor'
21
+ require 'nanite/actor_registry'
22
+ require 'nanite/streaming'
23
+ require 'nanite/nanite_dispatcher'
24
+ require 'nanite/agent'
25
+ require 'nanite/cluster'
26
+ require 'nanite/reaper'
27
+ require 'nanite/log'
28
+ require 'nanite/mapper_proxy'
29
+ require 'nanite/security_provider'
30
+ require 'nanite/security/cached_certificate_store_proxy'
31
+ require 'nanite/security/certificate'
32
+ require 'nanite/security/certificate_cache'
33
+ require 'nanite/security/distinguished_name'
34
+ require 'nanite/security/encrypted_document'
35
+ require 'nanite/security/rsa_key_pair'
36
+ require 'nanite/security/secure_serializer'
37
+ require 'nanite/security/signature'
38
+ require 'nanite/security/static_certificate_store'
39
+ require 'nanite/serializer'
40
+
41
+ module Nanite
42
+ VERSION = '0.4.1.10' unless defined?(Nanite::VERSION)
43
+
44
+ class MapperNotRunning < StandardError; end
45
+
46
+ class << self
47
+ attr_reader :mapper, :agent
48
+
49
+ def start_agent(options = {})
50
+ @agent = Nanite::Agent.start(options)
51
+ end
52
+
53
+ def start_mapper(options = {})
54
+ @mapper = Nanite::Mapper.start(options)
55
+ end
56
+
57
+ def request(*args, &blk)
58
+ ensure_mapper
59
+ @mapper.request(*args, &blk)
60
+ end
61
+
62
+ def push(*args)
63
+ ensure_mapper
64
+ @mapper.push(*args)
65
+ end
66
+
67
+ def ensure_mapper
68
+ @mapper ||= MapperProxy.instance
69
+ unless @mapper
70
+ raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper')
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,71 @@
1
+ module Nanite
2
+ # This mixin provides Nanite actor functionality.
3
+ #
4
+ # To use it simply include it your class containing the functionality to be exposed:
5
+ #
6
+ # class Foo
7
+ # include Nanite::Actor
8
+ # expose :bar
9
+ #
10
+ # def bar(payload)
11
+ # # ...
12
+ # end
13
+ #
14
+ # end
15
+ module Actor
16
+
17
+ def self.included(base)
18
+ base.class_eval do
19
+ include Nanite::Actor::InstanceMethods
20
+ extend Nanite::Actor::ClassMethods
21
+ end # base.class_eval
22
+ end # self.included
23
+
24
+ module ClassMethods
25
+ def default_prefix
26
+ to_s.to_const_path
27
+ end
28
+
29
+ def expose(*meths)
30
+ @exposed ||= []
31
+ meths.each do |meth|
32
+ @exposed << meth unless @exposed.include?(meth)
33
+ end
34
+ end
35
+
36
+ def provides_for(prefix)
37
+ return [] unless @exposed
38
+ @exposed.select do |meth|
39
+ if instance_methods.include?(meth.to_s) or instance_methods.include?(meth.to_sym)
40
+ true
41
+ else
42
+ Nanite::Log.warn("Exposing non-existing method #{meth} in actor #{name}")
43
+ false
44
+ end
45
+ end.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
46
+ end
47
+
48
+ def on_exception(proc = nil, &blk)
49
+ raise 'No callback provided for on_exception' unless proc || blk
50
+ @exception_callback = proc || blk
51
+ end
52
+
53
+ def exception_callback
54
+ @exception_callback
55
+ end
56
+
57
+ end # ClassMethods
58
+
59
+ module InstanceMethods
60
+ # send nanite request to another agent (through the mapper)
61
+ def request(*args, &blk)
62
+ MapperProxy.instance.request(*args, &blk)
63
+ end
64
+
65
+ def push(*args)
66
+ MapperProxy.instance.push(*args)
67
+ end
68
+ end # InstanceMethods
69
+
70
+ end # Actor
71
+ end # Nanite