shift-nanite 0.4.1.2

data/Rakefile

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require "spec/rake/spectask"
+begin; require 'rubygems'; rescue LoadError; end
+begin
+  require 'hanna/rdoctask'
+rescue LoadError
+  require 'rake/rdoctask'
+end
+require 'rake/clean'
+require 'lib/nanite'
+
+GEM = "nanite"
+AUTHOR = "Ezra Zygmuntowicz"
+EMAIL = "ezra@engineyard.com"
+HOMEPAGE = "http://github.com/ezmobius/nanite"
+SUMMARY = "self assembling fabric of ruby daemons"
+
+Dir.glob('tasks/*.rake').each { |r| Rake.application.add_import r }
+
+spec = Gem::Specification.new do |s|
+
+  s.name = GEM
+  s.version = Nanite::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+
+  s.bindir       = "bin"
+  s.executables  = %w( nanite-agent nanite-mapper nanite-admin )
+
+  s.add_dependency('amqp', '>= 0.6.0')
+
+  s.require_path = 'lib'
+  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+task :default => :spec
+
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{Nanite::VERSION}}
+end
+
+desc "Run unit specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ["--format", "specdoc", "--colour"]
+  t.spec_files = FileList["spec/**/*_spec.rb"]
+end
+
+desc 'Generate RDoc documentation'
+Rake::RDocTask.new do |rd|
+  rd.title = spec.name
+  rd.rdoc_dir = 'rdoc'
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("lib/**/*.rb", *spec.extra_rdoc_files)
+end
+CLOBBER.include(:clobber_rdoc)
+
+desc 'Generate and open documentation'
+task :docs => :rdoc do
+  case RUBY_PLATFORM
+  when /darwin/       ; sh 'open rdoc/index.html'
+  when /mswin|mingw/  ; sh 'start rdoc\index.html'
+  else 
+    sh 'firefox rdoc/index.html'
+  end
+end

data/TODO

@@ -0,0 +1,24 @@
+TODO:
+
+- The examples/crew.rb file is pointing towards a hard coded user dir.  Needs to be
+documented as part of a working example.
+- examples/async_rack_front/async_rack_front.ru needs to be documented and verified working.
+
+Ian:
+  - Sync Mapper/Agent#start with nanite-mapper/agent
+  - Update docs for Agent#start and Mapper#start
+  - Update docs in nanite-agent and nanite-mapper
+  - Ensure file transfer works
+  - Check secure stuff still works
+  - Check custom status_proc works
+  - Check documentation, only document public methods
+  - ensure the removal of threaded_actors option doesn't cause shit to block
+
+  - Look into using EM deferables for actors dispatch.
+  - Integration specs that spawn a small cluster of nanites
+  - Rename Ping to Status
+  - request/push should take *args for payload?
+
+Maybe:
+  - Make mapper queue durable and Results respect :persistent flag on the request
+  - Add a global result received callback

data/bin/nanite-admin

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+# To work without being installed as a gem:
+libdir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+$:.unshift libdir unless $:.include? libdir
+
+require 'nanite'
+require 'nanite/admin'
+require 'eventmachine'
+require 'thin'
+# IMPORTANT!
+# You need raggi's patched async version of Thin at the moment to use
+# the nanite-admin tool.
+#
+# raggi's Git repo contains a branch called 'async_for_rack' which contains the
+# version of Thin you want to install.  raggi has apparently removed the 'master'
+# branch of his Git repo so you may see a warning like that shown below.
+#
+# git clone git://github.com/raggi/thin.git thin-raggi-async
+# ...
+# warning: remote HEAD refers to nonexistent ref, unable to checkout.   <<==  IGNORE THIS
+#
+# cd thin-raggi-async/
+# git checkout --track -b async_for_rack origin/async_for_rack
+# warning: You appear to be on a branch yet to be born.                 <<==  IGNORE THIS
+# warning: Forcing checkout of origin/async_for_rack.                   <<==  IGNORE THIS
+# Branch async_for_rack set up to track remote branch refs/remotes/origin/async_for_rack.
+# Switched to a new branch "async_for_rack"
+
+# run : 'rake install' to build and install the Thin gem
+# cd <NANITE>
+# ./bin/nanite-admin
+
+# When you need to update this Thin install you should be able to do a 'git pull' on the
+# "async_for_rack" branch.
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'yaml'
+require "optparse"
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-admin [-flags] [argument]"
+  opts.define_head "Nanite Admin: a basic control interface for your nanite cluster."
+  opts.separator '*'*80
+
+  setup_mapper_options(opts, options)
+  
+  opts.on("--thin-debug", "Set the equivalent of the '--debug' flag on the Thin webserver.") do
+    options[:thin_debug] = true
+  end
+end
+
+opts.parse!
+
+EM.run do
+  Nanite.start_mapper(options)
+  Nanite::Log.info "starting nanite-admin"
+  Rack::Handler::Thin.run(Nanite::Admin.new(Nanite.mapper), :Port => 4000) do
+    Thin::Logging.debug = options[:thin_debug]
+  end
+end

data/bin/nanite-agent

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'optparse'
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-agent [-flag] [argument]"
+  opts.define_head "Nanite Agent: ruby process that acts upon messages passed to it by a mapper."
+  opts.separator '*'*80
+
+  setup_common_options(opts, options, 'agent')
+
+  opts.on("-n", "--nanite NANITE_ROOT", "Specify the root of your nanite agent project.") do |nanite|
+    options[:root] = nanite
+  end
+
+  opts.on("--ping-time PINGTIME", "Specify how often the agents contacts the mapper") do |ping|
+    options[:ping_time] = ping
+  end
+  
+  opts.on("--actors-dir DIR", "Path to directory containing actors (NANITE_ROOT/actors by default)") do |dir|
+    options[:actors_dir] = dir
+  end
+  
+  opts.on("--actors ACTORS", "Comma separated list of actors to load (all ruby files in actors directory by default)") do |a|
+    options[:actors] = a.split(',')
+  end
+
+  opts.on("--initrb FILE", "Path to agent initialization file (NANITE_ROOT/init.rb by default)") do |initrb|
+    options[:initrb] = initrb
+  end
+
+  opts.on("--single-threaded", "Run all operations in one thread") do
+    options[:single_threaded] = true
+  end
+  
+  opts.on("--threadpool COUNT", Integer, "Number of threads to run all operations in") do |tps|
+    options[:threadpool_size] = tps
+  end
+  
+end
+
+opts.parse!
+
+if ARGV[0] == 'stop' || ARGV[0] == 'status'
+  agent = Nanite::Agent.new(options)
+  pid_file = Nanite::PidFile.new(agent.identity, agent.options)
+  unless pid = pid_file.read_pid
+    puts "#{pid_file} not found"
+    exit
+  end
+  if ARGV[0] == 'stop'
+    puts "Stopping nanite agent #{agent.identity} (pid #{pid})"
+    begin
+      Process.kill('TERM', pid)
+    rescue Errno::ESRCH
+      puts "Process does not exist (pid #{pid})"
+      exit
+    end
+    puts 'Done.'
+  else
+    if Process.getpgid(pid) != -1
+      psdata = `ps up #{pid}`.split("\n").last.split
+      memory = (psdata[5].to_i / 1024)
+      puts "The agent is alive, using #{memory}MB of memory"
+    else
+      puts "The agent is not running but has a stale pid file at #{pid_file}"
+    end
+  end
+  exit
+end
+
+EM.run do
+  Nanite.start_agent(options)
+end

data/bin/nanite-mapper

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'optparse'
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-mapper [-flags] [argument]"
+  opts.define_head "Nanite Mapper: clustered head unit for self assembling cluster of ruby processes."
+  opts.separator '*'*80
+
+  setup_mapper_options(opts, options)
+end
+
+opts.parse!
+
+if ARGV[0] == 'stop' || ARGV[0] == 'status'
+  mapper = Nanite::Mapper.new(options)
+  pid_file = Nanite::PidFile.new(mapper.identity, mapper.options)
+  unless pid = pid_file.read_pid
+    puts "#{pid_file} not found"
+    exit
+  end
+  if ARGV[0] == 'stop'
+    puts "Stopping nanite mapper #{mapper.identity} (pid #{pid})"
+    begin
+      Process.kill('TERM', pid)
+    rescue Errno::ESRCH
+      puts "Process does not exist (pid #{pid})"
+      exit
+    end
+    puts 'Done.'
+  else
+    if Process.getpgid(pid) != -1
+      psdata = `ps up #{pid}`.split("\n").last.split
+      memory = (psdata[5].to_i / 1024)
+      puts "The mapper is alive, using #{memory}MB of memory"
+    else
+      puts "The mapper is not running but has a stale pid file at #{pid_file}"
+    end
+  end
+  exit
+end
+
+EM.run do
+  Nanite.start_mapper(options)
+end

data/lib/nanite.rb

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'amqp'
+require 'mq'
+require 'json'
+require 'logger'
+require 'yaml'
+require 'openssl'
+
+$:.unshift File.dirname(__FILE__)
+require 'nanite/amqp'
+require 'nanite/util'
+require 'nanite/config'
+require 'nanite/packets'
+require 'nanite/identity'
+require 'nanite/console'
+require 'nanite/daemonize'
+require 'nanite/pid_file'
+require 'nanite/job'
+require 'nanite/mapper'
+require 'nanite/actor'
+require 'nanite/actor_registry'
+require 'nanite/streaming'
+require 'nanite/nanite_dispatcher'
+require 'nanite/agent'
+require 'nanite/cluster'
+require 'nanite/reaper'
+require 'nanite/log'
+require 'nanite/mapper_proxy'
+require 'nanite/security_provider'
+require 'nanite/security/cached_certificate_store_proxy'
+require 'nanite/security/certificate'
+require 'nanite/security/certificate_cache'
+require 'nanite/security/distinguished_name'
+require 'nanite/security/encrypted_document'
+require 'nanite/security/rsa_key_pair'
+require 'nanite/security/secure_serializer'
+require 'nanite/security/signature'
+require 'nanite/security/static_certificate_store'
+require 'nanite/serializer'
+
+module Nanite
+  VERSION = '0.4.1.2' unless defined?(Nanite::VERSION)
+
+  class MapperNotRunning < StandardError; end
+
+  class << self
+    attr_reader :mapper, :agent
+
+    def start_agent(options = {})
+      @agent = Nanite::Agent.start(options)
+    end
+
+    def start_mapper(options = {})
+      @mapper = Nanite::Mapper.start(options)
+    end
+
+    def request(*args, &blk)
+      ensure_mapper
+      @mapper.request(*args, &blk)
+    end
+
+    def push(*args)
+      ensure_mapper
+      @mapper.push(*args)
+    end
+
+    def ensure_mapper
+      @mapper ||= MapperProxy.instance
+      unless @mapper
+        raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper')
+      end
+    end
+  end
+end

data/lib/nanite/actor.rb

@@ -0,0 +1,71 @@
+module Nanite
+  # This mixin provides Nanite actor functionality.
+  #
+  # To use it simply include it your class containing the functionality to be exposed:
+  #
+  #   class Foo
+  #     include Nanite::Actor
+  #     expose :bar
+  #
+  #     def bar(payload)
+  #       # ...
+  #     end
+  #
+  #   end
+  module Actor
+
+    def self.included(base)
+      base.class_eval do 
+        include Nanite::Actor::InstanceMethods
+        extend  Nanite::Actor::ClassMethods
+      end # base.class_eval
+    end # self.included
+    
+    module ClassMethods
+      def default_prefix
+        to_s.to_const_path
+      end
+
+      def expose(*meths)
+        @exposed ||= []
+        meths.each do |meth|
+          @exposed << meth unless @exposed.include?(meth)
+        end
+      end
+
+      def provides_for(prefix)
+        return [] unless @exposed
+        @exposed.select do |meth|
+          if instance_methods.include?(meth.to_s) or instance_methods.include?(meth.to_sym)
+            true
+          else
+            Nanite::Log.warn("Exposing non-existing method #{meth} in actor #{name}")
+            false
+          end
+        end.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
+      end
+
+      def on_exception(proc = nil, &blk)
+        raise 'No callback provided for on_exception' unless proc || blk
+        @exception_callback = proc || blk
+      end
+
+      def exception_callback
+        @exception_callback
+      end
+      
+    end # ClassMethods     
+    
+    module InstanceMethods
+      # send nanite request to another agent (through the mapper)
+      def request(*args, &blk)
+        MapperProxy.instance.request(*args, &blk)
+      end
+      
+      def push(*args)
+        MapperProxy.instance.push(*args)
+      end
+    end # InstanceMethods
+    
+  end # Actor
+end # Nanite

data/lib/nanite/actor_registry.rb

@@ -0,0 +1,26 @@
+module Nanite
+  class ActorRegistry
+    attr_reader :actors
+
+    def initialize
+      @actors = {}
+    end
+
+    def register(actor, prefix)
+      raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
+      log_msg = "[actor] #{actor.class.to_s}"
+      log_msg += ", prefix #{prefix}" if prefix && !prefix.empty?
+      Nanite::Log.info(log_msg)
+      prefix ||= actor.class.default_prefix
+      actors[prefix.to_s] = actor
+    end
+
+    def services
+      actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
+    end
+
+    def actor_for(prefix)
+      actor = actors[prefix]
+    end
+  end # ActorRegistry
+end # Nanite