br-nanite 0.3.0

data/Rakefile

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require "spec/rake/spectask"
+begin; require 'rubygems'; rescue LoadError; end
+begin
+  require 'hanna/rdoctask'
+rescue LoadError
+  require 'rake/rdoctask'
+end
+require 'rake/clean'
+
+GEM = "br-nanite"
+VER = "0.3.0"
+AUTHOR = "Ezra Zygmuntowicz"
+EMAIL = "ezra@engineyard.com"
+HOMEPAGE = "http://github.com/ezmobius/nanite"
+SUMMARY = "self assembling fabric of ruby daemons"
+
+Dir.glob('tasks/*.rake').each { |r| Rake.application.add_import r }
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = ::VER
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+
+  s.bindir       = "bin"
+  s.executables  = %w( nanite-agent nanite-mapper nanite-admin )
+
+  s.add_dependency "extlib"
+  s.add_dependency('amqp', '>= 0.6.0')
+
+  s.require_path = 'lib'
+  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{VER}}
+end
+
+desc "Run unit specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ["--format", "specdoc", "--colour"]
+  t.spec_files = FileList["spec/**/*_spec.rb"]
+end
+
+desc 'Generate RDoc documentation'
+Rake::RDocTask.new do |rd|
+  rd.title = spec.name
+  rd.rdoc_dir = 'rdoc'
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("lib/**/*.rb", *spec.extra_rdoc_files)
+end
+CLOBBER.include(:clobber_rdoc)
+
+desc 'Generate and open documentation'
+task :docs => :rdoc do
+  case RUBY_PLATFORM
+  when /darwin/       ; sh 'open rdoc/index.html'
+  when /mswin|mingw/  ; sh 'start rdoc\index.html'
+  else 
+    sh 'firefox rdoc/index.html'
+  end
+end

data/TODO

@@ -0,0 +1,24 @@
+TODO:
+
+- The examples/crew.rb file is pointing towards a hard coded user dir.  Needs to be
+documented as part of a working example.
+- examples/async_rack_front/async_rack_front.ru needs to be documented and verified working.
+
+Ian:
+  - Sync Mapper/Agent#start with nanite-mapper/agent
+  - Update docs for Agent#start and Mapper#start
+  - Update docs in nanite-agent and nanite-mapper
+  - Ensure file transfer works
+  - Check secure stuff still works
+  - Check custom status_proc works
+  - Check documentation, only document public methods
+  - ensure the removal of threaded_actors option doesn't cause shit to block
+
+  - Look into using EM deferables for actors dispatch.
+  - Integration specs that spawn a small cluster of nanites
+  - Rename Ping to Status
+  - request/push should take *args for payload?
+
+Maybe:
+  - Make mapper queue durable and Results respect :persistent flag on the request
+  - Add a global result received callback

data/bin/nanite-admin

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+# To work without being installed as a gem:
+libdir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+$:.unshift libdir unless $:.include? libdir
+
+require 'nanite'
+require 'nanite/admin'
+require 'eventmachine'
+require 'thin'
+# IMPORTANT!
+# You need raggi's patched async version of Thin at the moment to use
+# the nanite-admin tool.
+#
+# raggi's Git repo contains a branch called 'async_for_rack' which contains the
+# version of Thin you want to install.  raggi has apparently removed the 'master'
+# branch of his Git repo so you may see a warning like that shown below.
+#
+# git clone git://github.com/raggi/thin.git thin-raggi-async
+# ...
+# warning: remote HEAD refers to nonexistent ref, unable to checkout.   <<==  IGNORE THIS
+#
+# cd thin-raggi-async/
+# git checkout --track -b async_for_rack origin/async_for_rack
+# warning: You appear to be on a branch yet to be born.                 <<==  IGNORE THIS
+# warning: Forcing checkout of origin/async_for_rack.                   <<==  IGNORE THIS
+# Branch async_for_rack set up to track remote branch refs/remotes/origin/async_for_rack.
+# Switched to a new branch "async_for_rack"
+
+# run : 'rake install' to build and install the Thin gem
+# cd <NANITE>
+# ./bin/nanite-admin
+
+# When you need to update this Thin install you should be able to do a 'git pull' on the
+# "async_for_rack" branch.
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'yaml'
+require "optparse"
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-admin [-flags] [argument]"
+  opts.define_head "Nanite Admin: a basic control interface for your nanite cluster."
+  opts.separator '*'*80
+
+  setup_mapper_options(opts, options)
+end
+
+opts.parse!
+
+EM.run do
+  Nanite.start_mapper(options)
+  puts "starting nanite-admin"
+  Rack::Handler::Thin.run(Nanite::Admin.new(Nanite.mapper), :Port => 4000)
+end

data/bin/nanite-agent

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'optparse'
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-agent [-flag] [argument]"
+  opts.define_head "Nanite Agent: ruby process that acts upon messages passed to it by a mapper."
+  opts.separator '*'*80
+
+  setup_common_options(opts, options, 'agent')
+
+  opts.on("-n", "--nanite NANITE_ROOT", "Specify the root of your nanite agent project.") do |nanite|
+    options[:root] = nanite
+  end
+
+  opts.on("--ping-time PINGTIME", "Specify how often the agents contacts the mapper") do |ping|
+    options[:ping_time] = ping
+  end
+end
+
+opts.parse!
+
+EM.run do
+  Nanite.start_agent(options)
+end

data/bin/nanite-mapper

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/nanite'
+require 'optparse'
+
+include Nanite::CommonConfig
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: nanite-mapper [-flags] [argument]"
+  opts.define_head "Nanite Mapper: clustered head unit for self assembling cluster of ruby processes."
+  opts.separator '*'*80
+
+  setup_mapper_options(opts, options)
+end
+
+opts.parse!
+
+EM.run do
+  Nanite.start_mapper(options)
+end

data/lib/nanite.rb

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'amqp'
+require 'mq'
+require 'json'
+require 'logger'
+require 'yaml'
+
+$:.unshift File.dirname(__FILE__)
+require 'nanite/amqp'
+require 'nanite/util'
+require 'nanite/config'
+require 'nanite/packets'
+require 'nanite/identity'
+require 'nanite/console'
+require 'nanite/daemonize'
+require 'nanite/job'
+require 'nanite/mapper'
+require 'nanite/actor'
+require 'nanite/actor_registry'
+require 'nanite/streaming'
+require 'nanite/dispatcher'
+require 'nanite/agent'
+require 'nanite/cluster'
+require 'nanite/reaper'
+require 'nanite/serializer'
+require 'nanite/log'
+
+module Nanite
+  VERSION = '0.3.0' unless defined?(Nanite::VERSION)
+
+  class MapperNotRunning < StandardError; end
+
+  class << self
+    attr_reader :mapper, :agent
+
+    def start_agent(options = {})
+      @agent = Nanite::Agent.start(options)
+    end
+
+    def start_mapper(options = {})
+      @mapper = Nanite::Mapper.start(options)
+    end
+
+    def request(*args, &blk)
+      ensure_mapper
+      @mapper.request(*args, &blk)
+    end
+
+    def push(*args)
+      ensure_mapper
+      @mapper.push(*args)
+    end
+
+    def ensure_mapper
+      raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper') unless @mapper
+    end
+  end
+end

data/lib/nanite/actor.rb

@@ -0,0 +1,64 @@
+module Nanite
+  # This mixin provides Nanite actor functionality.
+  #
+  # To use it simply include it your class containing the functionality to be exposed:
+  #
+  #   class Foo
+  #     include Nanite::Actor
+  #     expose :bar
+  #
+  #     def bar(payload)
+  #       # ...
+  #     end
+  #
+  #   end
+  module Actor
+
+    def self.included(base)
+      base.class_eval do 
+        include Nanite::Actor::InstanceMethods
+        extend  Nanite::Actor::ClassMethods
+      end # base.class_eval
+    end # self.included
+    
+    module ClassMethods
+      def default_prefix
+        to_s.to_const_path
+      end
+
+      def expose(*meths)
+        @exposed ||= []
+        meths.each do |meth|
+          @exposed << meth unless @exposed.include?(meth)
+        end
+      end
+
+      def provides_for(prefix)
+        return [] unless @exposed
+        @exposed.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
+      end
+
+      def on_exception(proc = nil, &blk)
+        raise 'No callback provided for on_exception' unless proc || blk
+        if Nanite::Actor == self
+          raise 'Method name callbacks cannot be used on the Nanite::Actor superclass' if Symbol === proc || String === proc
+          @superclass_exception_callback = proc || blk
+        else
+          @instance_exception_callback = proc || blk
+        end
+      end
+
+      def superclass_exception_callback
+        @superclass_exception_callback
+      end
+
+      def instance_exception_callback
+        @instance_exception_callback
+      end
+    end # ClassMethods     
+    
+    module InstanceMethods
+    end # InstanceMethods
+    
+  end # Actor
+end # Nanite

data/lib/nanite/actor_registry.rb

@@ -0,0 +1,25 @@
+module Nanite
+  class ActorRegistry
+    attr_reader :actors, :log
+
+    def initialize(log)
+      @log = log
+      @actors = {}
+    end
+
+    def register(actor, prefix)
+      raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
+      log.info("Registering #{actor.inspect} with prefix #{prefix.inspect}")
+      prefix ||= actor.class.default_prefix
+      actors[prefix.to_s] = actor
+    end
+
+    def services
+      actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
+    end
+
+    def actor_for(prefix)
+      actor = actors[prefix]
+    end
+  end # ActorRegistry
+end # Nanite 

data/lib/nanite/admin.rb

@@ -0,0 +1,147 @@
+require 'rack'
+
+module Nanite
+  # This is a Rack app for nanite-admin.  You need to have an async capable
+  # version of Thin installed for this to work.  See bin/nanite-admin for install
+  # instructions.
+  class Admin
+    def initialize(mapper)
+      @mapper = mapper
+    end
+
+    AsyncResponse = [-1, {}, []].freeze
+
+    def call(env)
+      req = Rack::Request.new(env)
+      if cmd = req.params['command']
+        @command = cmd
+        @selection = req.params['type'] if req.params['type']
+        
+        options = {}
+        case @selection
+        when 'least_loaded', 'random', 'all', 'rr'
+          options[:selector] = @selection
+        else
+          options[:target] = @selection
+        end
+
+        @mapper.request(cmd, req.params['payload'], options) do |response|
+          env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response))]]
+        end
+        AsyncResponse
+      else
+        [200, {'Content-Type' => 'text/html'}, layout]
+      end
+    end
+
+    def services
+      buf = "<select name='command'>"
+      @mapper.cluster.nanites.map{|n,s| s[:services] }.flatten.each do |srv|
+        buf << "<option value='#{srv}' #{@command == srv ? 'selected="true"' : ''}>#{srv}</option>"
+      end
+      buf << "</select>"
+      buf
+    end
+
+    def ul(hash)
+      buf = "<ul>"
+      hash.each do |k,v|
+        buf << "<li><div class=\"nanite\">#{k}:</div><div class=\"response\">#{v.inspect}</div></li>"
+      end
+      buf << "</ul>"
+      buf
+    end
+
+    def layout(content=nil)
+      %Q{
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns='http://www.w3.org/1999/xhtml'>
+          <head>
+            <meta content='text/html; charset=utf-8' http-equiv='Content-Type' />
+            <meta content='en' http-equiv='Content-Language' />
+            <meta content='Engineyard' name='author' />
+            <title>Nanite Control Tower</title>
+
+            <!-- Google AJAX Libraries API -->
+            <script src="http://www.google.com/jsapi"></script>
+            <script type="text/javascript">
+              google.load("jquery", "1");
+            </script>
+
+            <script type="text/javascript">
+            $(document).ready(function(){
+
+              // set the focus to the payload field
+              $("#payload").focus();
+
+            });
+            </script>
+
+            <style>
+              body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
+              ul {margin: 0; padding: 0; margin-left: 10px}
+              li {list-style-type: none; margin-bottom: 6px}
+              li .nanite {font-weight: bold; font-size: 12px}
+              li .response {padding: 8px}
+              h1, h2, h3 {margin-top: none; padding: none; margin-left: 40px;}
+              h1 {font-size: 22px; margin-top: 40px; margin-bottom: 30px; border-bottom: 1px solid #ddd; padding-bottom: 6px;
+                margin-right: 40px}
+              h2 {font-size: 16px;}
+              h3 {margin-left: 0; font-size: 14px}
+              .section {border: 1px solid #ccc; background-color: #fefefe; padding: 10px; margin: 20px 40px; padding: 20px;
+                font-size: 14px}
+              #footer {text-align: center; color: #AAA; font-size: 12px}
+            </style>
+
+          </head>
+
+          <body>
+            <div id="header">
+              <h1>Nanite Control Tower</h1>
+            </div>
+
+            <h2>#{@mapper.options[:vhost]}</h2>
+            <div class="section">
+              <form method="post" action="/">
+                <input type="hidden" value="POST" name="_method"/>
+
+                  <label>Send</label>
+                  <select name="type">
+                    <option #{@selection == 'least_loaded' ? 'selected="true"' : ''} value="least_loaded">the least loaded nanite</option>
+                    <option #{@selection == 'random' ? 'selected="true"' : ''} value="random">a random nanite</option>
+                    <option #{@selection == 'all' ? 'selected="true"' : ''} value="all">all nanites</option>
+                    <option #{@selection == 'rr' ? 'selected="true"' : ''} value="rr">a nanite chosen by round robin</option>
+                    #{@mapper.cluster.nanites.map {|k,v| "<option #{@selection == k ? 'selected="true"' : ''} value='#{k}'>#{k}</option>" }.join}
+                  </select>
+
+                  <label>providing service</label>
+                  #{services}
+
+                  <label>the payload</label>
+                  <input type="text" class="text" name="payload" id="payload"/>
+
+                  <input type="submit" class="submit" value="Go!" name="submit"/>
+              </form>
+
+              #{"<h3>Responses</h3>" if content}
+              #{content}
+            </div>
+
+            <h2>Running nanites</h2>
+            <div class="section">
+              #{"No nanites online." if @mapper.cluster.nanites.size == 0}
+              <ul>
+                #{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].inspect}</li>" }.join}
+              </ul>
+            </div>
+            <div id="footer">
+              Nanite #{Nanite::VERSION}
+              <br />
+              &copy; 2009 a bunch of random geeks
+            </div>
+          </body>
+        </html>
+      }
+    end
+  end
+end