nodule 0.0.34

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.yardoc/*
+doc/*

data/.yardopts

@@ -0,0 +1 @@
+--no-private --protected

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+source "http://gems.sv2"  # For prerelease cassandra gem
+
+# Specify your gem's dependencies in nodule.gemspec
+gemspec

data/README.md

@@ -0,0 +1,100 @@
+Warning
+-------
+
+Nodule is really new software (all of 2 weeks old). It was extracted from a larger project
+where it is used for some really large integration tests. We'll remove this message once
+Nodule has a reasonable set of tests of its own.
+
+Overview
+--------
+
+Nodule is an integration test harness that simplifies the setup and teardown of multi-process
+or large multi-component applications.
+
+When setting up complex tests with multiple processes, a lot of work ends up going into generating
+configuration, command-line parameters, and other fiddly things that are their own sources
+of annoying bugs. Nodule tries to tidy some of that up by providing a way to define a test
+topology that automatically injects data in the right places lazily so it doesn't all have to
+be done up-front.
+
+Symbols
+-------
+
+In as many places as possible, Nodule allows you to use a symbol as a placeholder for a value,
+which it will automatically resolve when it's needed and no earlier, so you aren't forced to worry
+about ordering.
+
+Saying ":file => Nodule::Tempfile.new" means that :file will resolve to a temporary file's path
+(via .to_s) in any place it appears as a placeholder. :file can be used before it's associated with
+a tempfile.
+
+Procs
+-----
+
+In many places, procs can be used as placeholders and will be automatically called when needed.
+
+Arrays
+------
+
+The process module requires commands to be specified as argv-style arrays. The array is scanned
+for placeholders, which are converted as already described. The one addition is sub-arrays, which
+are resolved and concatenated without padding. This is useful for parameters that use '=' without
+spaces, for example, "dd if=<filename>" would be specified as ['dd', ['if=', :file]]. These sub-arrays
+are resolved recursively, so multiple levels are allowed.
+
+Example
+-------
+
+    #!/usr/bin/env ruby
+
+    require "test/unit"
+    require 'nodule/process'
+    require 'nodule/topology'
+    require 'nodule/tempfile'
+    require 'nodule/console'
+
+    class NoduleSimpleTest < Test::Unit::TestCase
+      def setup
+        @topo = Nodule::Topology.new(
+          :greenio => Nodule::Console.new(:fg => :green),
+          :redio   => Nodule::Console.new(:fg => :red),
+          :file1   => Nodule::Tempfile.new(".html"),
+          :wget    => Nodule::Process.new(
+            '/usr/bin/wget', '-O', :file1, 'http://www.ooyala.com',
+            :stdout => :greenio, :stderr => :redio
+          )
+        )
+
+        @topo.run_serially
+      end
+
+      def teardown
+        @topo.cleanup
+      end
+
+      def test_heartbeat
+        filename = @topo[:file1].to_s
+        assert File.exists? filename
+      end
+    end
+
+Authors
+-------
+
+* Viet Nguyen
+* Noah Gibbs
+* Al Tobey
+* Jay Bhat
+
+Dependencies
+------------
+
+* Ruby 1.9 (tested on 1.9.2p290)
+* rainbow
+* ffi-rzmq (for Nodule::ZeroMQ, optional)
+
+License
+-------
+
+Nodule is released under [the MIT license](http://www.opensource.org/licenses/mit-license.php).
+

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+namespace "test" do
+  desc "Unit tests"
+  Rake::TestTask.new(:units) do |t|
+    t.libs += ["test"]  # require from test subdir
+    t.test_files = Dir["test/nodule_*_test.rb"]
+    t.verbose = true
+  end
+end
+
+task :test => ["test:units"] do
+  puts "All tests completed..."
+end

data/ci_jobs/nodule-units/run.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -l
+
+if [[ -z $WORKSPACE ]] ; then
+  # Support execution from the shell
+  export PROJECT_DIR=$(pwd);
+else
+  export PROJECT_DIR=$WORKSPACE/nodule;
+fi
+
+cd $PROJECT_DIR
+
+echo "Loading RVM..."
+source $HOME/.rvm/scripts/rvm || echo "couldn't load RVM script"
+echo "Using Ruby 1.9.2"
+rvm use --create 1.9.2-p290@nodule
+
+echo "Starting bundle update"
+bundle update
+
+# Disallow errors
+set -e
+
+echo START TASK: tests
+bundle exec rake test
+echo END TASK

data/examples/cat_test.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require "fileutils"
+
+require "test/unit"
+require 'nodule/process'
+require 'nodule/topology'
+require 'nodule/tempfile'
+require 'nodule/console'
+
+class NoduleDDCatTest < Test::Unit::TestCase
+  BYTES=65536
+
+  def setup
+    @topo = Nodule::Topology.new(
+      :redio   => Nodule::Console.new(:fg => :red),
+      :greenio => Nodule::Console.new(:fg => :green),
+      :file1   => Nodule::Tempfile.new(:suffix => ".rand"),
+      :file2   => Nodule::Tempfile.new(:suffix => ".copy"),
+      :dd      => Nodule::Process.new(
+        '/bin/dd', 'if=/dev/urandom', ['of=', :file1], "bs=#{BYTES}", 'count=1',
+        :stderr => :redio, :verbose => :greenio
+      ),
+      :ls      => Nodule::Process.new('ls', '-l', :stdout => :greenio),
+      :copy    => Nodule::Process.new('/bin/cp', :file1, :file2, :stderr => :redio),
+    )
+
+    # start up and run in order
+    @topo.run_serially
+  end
+
+  def teardown
+    @topo.cleanup
+  end
+
+  def test_heartbeat
+    file1 = @topo[:file1]
+    file2 = @topo[:file2]
+
+    assert_equal BYTES, File.new(file1.to_s).size
+    assert_equal BYTES, File.new(file2.to_s).size
+    assert FileUtils.cmp(file1.to_s, file2.to_s)
+  end
+end
+

data/examples/wget.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require "test/unit"
+require 'nodule/process'
+require 'nodule/topology'
+require 'nodule/tempfile'
+require 'nodule/console'
+require 'multi_json'
+
+class NoduleSimpleTest < Test::Unit::TestCase
+  def setup
+    @topo = Nodule::Topology.new(
+      :greenio => Nodule::Console.new(:fg => :green),
+      :redio   => Nodule::Console.new(:fg => :red),
+      :file1   => Nodule::Tempfile.new(:suffix => ".html"),
+      :wget    => Nodule::Process.new(
+        '/usr/bin/wget', '-O', :file1, 'http://www.ooyala.com',
+        :stdout => :greenio
+      )
+    )
+
+    @topo.start_all
+  end
+
+  def teardown
+    @topo.cleanup
+  end
+
+  def test_heartbeat
+    @topo[:wget].wait
+    filename = @topo[:file1].to_s
+    assert File.exists? filename
+  end
+end
+

data/lib/nodule/alarm.rb

@@ -0,0 +1,25 @@
+# Import alarm() from libc.
+
+require 'nodule/base'
+require 'ffi'
+
+module Nodule
+  module PosixAlarmImport
+    extend FFI::Library
+    ffi_lib FFI::Library::LIBC
+    # unistd.h: unsigned alarm(unsigned seconds);
+    attach_function :alarm, [ :uint ], :uint
+  end
+
+  class Alarm < Base
+    def initialize(opts={})
+      if opts[:timeout]
+        PosixAlarmImport.alarm(opts[:timeout])
+      end
+
+      Signal.trap("ALRM") { abort "Got SIGALRM. Aborting."; }
+
+      super(opts)
+    end
+  end
+end

data/lib/nodule/base.rb

@@ -0,0 +1,258 @@
+require 'nodule/version'
+require 'nodule/topology'
+
+module Nodule
+  @sequence = 0
+  def self.next_seq
+    @sequence += 1
+  end
+
+  class Base
+    attr_reader :readers, :running, :read_count, :topology
+    attr_accessor :prefix
+
+    #
+    # Create a new Nodule handler. This is meant to be a bass class for higher-level
+    # Nodule types.
+    #
+    # @param [Hash{Symbol => String,Symbol,Proc}] opts
+    # @option opts [String] :prefix text prefix for output/logs/etc.
+    # @option opts [Symbol, Proc] :reader a symbol for a built-in reader, e.g. ":drain" or a proc
+    # @option opts [Enumerable] :readers a list of readers instead of one :reader
+    # @option opts [TrueClass] :run run immediately
+    # @option opts [TrueClass] :verbose print verbose information to STDERR
+    #
+    def initialize(opts={})
+      @read_count = 0
+      @readers ||= []
+      @output  ||= []
+      @prefix  = opts[:prefix] || ''
+      @verbose = opts[:verbose]
+      @done    = false
+      @topology = nil
+      @capture_enabled = false
+
+      add_readers(opts[:reader]) if opts[:reader]
+
+      run if opts[:run]
+    end
+
+    def join_topology!(t, name='')
+      @topology = t
+      @prefix = name if @prefix.nil? || @prefix.empty?
+    end
+
+    def run
+      @done = false
+
+      unless @topology
+        @toplogy = Nodule::Topology.new(:auto => self)
+      end
+
+      # automatically determine a prefix for console output based on the key name known to the topology
+      if name = @topology.key(self)
+        @prefix = "[#{name}]: "
+      end
+    end
+
+    def stop
+      @done = true
+    end
+
+    def stop!
+      @done = true
+    end
+
+    def done?
+      @done
+    end
+
+    def wait(timeout=nil)
+    end
+
+    #
+    # Returns whether or not any output has been captured.
+    # Will raise an exception if capture is not enabled.
+    # @return [TrueClass,FalseClass]
+    #
+    def output?
+      raise "output is not captured unless you enable :capture" unless @capture_enabled
+      @output.any?
+    end
+
+    #
+    # Returns a copy of currently captured output. Line data is not chomped or anything of the sort.
+    # Will raise an exception if capture is not enabled.
+    # @return [Array<String>]
+    #
+    def output
+      raise "output is not captured unless you enable :capture" unless @capture_enabled
+      @output.clone
+    end
+
+    #
+    # Returns the captured output and clears the buffer (just like clear! but with a return value).
+    # Will raise an exception if capture is not enabled.
+    # @return [Array<String>]
+    #
+    def output!
+      raise "output is not captured unless you enable :capture" unless @capture_enabled
+      out = @output.clone
+      clear!
+      out
+    end
+
+    #
+    # Reset the read count to zero and clear any captured output.
+    #
+    def clear!
+      @read_count = 0
+      @output.clear
+    end
+
+    #
+    # A dead-simple backoff loop. Calls your block every @sleeptime seconds, increasing the sleep
+    # time by sleeptime every iteration, up to timeout, at which point false will be returned. If
+    # the block returns an non-false/nil value, it is returned.
+    # @param [Float,Fixnum] timeout
+    # @param [Float,Fixnum] sleeptime
+    # @yield block that returns false to continue waiting, non-false to return that value
+    # @return block return value or false if timeout
+    #
+    def wait_with_backoff(timeout, sleeptime=0.01)
+      raise "a block to execute on each iteration is required" unless block_given?
+      started = Time.now
+      loop do
+        val = yield
+        return val if val
+        return false if Time.now - started > timeout
+        sleep sleeptime
+        if sleeptime < timeout / 4
+          sleeptime += sleeptime
+        end
+      end
+    end
+
+    #
+    # Wait in a sleep loop until a condition is true or the timeout is
+    # reached.  On timeout an exception is raised.  Has no impact on
+    # normal readers.
+    #
+    # @param [Hash] opts Options for how to read and wait
+    # @option opts [Float] :max_sleep maximum number of seconds to wait
+    # @option opts [Float] :sleep_by how long to sleep by each time, default 0.1
+    # @yield block to call to check condition, returns true or false
+    # @example act.read_until(:max_sleep => 1.0) { File.exist?("/tmp/file") }
+    #
+    def read_until(opts = {}, &block)
+      raise "No block given to read_until!" unless block_given?
+      started = Time.now
+      until block.call
+        sleep (opts[:sleep_by] || 0.1)
+        if Time.now - started >= (opts[:max_sleep] || 10.0)
+          raise "Timeout!"
+        end
+      end
+    end
+
+    #
+    # Wait in a sleep(0.1) loop for the number of reads on the handler to reach <count>.
+    # Returns when the number of reads is given. On timeout, if a block was provided,
+    # it's called before return. Otherwise, an exception is raised.
+    # Has no impact on normal readers.
+    #
+    # @param [Fixnum] count how many reads to wait for
+    # @param [Float] max_sleep maximum number of seconds to wait for the count
+    # @yield optional block to run on timeout
+    # @example act.require_read_timeout 1, 10 { fail }
+    # @example act.require_read_timeout 1, 10 rescue nil
+    #
+    def require_read_count(count, max_sleep=10)
+      read_until(:max_sleep => max_sleep) { @read_count == count }
+    end
+
+    #
+    # Add a reader action. Can be a block which will be executed for each unit of input, :capture
+    # to capture all items emitted by the target to a list (accessible with .output), :ignore, or
+    # nil (which will be ignored).
+    # @param [Symbol, Proc] action Action to take on each item read from the handler
+    # @option action [Symbol] :capture capture the items into an array (access with .output)
+    # @option action [Symbol] :drain read items but throw them away
+    # @option action [Symbol] :stderr print the item to stderr (with prefix, in color)
+    # @option action [Symbol] :ignore don't do anything
+    # @option action [Proc] run the block, passing it the item (e.g. a line of stdout)
+    # @yield optionally pass a proc in with normal block syntax
+    #
+    def add_reader(action=nil, &block)
+      if block_given?
+        @readers << block
+        return unless action
+      end
+
+      if action.respond_to? :call
+        @readers << action
+      elsif action == :capture
+        @capture_enabled = true
+        @readers << proc { |item| @output.push(item) }
+      elsif action == :drain
+        @readers << proc { |_| }
+      elsif action == :ignore
+      # nothing to do here
+      # if it's an unrecognized symbol, defer resolution against the containing topology
+      elsif action.kind_of? Symbol
+        @readers << proc do |item|
+          raise "Topology is not set up!" unless @topology
+          raise ":#{action} is not a valid topology symbol in #{@topology.to_hash.inspect}" unless @topology.has_key?(action)
+          @topology[action].run_readers(item, self)
+        end
+      else
+        raise ArgumentError.new "Invalid add_reader class: #{action.class}"
+      end
+    end
+
+    #
+    # Add reader arguments with add_reader. Can be a single item or list.
+    # @param [Array<Symbol,Proc,Nodule::Base>] args
+    #
+    def add_readers(*args)
+      args.flatten.each do |reader|
+        add_reader(reader)
+      end
+    end
+
+    #
+    # Run all of the registered reader blocks. The block should expect a single argument
+    # that is an item of input. If the block has an arity of two, it will also be handed
+    # the nodule object provided to run_readers (if it was provided; no guarantee is made that
+    # it will be available). The arity-2 version is provided mostly as a clean way for
+    # Nodule::Console to add prefixes to output, but could be useful elsewhere.
+    # @param [Object] item the item to pass to the readers, often a String (but could be anything)
+    # @param [Nodule::Base] handler that generated the item, optional, untyped
+    #
+    def run_readers(item, src=nil)
+      @read_count += 1
+      verbose "READ(#{@read_count}): #{item}"
+      @readers.each do |reader|
+        if reader.arity == 2
+          reader.call(item, src)
+        else
+          reader.call(item)
+        end
+      end
+    end
+
+    #
+    # Verbose Nodule output.
+    # @param [Array<String>] out strings to output, will be joined with ' '
+    #
+    def verbose(*out)
+      if @verbose
+        if @topology.respond_to? :[] and @topology[@verbose]
+          @topology[@verbose].run_readers out.join(' ')
+        else
+          STDERR.print "#{out.join(' ')}\n"
+        end
+      end
+    end
+  end
+end