nodule 0.0.34

data/lib/nodule/util.rb

@@ -0,0 +1,56 @@
+require 'socket'
+
+module Nodule
+  module Util
+    @seen = {}
+
+    def self.random_tcp_port(max_tries=500)
+      self._random_port do |port|
+        TCPServer.new port
+      end
+    end
+
+    def self.random_udp_port(max_tries=500)
+      self._random_port do |port|
+        socket = UDPSocket.new
+        socket.bind("0.0.0.0", port)
+        socket
+      end
+    end
+
+    #
+    # Try random ports > 10_000 looking for one that's free.
+    # @param [Fixnum] max number of tries to find a free port
+    # @return [Fixnum] port number
+    # @yield [Fixnum] port
+    #
+    def self._random_port(max_tries=500)
+      tries = 0
+
+      while tries < max_tries
+        port = random_port
+        next if @seen.has_key? port
+
+        socket = begin
+          yield port
+        rescue Errno::EADDRINUSE
+          @seen[port] = true
+          tries += 1
+          next
+        end
+
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+        socket.close
+        return port
+      end
+    end
+
+    #
+    # Return a random integer between 10_000 and 65_534
+    # @return [Fixnum]
+    #
+    def self.random_port
+      rand(55534) + 10_000
+    end
+  end
+end

data/lib/nodule/version.rb

@@ -0,0 +1,3 @@
+module Nodule
+  VERSION = "0.0.34"
+end

data/lib/nodule/zeromq.rb

@@ -0,0 +1,280 @@
+require 'nodule/tempfile'
+require 'ffi-rzmq'
+require 'thread'
+
+module Nodule
+  #
+  # A resource for setting up and testing ZeroMQ message flows. The most basic usage will provide
+  # auto-generated IPC URI's, which can be handy for testing. More advanced usage uses the built-in
+  # tap device to sniff messages while they're in-flight.
+  #
+  class ZeroMQ < Tempfile
+    attr_reader :ctx, :uri, :method, :type, :limit, :error_count
+
+    private
+
+    def setsockopt(socket, option, value)
+      if option == :hwm && ::ZMQ::LibZMQ.version3?
+        rc = socket.setsockopt(::ZMQ::SNDHWM, value)
+        rc = socket.setsockopt(::ZMQ::RCVHWM, value) if rc > -1
+      else
+        option = ::ZMQ::HWM if option == :hwm
+        rc = socket.setsockopt(option, value)
+      end
+
+      rc
+    end
+
+    public
+
+    #
+    # @param [Hash{Symbol => Object}] opts named parameters
+    # @option [String] :uri either :gen/:generate or a string, :gen means generate an IPC URI,
+    #                       a string must be a valid URI.
+    # @option [Fixnum] :limit exit the read loop after :limit messages are received
+    # @option [String,Symbol] :connect create a socket and connect to the URI
+    # @option [String,Symbol] :bind create a socket and bind to the URI
+    #
+    # :connect and :bind are allowed at the same time and must be of the same socket type.
+    # ZMQ::SUB sockets that are connected/bound will subscribe to "" by default.
+    #
+    # For the rest of the options, see Hastur::Test::Resource::Base.
+    #
+    def initialize(opts)
+      opts[:suffix] ||= '.zmq'
+
+      super(opts)
+
+      @ctx = ::ZMQ::Context.new
+      @zmq_thread = nil
+      @error_count = 0
+      @sockprocs = []
+      @limit = nil
+      @timeout_started = false
+      @stopped = false
+
+      # Sockets cannot be used across thread boundaries, so use a ZMQ::PAIR socket both to synchronize thread
+      # startup and pass writes form main -> thread. The .socket method will return the PAIR socket.
+      @pipe_uri = "inproc://pair-#{Nodule.next_seq}"
+      @pipe = @ctx.socket(::ZMQ::PAIR)
+      @child = @ctx.socket(::ZMQ::PAIR)
+      setsockopt(@pipe, :hwm, 1)
+      setsockopt(@child, :hwm, 1)
+      setsockopt(@pipe, ::ZMQ::LINGER, 1.0)
+      setsockopt(@child, ::ZMQ::LINGER, 1.0)
+      @pipe.bind(@pipe_uri)
+      @child.connect(@pipe_uri)
+
+      case opts[:uri]
+        # Socket files are specified so they land in PWD, in the future we might want to specify a temp
+        # dir, but that has a whole different bag of issues, so stick with simple until it's needed.
+        when :gen, :generate
+          @uri = "ipc://#{@file.to_s}"
+        when String
+          @uri = opts[:uri]
+        else
+          raise ArgumentError.new "Invalid URI specifier: (#{opts[:uri].class}) '#{opts[:uri]}'"
+      end
+
+      if opts[:connect] and opts[:bind] and opts[:connect] != opts[:bind]
+        raise ArgumentError.new "ZMQ socket types must be the same when enabling :bind and :connect"
+      end
+
+      # only set type and create a socket if :bind or :connect is specified
+      # otherwise, the caller probably just wants to generate a URI, or possibly
+      # use a pre-created socket? (not supported yet)
+      if @type = (opts[:connect] || opts[:bind])
+        @socket = @ctx.socket(@type)
+        setsockopt(@socket, :hwm, 1)
+        setsockopt(@socket, ::ZMQ::LINGER, 1.0)
+
+        if opts[:connect]
+          @sockprocs << proc { @socket.connect(@uri) } # deferred
+        end
+
+        if opts[:bind]
+          @sockprocs << proc { @socket.bind(@uri) } # deferred
+        end
+
+        # by default, subscribe to "" on ZMQ::SUB sockets, since that's what we want
+        # the vast majority of the time. Also allow specifying a subscription for those
+        # times we want something else.
+        if @type == ::ZMQ::SUB
+          if opts[:subscribe]
+            @sockprocs << proc { subscribe(opts[:subscribe]) }
+          else
+            @sockprocs << proc { subscribe("") }
+          end
+        end
+      end
+
+      if opts[:limit]
+        @limit = opts[:limit]
+      end
+    end
+
+    def run
+      super
+      return if @sockprocs.empty?
+
+      # wrap the block in a block so errors don't simply vanish until join time
+      @zmq_thread = Thread.new do
+        Thread.current.abort_on_exception
+
+        # sockets have to be created inside the thread that uses them
+        @sockprocs.each { |p| p.call }
+
+        _zmq_read()
+        verbose "child thread #{Thread.current} shutting down"
+
+        @child.close
+        @socket.close if @socket
+      end
+
+      Thread.pass
+
+      @stopped = @zmq_thread.alive? ? false : true
+    end
+
+    def socket
+      @pipe
+    end
+
+    #
+    # For PUB sockets only, subscribe to a prefix.
+    # @param [String] subscription prefix, usually ""
+    #
+    def subscribe(subscription)
+      @pipe.send_strings ["subscribe", subscription]
+    end
+
+    def done?
+      @stopped
+    end
+
+    #
+    # Wait for the ZMQ thread to exit on its own, mostly useful with :limit => Fixnum.
+    #
+    # This does not signal the child thread.
+    #
+    def wait(timeout=60)
+      countdown = timeout.to_f
+
+      while countdown > 0
+        if @zmq_thread and @zmq_thread.alive?
+          sleep 0.1
+          countdown = countdown - 0.1
+        else
+          break
+        end
+      end
+
+      super()
+    end
+
+    #
+    # If the thread is still alive, force an exception in the thread and
+    # continue to do the things stop does.
+    #
+    def stop!
+      if @zmq_thread.alive?
+        STDERR.puts "force stop! called, issuing Thread.raise"
+        @zmq_thread.raise "force stop! called"
+      end
+
+      stop
+      wait 1
+
+      @zmq_thread.join if @zmq_thread
+      @pipe.close if @pipe
+
+      @stopped = true
+    end
+
+    #
+    # send a message to the child thread telling it to exit and join the thread
+    #
+    def stop
+      return if @stopped
+
+      @pipe.send_strings(["exit"], 1)
+
+      Thread.pass
+
+      super
+
+      @zmq_thread.join if @zmq_thread
+      @pipe.close if @pipe
+
+      @stopped = true
+    end
+
+    #
+    # Return the URI generated/provided for this resource. For tapped devices, the "front" side
+    # of the tap is returned.
+    #
+    def to_s
+      @uri
+    end
+
+    private
+
+    #
+    # Run a poll loop (using the zmq poller) on a 1/5 second timer, reading data
+    # from the socket and calling the registered procs.
+    # If :limit was set, will exit after that many messages are seen/processed.
+    # Otherwise, exits on the next iteration if the mutex is locked (which is done in stop).
+    # Takes no arguments, doesn't return anything meaningful.
+    #
+    def _zmq_read
+      return unless @socket
+      @poller = ::ZMQ::Poller.new
+
+      @poller.register_readable @socket
+      @poller.register_readable @child
+
+      # read on the socket(s) and call the registered reader blocks for every message, always using
+      # multipart and converting to ruby strings to avoid ZMQ::Message cleanup issues.
+      count = 0
+      @running = true
+      while @running
+        rc = @poller.poll(1)
+        unless rc > 0
+          sleep 0.01
+          next
+        end
+
+        @poller.readables.each do |sock|
+          rc = sock.recv_strings messages=[]
+          if rc > -1
+            if sock == @socket
+              count += 1
+              run_readers(messages, self)
+            # the main thread can send messages through to be resent or "exit" to shut down this thread
+            elsif sock == @child
+              if messages[0] == "exit"
+                verbose "Got exit message. Exiting."
+                @running = false
+              elsif messages[0] == "subscribe"
+                @socket.setsockopt ::ZMQ::SUBSCRIBE, messages[1]
+              else
+                @socket.send_strings messages
+              end
+            else
+              raise "BUG: couldn't match socket to a known socket"
+            end
+
+            # stop reading after a set number of messages, regardless of whether there are any more waiting
+            break if @limit and count >= @limit
+            break unless @running
+          else
+            @error_count += 1
+            break
+          end
+        end # @poller.readables.each
+
+        break if @limit and count >= @limit
+      end # while @running
+    end
+  end
+end

data/lib/nodule.rb

@@ -0,0 +1,10 @@
+require "nodule/version"
+
+require "nodule/console"
+require "nodule/monkeypatch"
+require "nodule/process"
+require "nodule/base"
+require "nodule/topology"
+
+module Nodule
+end

data/nodule.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "nodule/version"
+
+Gem::Specification.new do |s|
+  s.name        = "nodule"
+  s.version     = Nodule::VERSION
+  s.authors     = ["Al Tobey", "Noah Gibbs", "Viet Nguyen", "Jay Bhat"]
+  s.email       = ["al@ooyala.com", "noah@ooyala.com", "viet@ooyala.com", "bhat@ooyala.com"]
+  s.homepage    = ""
+  s.summary     = %q{Nodule starts, stops, tests and redirects groups of processes}
+  s.description = %q{Nodule lets you declare Topologies of processes, which can be started or stopped together.  You can also redirect sockets, set up interprocess communication, make assertions on captured packets between processes and generally monitor or change the interaction of any of your processes.  Nodule is great for integration testing or for bringing up complicated interdependent sets of processes on a single host.}
+
+  s.rubyforge_project = "nodule"
+  s.required_ruby_version = ">= 1.9.2"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "minitest"
+  # s.add_runtime_dependency "rest-client"
+  s.add_runtime_dependency "ffi-rzmq"
+  s.add_runtime_dependency "cassandra", "=0.12.2ooyala2"
+  s.add_runtime_dependency "rainbow"
+end

data/test/helper.rb

@@ -0,0 +1 @@
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')

data/test/nodule_cassandra_test.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require_relative 'helper'
+require 'minitest/autorun'
+require 'nodule/cassandra'
+
+class NoduleCassandraTest < MiniTest::Unit::TestCase
+  KEYSPACE = "NoduleTest"
+
+  def test_cassandra
+    cass = nil
+    assert (cass = Nodule::Cassandra.new({:keyspace => KEYSPACE, :verbose => true})),
+      "Can't create Nodule::Cassandra!"
+
+    cass.run
+    cass.create_keyspace
+
+    assert_nil cass.waitpid
+
+    assert_kind_of Cassandra, cass.client
+
+    cfdef = CassandraThrift::CfDef.new :name => "foo", :keyspace => KEYSPACE
+    refute_nil cass.client.add_column_family cfdef
+
+    assert File.directory?(File.join(cass.data, KEYSPACE))
+
+    cass.stop
+
+    assert File.directory?(cass.tmp) != true
+  end
+end

data/test/nodule_console_test.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/console'
+
+class NoduleConsoleTest < MiniTest::Unit::TestCase
+  def test_console
+  end
+end

data/test/nodule_lineio_test.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/line_io'
+
+class NoduleLineIOTest < MiniTest::Unit::TestCase
+  def setup
+      @r_pipe, @w_pipe = IO.pipe
+  end
+
+  def test_stdio
+    io = Nodule::LineIO.new :io => @r_pipe, :run => true, :reader => :capture
+
+    @w_pipe.puts "x"
+    io.require_read_count 1, 10
+    assert_equal "x", io.output.first.chomp, "read data from pipe"
+
+    assert_equal 1, io.read_count
+    @w_pipe.puts "y"
+    io.require_read_count 2, 10
+
+    assert_equal 2, io.read_count
+    io.clear!
+    assert_equal 0, io.read_count
+    @w_pipe.puts "y"
+
+    io.require_read_count 1, 10
+    assert_equal "y", io.output.first.chomp, "read data from pipe"
+  end
+end

data/test/nodule_process_test.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/process'
+
+class NoduleProcessTest < MiniTest::Unit::TestCase
+  def test_process
+    true_bin = File.exist?("/bin/true") ? "/bin/true" : "/usr/bin/true"
+    p = Nodule::Process.new true_bin
+    p.run
+    p.wait 2
+    assert p.done?, "true exits immediately, done? should be true"
+    p.stop
+
+    echo = Nodule::Process.new '/bin/echo', 'foobar', :run => true
+    echo.wait 2
+
+    assert_equal 'foobar', echo.output.first.chomp
+    assert echo.done?, "true exits immediately, done? should be true"
+
+    echo.stop
+  end
+end

data/test/nodule_tempfile_test.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require_relative 'helper'
+require 'minitest/autorun'
+require 'nodule/tempfile'
+
+class NoduleTempfileTest < MiniTest::Unit::TestCase
+  def test_basic
+    tfile = nil
+    assert (tfile = Nodule::Tempfile.new), "Can't create Nodule::Tempfile!"
+
+    assert_kind_of Nodule::Tempfile, tfile
+    assert_kind_of Nodule::Base, tfile
+
+    assert (tfile = Nodule::Tempfile.new(:directory => true)),
+      "Can't create Nodule::Tempfile on a directory!"
+    assert_kind_of Nodule::Tempfile, tfile
+    assert_kind_of Nodule::Base, tfile
+
+    tfile.stop
+  end
+end

data/test/nodule_topology_test.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/topology'
+
+class NoduleTopologyTest < MiniTest::Unit::TestCase
+  def test_topology
+  end
+end

data/test/nodule_unixsocket_test.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/unixsocket'
+
+class NoduleUnixsocketTest < MiniTest::Unit::TestCase
+  def test_unixsocket
+  end
+end

data/test/nodule_util_test.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require_relative 'helper'
+require 'minitest/autorun'
+require 'nodule/util'
+
+class NoduleUtilTest < MiniTest::Unit::TestCase
+  def test_random_tcp_port
+    port = nil
+    assert (port = Nodule::Util.random_tcp_port),
+      "Can't create Nodule::Util with random TCP port!"
+    assert_kind_of Fixnum, port
+    assert port > 1024
+    assert port < 65536
+  end
+
+  def test_random_udp_port
+    port = nil
+    assert (port = Nodule::Util.random_udp_port),
+      "Can't create Nodule::Util with random UDP port!"
+    assert_kind_of Fixnum, port
+    assert port > 1024
+    assert port < 65536
+  end
+end

data/test/nodule_zeromq_test.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'nodule/zeromq'
+
+class NoduleZeromqTest < MiniTest::Unit::TestCase
+  def test_zeromq_setup
+    t1 = Nodule::ZeroMQ.new(:uri => :gen)
+    refute_nil t1
+    t2 = Nodule::ZeroMQ.new(:uri => :gen, :bind => ZMQ::PUSH)
+    refute_nil t2
+    t3 = Nodule::ZeroMQ.new(:uri => t2.uri, :connect => ZMQ::PULL)
+    refute_nil t3
+
+    t1.stop
+    t2.stop
+    t3.stop
+  end
+
+  def test_zeromq_pubsub
+    pub = Nodule::ZeroMQ.new(:uri => :gen, :bind => ZMQ::PUB)
+    refute_nil pub
+    sub = Nodule::ZeroMQ.new(:uri => pub.uri, :connect => ZMQ::SUB, :reader => :capture)
+    refute_nil sub
+    pub.run
+    refute pub.done?
+    sub.run
+    refute sub.done?
+
+    pub.socket.send_string "Hello Verld!"
+    5.times do
+      if sub.output.count > 0
+        assert_equal "Hello Verld!", sub.output.flatten[0]
+        break
+      end
+      sleep 0.1
+    end
+
+    pub.stop
+    sub.stop
+  end
+end