nodule 0.0.34

data/lib/nodule/process.rb

@@ -0,0 +1,386 @@
+require 'nodule/version'
+require 'nodule/line_io'
+
+module Nodule
+  class ProcessNotRunningError < StandardError; end
+  class ProcessAlreadyRunningError < StandardError; end
+  class ProcessStillRunningError < StandardError; end
+  class TopologyUnknownSymbolError < StandardError; end
+
+  class Process < Base
+    attr_reader :argv, :pid, :started, :ended
+    attr_accessor :topology
+
+    # @param [Array] command, argv
+    # @param [Hash] opts
+    def initialize(*argv)
+      @opts = argv[-1].is_a?(Hash) ? argv.pop : {}
+      @env = argv[0].is_a?(Hash) ? argv.shift : {}
+      @status = nil
+      @started = -1   # give started and ended default values
+      @ended = -2
+      @pid = nil
+      @argv = argv
+      @stdout_opts = @opts.delete(:stdout) || :capture
+      @stderr_opts = @opts.delete(:stderr) || :capture
+
+      super(@opts)
+    end
+
+    # convert symbol arguments to the to_s result of a topology item if it exists,
+    # run procs, and flatten enumerbles, so
+    # :foobar will access the topology's entry for :foobar and call .to_s on it
+    # proc { "abc" } will become "abc"
+    # ['if=', :foobar] will resolve :foobar (this is recursive) and join all the results with no padding
+    # anything left unmatched will be coerced into a string with .to_s
+    def _apply_topology(arg)
+      # only symbols are auto-translated to resource strings, String keys intentionally do not match
+      if arg.kind_of? Symbol
+        if @topology.has_key? arg
+          @topology[arg].to_s
+        else
+          raise TopologyUnknownSymbolError.new "Unresolvable topology symbol, :#{arg}"
+        end
+      # sub-lists are recursed then joined with no padding, so:
+      # ["if=", :foo] would become "if=value"
+      elsif arg.respond_to? :map
+        new = arg.map { |a| _apply_topology(a) }
+        new.join('')
+      else
+        arg.to_s
+      end
+    end
+
+    def run
+      # raise exception only if the start time comes after the end time
+      if @started > @ended
+        raise ProcessAlreadyRunningError.new if @pid
+      end
+
+      argv = @argv.map { |arg| _apply_topology(arg) }
+
+      # Simply calling spawn with *argv isn't good enough, it really needs the command
+      # to be a completely separate argument. This is likely due to a bug in spawn().
+      command = argv.shift
+
+      verbose "Spawning: #{command} #{argv.join(' ')}"
+
+      @stdin_r, @stdin    = IO.pipe
+      @stdout,  @stdout_w = IO.pipe
+      @stderr,  @stderr_w = IO.pipe
+
+      @stdout_handler = Nodule::LineIO.new :io => @stdout, :reader => @stdout_opts, :topology => @topology, :run => true
+      @stderr_handler = Nodule::LineIO.new :io => @stderr, :reader => @stderr_opts, :topology => @topology, :run => true
+
+      @pid = spawn(@env, command, *argv,
+        :in  => @stdin_r,
+        :out => @stdout_w,
+        :err => @stderr_w,
+      )
+
+      @started = Time.now
+
+      @stdin_r.close
+      @stdout_w.close
+      @stderr_w.close
+
+      super
+    end
+
+    #
+    # Clear all of the state and prepare to be able to .run again.
+    # Raises ProcessStillRunningError if the child is still running.
+    #
+    def reset
+      raise ProcessStillRunningError.new unless done?
+      @stdout_handler.stop
+      @stderr_handler.stop
+      close
+      @pid = nil
+    end
+
+    def _kill(sig)
+      # Do not use negative signals. You will _always_ get ESRCH for child processes, since they are
+      # by definition not process group leaders, which is usually synonymous with the process group id
+      # that "kill -9 $PID" relies on.  See kill(2).
+      raise ArgumentError.new "negative signals are wrong and unsupported" unless sig > 0
+      raise ProcessNotRunningError.new unless @pid
+
+      verbose "Sending signal #{sig} to process #{@pid}."
+      ::Process.kill(sig, @pid)
+      # do not catch ESRCH - ESRCH means we did something totally buggy, likewise, an exception
+      # should fire if the process is not running since there's all kinds of code already checking
+      # that it is running before getting this far.
+    end
+
+    #
+    # Call Process.waitpid2, save the status (accessible with obj.status) and return just the pid value
+    # returned by waitpid2.
+    #
+    def waitpid(flag=::Process::WNOHANG)
+      raise ProcessNotRunningError.new "pid is not known" unless @pid
+      raise ProcessNotRunningError.new "process seems to have exited #{@status.inspect}" if @status
+
+      pid, @status = ::Process.waitpid2(@pid, flag)
+
+      # this is as accurate as we can get, and it will generally be good enough for test work
+      @ended = Time.now if pid == @pid
+
+      pid
+    end
+
+    #
+    # Call waitpid and block until the process exits or timeout is reached.
+    #
+    alias :iowait :wait
+    def wait(timeout=nil)
+      pid = nil # silence warning
+
+      # block indefinitely on nil/0 timeout
+      unless timeout
+        return waitpid(0)
+      end
+
+      wait_with_backoff timeout do
+        if @status
+          true
+        else
+          pid = waitpid(::Process::WNOHANG)
+          done?
+        end
+      end
+
+      pid
+    end
+
+    #
+    # Send SIGTERM (15) to the child process, sleep 1/25 of a second, then call waitpid. For well-behaving
+    # processes, this should be enough to make it stop.
+    # Returns true/false just like done?
+    #
+    def stop
+      return if done?
+      _kill 15 # never negative!
+      @stdout_handler.stop
+      @stderr_handler.stop
+      sleep 0.05
+      @pid == waitpid
+      close
+    end
+
+    #
+    # Send SIGKILL (9) to the child process, sleep 1/10 of a second, then call waitpid and return.
+    # Returns true/false just like done?
+    #
+    def stop!
+      raise ProcessNotRunningError.new unless @pid
+      return if done?
+
+      _kill 9 # never negative!
+      @stdout_handler.stop!
+      @stderr_handler.stop!
+      sleep 0.1
+      @pid == waitpid
+      close
+    end
+
+    #
+    # Return Process::Status as returned by Process::waitpid2.
+    #
+    def status
+      raise ProcessNotRunningError.new "#@prefix called .status before .run." unless @pid
+      waitpid unless @status
+      @status
+    end
+
+    #
+    # Check whether the process has exited or been killed and cleaned up.
+    # Calls waitpid2 behind the scenes if necessary.
+    # Throws ProcessNotRunningError if called before .run.
+    #
+    alias :iodone? :done?
+    def done?
+      raise ProcessNotRunningError.new "#@prefix called .done? before .run." unless @pid
+      waitpid unless @status
+      return true if @status
+      waitpid == @pid
+    end
+
+    #
+    # Return the elapsed time in milliseconds.
+    #
+    def elapsed
+      raise ProcessNotRunningError.new unless @started
+      raise ProcessStillRunningError.new unless @ended
+      @ended - @started
+    end
+
+    #
+    # Returns whether or not any stdout has been captured.
+    # Will raise an exception if capture is not enabled.
+    # proxies: Nodule::Base.output?
+    # @return [TrueClass,FalseClass]
+    #
+    def stdout?
+      @stdout_handler.output?
+    end
+    alias :output? :stdout?
+
+    #
+    # Get all currently captured stdout. Does not clear the buffer.
+    # proxies: Nodule::Base.output
+    # @return [Array{String}]
+    #
+    def stdout
+      @stdout_handler.output
+    end
+    alias :output :stdout
+
+    #
+    # Get all currently captured stdout. Resets the buffer and counts.
+    # proxies: Nodule::Base.output!
+    # @return [Array{String}]
+    #
+    def stdout!
+      @stdout_handler.output!
+    end
+    alias :output! :stdout!
+
+    #
+    # Clear the stdout buffer and reset the counter.
+    # proxies: Nodule::Base.clear!
+    #
+    def clear_stdout!
+      @stdout_handler.clear!
+    end
+    alias :clear! :clear_stdout!
+
+    #
+    # Proxies to stdout require_read_count.
+    #
+    def require_stdout_count(count, max_sleep=10)
+      @stdout_handler.require_read_count count, max_sleep
+    end
+    alias :require_read_count :require_stdout_count
+
+    #
+    # Returns whether or not any stderr has been captured.
+    # Will raise an exception if capture is not enabled.
+    # proxies: Nodule::Base.output?
+    # @return [TrueClass,FalseClass]
+    #
+    def stderr?
+      @stderr_handler.output?
+    end
+
+    #
+    # Get all currently captured stderr. Does not clear the buffer.
+    # proxies: Nodule::Base.output
+    # @return [Array{String}]
+    #
+    def stderr
+      @stderr_handler.output
+    end
+
+    #
+    # Get all currently captured stderr. Resets the buffer and counts.
+    # proxies: Nodule::Base.output!
+    # @return [Array{String}]
+    #
+    def stderr!
+      @stderr_handler.output!
+    end
+
+    #
+    # Clear the stderr buffer and reset the counter.
+    # proxies: Nodule::Base.clear!
+    #
+    def clear_stderr!
+      @stderr_handler.clear!
+    end
+
+    #
+    # Proxies to stderr require_read_count.
+    #
+    def require_stderr_count(count, max_sleep=10)
+      @stderr_handler.require_read_count count, max_sleep
+    end
+
+    #
+    # Write the to child process's stdin using IO.print.
+    # @param [String] see IO.print
+    #
+    def print(*args)
+      @stdin.print(*args)
+    end
+
+    #
+    # Write the to child process's stdin using IO.puts.
+    # @param [String] see IO.puts
+    #
+    def puts(*args)
+      @stdin.puts(*args)
+    end
+
+    #
+    # Access the STDIN pipe IO object of the handle.
+    # @return [IO]
+    #
+    def stdin_pipe
+      @stdin
+    end
+
+    #
+    # Access the STDOUT pipe IO object of the handle.
+    # @return [IO]
+    #
+    def stdout_pipe
+      @stdout
+    end
+
+    #
+    # Access the STDERR pipe IO object of the handle.
+    # @return [IO]
+    #
+    def stderr_pipe
+      @stderr
+    end
+
+    #
+    # Close all of the pipes.
+    #
+    def close
+      @stdin.close rescue nil
+      @stdout.close rescue nil
+      @stderr.close rescue nil
+    end
+
+    #
+    # Return most of the data about the process as a hash. This is safe to call at any point.
+    #
+    def to_hash
+      {
+        :argv    => @argv,
+        :started => @started.to_i,
+        :ended   => @ended.to_i,
+        :elapsed => elapsed,
+        :pid     => @pid,
+        :retval  => ((@status.nil? and @status.exited?) ? nil : @status.exitstatus)
+     }
+    end
+
+    #
+    # Returns the command as a string.
+    #
+    def to_s
+      @argv.join(' ')
+    end
+
+    #
+    # Returns to_hash.inspect
+    #
+    def inspect
+      to_hash.inspect
+    end
+  end
+end

data/lib/nodule/tempfile.rb

@@ -0,0 +1,57 @@
+require 'nodule/base'
+require 'fileutils'
+
+module Nodule
+  class Tempfile < Base
+    attr_reader :file
+
+    def initialize(opts={})
+      suffix = opts[:suffix] || ''
+      prefix = opts[:prefix] || 'nodule'
+      @file = "#{prefix}-#{::Process.pid}-#{Nodule.next_seq}#{suffix}"
+
+      if opts[:directory]
+        @is_dir = true
+        if opts[:directory].kind_of? String
+          FileUtils.mkdir_p File.join(opts[:directory], @file)
+        else
+          FileUtils.mkdir @file
+        end
+      else
+        @is_dir = false
+        # require an explicit request to create an empty file
+        if opts[:touch]
+          File.open @file, "w" do |f| f.puts "" end
+        end
+      end
+
+      @cleanup = opts.has_key?(:cleanup) ? opts[:cleanup] : true
+
+      super(opts)
+    end
+
+    def touch(target=nil)
+      File.open(@file, "w+").close
+      @file
+    end
+
+    def stop
+      if @cleanup
+        # Ruby caches stat_t somewhere and causes race conditions, but we don't really
+        # care here as long as the file is gone.
+        begin
+          FileUtils.rm_r(@file) if @is_dir
+          File.unlink(@file)
+        rescue Errno::ENOENT
+        end
+      end
+
+      super
+    end
+
+    def to_s
+      @file
+    end
+  end
+end
+

data/lib/nodule/topology.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/env ruby
+
+# To test, you'll be creating a Topology, representing a cluster of
+# interconnected processes.  You'll also optionally declare a number
+# of resources for the test framework to verify - files it can read,
+# network connections it can snoop or spoof and so on.  By declaring
+# these resources, you gain the ability to make assertions against
+# them.
+
+# After creating the Topology and adding processes to it, you run it.
+# When you do, the framework will allocate resources and rework the
+# command line of every node to use the resources that the framework
+# has allocated, faked or mocked.  For instance, for a ZeroMQ socket
+# the framework will create an identical forwarding socket that
+# records traffic before resending to the application's actual socket.
+
+# Since the test framework doesn't know the command line of every
+# possible executable, you'll need to write your command lines in
+# terms of those resources.  Erb is used to let you do logic in the
+# command-line declarations, and variables are passed in for the
+# resources that the test framework has created.
+
+#
+# Module to help build a topology on a single machine. All pieces of the topology
+# that run in subprocesses will be referenceable through this wrapper.
+#
+module Nodule
+  class TopologyProcessStillRunningError < StandardError; end
+  class TopologyIntegrationRequiredError < StandardError; end
+
+  class Topology
+    def initialize(opts={})
+      @resources = {}
+      @started = {}
+
+      opts.each do |name,value|
+        inject_topology(name, value)
+        @resources[name] = value
+      end
+
+      @all_stopped = true
+    end
+
+    def inject_topology(name, value)
+      unless value.respond_to? :join_topology!
+        raise TopologyIntegrationRequiredError.new "#{name} => #{value} does not respond to :join_topology!"
+      end
+      value.join_topology! self, name
+    end
+
+    def [](key)
+      @resources[key]
+    end
+
+    def []=(key, value)
+      inject_topology(key, value)
+      @resources[key] = value
+    end
+
+    def has_key?(key)
+      @resources.has_key?(key)
+    end
+
+    def keys
+      @resources.keys
+    end
+
+    def key(object)
+      @resources.key(object)
+    end
+
+    def to_hash
+      @resources
+    end
+
+    def start_all
+      @resources.keys.each do |key|
+        start key unless @started[key]
+      end
+
+      # If we do many cycles, this will wind up getting called repeatedly.
+      # The @all_stopped variable will make sure that's a really fast
+      # operation.
+      at_exit { stop_all }
+    end
+
+    #
+    # Run each process in order, waiting for each one to complete & return before
+    # running the next.
+    #
+    # Resources are all started up at once.
+    #
+    def run_serially
+      @all_stopped = false
+
+      @resources.each do |name,object|
+        object.run
+        if object.respond_to? :wait
+          object.wait
+        else
+          object.stop
+        end
+      end
+
+      @all_stopped = true
+    end
+
+    #
+    # Starts the node in the topology. Looks up the node's command
+    # given that the topology hash is keyed off of the node's name.
+    #
+    def start(*names)
+      @all_stopped = false
+      names.flatten.each do |name|
+        # run the command that starts up the node and store the subprocess for later manipulation
+        @resources[name].run unless @started[name]
+
+        @started[name] = true
+      end
+    end
+
+    #
+    # Immediately kills a node given its topology name
+    #
+    def stop(*names)
+      names.flatten.each do |name|
+        object = @resources[name]
+        object.stop
+        object.wait 1 unless object.done?
+        object.stop!  unless object.done?
+        object.wait 1 unless object.done?
+        unless object.done?
+          raise "Could not stop resource: #{object.class} #{object.inspect}"
+        end
+
+        @started[name] = false
+      end
+    end
+
+    #
+    # Kills all of the nodes in the topology.
+    #
+    def stop_all
+      @resources.each { |name,object| stop name unless object.done? } unless @all_stopped
+    end
+
+    def started?(key)
+      @started[key.to_sym] == true
+    end
+
+    def start_all_but(*resources)
+      @resources.keys.each do |key|
+        if !@started[key] && !resources.flatten.map(&:to_sym).include?(key)
+          start key
+        end
+      end
+
+      at_exit { stop_all_but resources }
+    end
+
+    def stop_all_but(*resources)
+      @resources.each do |name,object|
+        if !resources.flatten.map(&:to_sym).include?(name.to_sym) && !object.done?
+          stop name
+        end
+      end unless @all_stopped
+    end
+
+    def cleanup
+      @resources.each { |_,object| object.stop }
+    end
+
+    def wait(name, timeout=60)
+      @resources[name].wait timeout
+    end
+
+    #
+    # Wait for all resources to exit normally.
+    #
+    def wait_all
+      @resources.each do |name,object|
+        object.wait if object.respond_to? :wait
+      end
+    end
+
+    #
+    # Reset all processes for restart.
+    #
+    def reset_all
+      raise TopologyProcessStillRunningError.new unless @all_stopped
+      @resources.each { |_, object| object.reset }
+    end
+
+  end
+end

data/lib/nodule/unixsocket.rb

@@ -0,0 +1,54 @@
+require 'socket'
+require 'nodule/tempfile'
+
+module Nodule
+  class UnixSocket < Tempfile
+    attr_reader :family, :address, :connected
+
+    def initialize(opts={})
+      super(opts)
+      @family = opts[:family] || :DGRAM
+      @socket = Socket.new(:UNIX, @family, 0)
+      @address = Addrinfo.unix(@file)
+      @connected = false
+    end
+
+    #
+    # sock1 = Nodule::UnixSocket.new
+    #
+    def send(data)
+      @socket.connect(@address) unless @connected
+      @connected = true
+
+      if @family == :DGRAM
+        @socket.sendmsg(data, 0)
+      else
+        @socket.send(data, 0)
+      end
+    end
+
+    def stop
+      @socket.close
+      super
+    end
+  end
+
+  class UnixServer < Tempfile
+    def run
+      super
+      @thread = Thread.new do
+        Thread.current.abort_on_exception
+
+        server = Socket.new(:UNIX, @family, 0)
+        address = Addrinfo.unix(@file)
+        server.bind(address)
+
+        message, = server.recvmsg(65536, 0) if sock
+      end
+    end
+
+    def to_s
+      @sockfile
+    end
+  end
+end