sskirby-hydra 0.16.9

data/lib/hydra/runner.rb

@@ -0,0 +1,214 @@
+require 'test/unit'
+require 'test/unit/testresult'
+Test::Unit.run = true
+
+module Hydra #:nodoc:
+  # Hydra class responsible for running test files.
+  #
+  # The Runner is never run directly by a user. Runners are created by a
+  # Worker to run test files.
+  #
+  # The general convention is to have one Runner for each logical processor
+  # of a machine.
+  class Runner
+    include Hydra::Messages::Runner
+    traceable('RUNNER')
+    # Boot up a runner. It takes an IO object (generally a pipe from its
+    # parent) to send it messages on which files to execute.
+    def initialize(opts = {})
+      @io = opts.fetch(:io) { raise "No IO Object" } 
+      @verbose = opts.fetch(:verbose) { false }      
+      $stdout.sync = true
+      trace 'Booted. Sending Request for file'
+
+      @io.write RequestFile.new
+      begin
+        process_messages
+      rescue => ex
+        trace ex.to_s
+        raise ex
+      end
+    end
+
+    # Run a test file and report the results
+    def run_file(file)
+      trace "Running file: #{file}"
+
+      output = ""
+      if file =~ /_spec.rb$/
+        output = run_rspec_file(file)
+      elsif file =~ /.feature$/
+        output = run_cucumber_file(file)
+      else
+        output = run_test_unit_file(file)
+      end
+
+      output = "." if output == ""
+
+      @io.write Results.new(:output => output, :file => file)
+      return output
+    end
+
+    # Stop running
+    def stop
+      @running = false
+    end
+
+    private
+
+    # The runner will continually read messages and handle them.
+    def process_messages
+      trace "Processing Messages"
+      @running = true
+      while @running
+        begin
+          message = @io.gets
+          if message and !message.class.to_s.index("Worker").nil?
+            trace "Received message from worker"
+            trace "\t#{message.inspect}"
+            message.handle(self)
+          else
+            @io.write Ping.new
+          end
+        rescue IOError => ex
+          trace "Runner lost Worker"
+          @running = false
+        end
+      end
+    end
+
+    # Run all the Test::Unit Suites in a ruby file
+    def run_test_unit_file(file)
+      begin
+        require file
+      rescue LoadError => ex
+        trace "#{file} does not exist [#{ex.to_s}]"
+        return ex.to_s
+      end
+      output = []
+      @result = Test::Unit::TestResult.new
+      @result.add_listener(Test::Unit::TestResult::FAULT) do |value|
+        output << value
+      end
+
+      klasses = Runner.find_classes_in_file(file)
+      begin
+        klasses.each{|klass| klass.suite.run(@result){|status, name| ;}}
+      rescue => ex
+        output << ex.to_s
+      end
+
+      return output.join("\n")
+    end
+
+    # run all the Specs in an RSpec file (NOT IMPLEMENTED)
+    def run_rspec_file(file)
+      # pull in rspec
+      begin
+        require 'spec'
+        require 'hydra/spec/hydra_formatter'
+        # Ensure we override rspec's at_exit
+        require 'hydra/spec/autorun_override'
+      rescue LoadError => ex
+        return ex.to_s
+      end
+      hydra_output = StringIO.new
+      Spec::Runner.options.instance_variable_set(:@formatters, [
+        Spec::Runner::Formatter::HydraFormatter.new(
+          Spec::Runner.options.formatter_options,
+          hydra_output
+        )
+      ])
+      Spec::Runner.options.instance_variable_set(
+        :@example_groups, []
+      )
+      Spec::Runner.options.instance_variable_set(
+        :@files, [file]
+      )
+      Spec::Runner.options.instance_variable_set(
+        :@files_loaded, false
+      )
+      Spec::Runner.options.run_examples
+      hydra_output.rewind
+      output = hydra_output.read.chomp
+      output = "" if output.gsub("\n","") =~ /^\.*$/
+
+      return output
+    end
+
+    # run all the scenarios in a cucumber feature file
+    def run_cucumber_file(file)
+
+      files = [file]
+      dev_null = StringIO.new
+      hydra_response = StringIO.new
+
+      unless @step_mother
+        require 'cucumber'
+        require 'hydra/cucumber/formatter'
+        @step_mother = Cucumber::StepMother.new
+        @cuke_configuration = Cucumber::Cli::Configuration.new(dev_null, dev_null)
+        @cuke_configuration.parse!(['features']+files)
+
+        @step_mother.options = @cuke_configuration.options
+        @step_mother.log = @cuke_configuration.log
+        @step_mother.load_code_files(@cuke_configuration.support_to_load)
+        @step_mother.after_configuration(@cuke_configuration)
+        @step_mother.load_code_files(@cuke_configuration.step_defs_to_load)
+      end
+      cuke_formatter = Cucumber::Formatter::Hydra.new(
+        @step_mother, hydra_response, @cuke_configuration.options
+      )
+
+      cuke_runner ||= Cucumber::Ast::TreeWalker.new(
+        @step_mother, [cuke_formatter], @cuke_configuration.options, dev_null
+      )
+      @step_mother.visitor = cuke_runner
+
+      features = @step_mother.load_plain_text_features(files)
+      tag_excess = tag_excess(features, @cuke_configuration.options[:tag_expression].limits)
+      @cuke_configuration.options[:tag_excess] = tag_excess
+
+      cuke_runner.visit_features(features)
+
+      hydra_response.rewind
+      return hydra_response.read
+    end
+
+    # find all the test unit classes in a given file, so we can run their suites
+    def self.find_classes_in_file(f)
+      code = ""
+      File.open(f) {|buffer| code = buffer.read}
+      matches = code.scan(/class\s+([\S]+)/)
+      klasses = matches.collect do |c|
+        begin
+          if c.first.respond_to? :constantize
+            c.first.constantize
+          else
+            eval(c.first)
+          end
+        rescue NameError
+          # means we could not load [c.first], but thats ok, its just not
+          # one of the classes we want to test
+          nil
+        rescue SyntaxError
+          # see above
+          nil
+        end
+      end
+      return klasses.select{|k| k.respond_to? 'suite'}
+    end
+
+    # Yanked a method from Cucumber
+    def tag_excess(features, limits)
+      limits.map do |tag_name, tag_limit|
+        tag_locations = features.tag_locations(tag_name)
+        if tag_limit && (tag_locations.length > tag_limit)
+          [tag_name, tag_limit, tag_locations]
+        else
+          nil
+        end
+      end.compact
+    end
+  end
+end

data/lib/hydra/safe_fork.rb

@@ -0,0 +1,31 @@
+class SafeFork
+  def self.fork
+    begin
+      # remove our connection so it doesn't get cloned
+      connection = ActiveRecord::Base.remove_connection if defined?(ActiveRecord)
+      # fork a process
+      child = Process.fork do
+        begin
+          # create a new connection and perform the action
+          begin
+          ActiveRecord::Base.establish_connection((connection || {}).merge({:allow_concurrency => true})) if defined?(ActiveRecord)
+          rescue ActiveRecord::AdapterNotSpecified
+            # AR was defined but we didn't have a connection
+          end
+          yield
+        ensure
+          # make sure we remove the connection before we're done
+          ActiveRecord::Base.remove_connection if defined?(ActiveRecord)
+        end      
+      end
+    ensure
+      # make sure we re-establish the connection before returning to the main instance
+      begin
+        ActiveRecord::Base.establish_connection((connection || {}).merge({:allow_concurrency => true})) if defined?(ActiveRecord)
+      rescue ActiveRecord::AdapterNotSpecified
+        # AR was defined but we didn't have a connection
+      end
+    end
+    return child
+  end
+end

data/lib/hydra/spec/autorun_override.rb

@@ -0,0 +1,12 @@
+if defined?(Spec)
+  module Spec
+    module Runner
+      class << self
+        # stop the auto-run at_exit
+        def run
+          return 0
+        end 
+      end
+    end
+  end
+end

data/lib/hydra/spec/hydra_formatter.rb

@@ -0,0 +1,17 @@
+require 'spec/runner/formatter/progress_bar_formatter'
+module Spec
+  module Runner
+    module Formatter
+      class HydraFormatter < ProgressBarFormatter
+        # Stifle the post-test summary
+        def dump_summary(duration, example, failure, pending)
+        end
+
+        # Stifle the output of pending examples
+        def example_pending(*args)
+        end
+      end
+    end
+  end
+end
+

data/lib/hydra/ssh.rb

@@ -0,0 +1,40 @@
+require 'open3'
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write with an ssh connection. For example:
+  #   @ssh = Hydra::SSH.new(
+  #     'localhost', # connect to this machine
+  #     '/home/user', # move to the home directory
+  #     "ruby hydra/test/echo_the_dolphin.rb" # run the echo script
+  #   )
+  #   @message = Hydra::Messages::TestMessage.new("Hey there!")
+  #   @ssh.write @message
+  #   puts @ssh.gets.text
+  #     => "Hey there!"
+  #
+  # Note that what ever process you run should respond with Hydra messages.
+  class SSH
+    include Open3
+    include Hydra::MessagingIO
+
+    # Initialize new SSH connection.
+    # The first parameter is passed directly to ssh for starting a connection.
+    # The second parameter is the directory to CD into once connected.
+    # The third parameter is the command to run
+    # So you can do:
+    #   Hydra::SSH.new('-p 3022 user@server.com', '/home/user/Desktop', 'ls -l')
+    # To connect to server.com as user on port 3022, then CD to their desktop, then
+    # list all the files.
+    def initialize(connection_options, directory, command)
+      @writer, @reader, @error = popen3("ssh -tt #{connection_options}")
+      @writer.write("cd #{directory}\n")
+      @writer.write(command+"\n")
+    end
+
+    # Close the SSH connection
+    def close
+      @writer.write "exit\n"
+      super
+    end
+  end
+end

data/lib/hydra/stdio.rb

@@ -0,0 +1,16 @@
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write via stdout and stdin.
+  class Stdio
+    include Hydra::MessagingIO
+
+    # Initialize new Stdio
+    def initialize()
+      @reader = $stdin
+      @writer = $stdout
+      @reader.sync = true
+      @writer.sync = true
+    end
+  end
+end
+

data/lib/hydra/sync.rb

@@ -0,0 +1,99 @@
+require 'yaml'
+module Hydra #:nodoc:
+  # Hydra class responsible for delegate work down to workers.
+  #
+  # The Sync is run once for each remote worker.
+  class Sync
+    traceable('SYNC')
+    self.class.traceable('SYNC MANY')
+
+    attr_reader :connect, :ssh_opts, :remote_dir
+
+    # Create a new Sync instance to rsync source from the local machine to a remote worker
+    #
+    # Arguments:
+    # * :worker_opts
+    #   * A hash of the configuration options for a worker.
+    # * :sync
+    #   * A hash of settings specifically for copying the source directory to be tested
+    #     to the remote worked
+    # * :verbose
+    #   * Set to true to see lots of Hydra output (for debugging)
+    def initialize(worker_opts, sync_opts, verbose = false)
+      worker_opts ||= {}
+      worker_opts.stringify_keys!
+      @verbose = verbose
+      @connect = worker_opts.fetch('connect') { raise "You must specify an SSH connection target" }
+      @ssh_opts = worker_opts.fetch('ssh_opts') { "" }
+      @remote_dir = worker_opts.fetch('directory') { raise "You must specify a remote directory" }
+
+      return unless sync_opts
+      sync_opts.stringify_keys!
+      @local_dir = sync_opts.fetch('directory') { raise "You must specify a synchronization directory" }
+      @exclude_paths = sync_opts.fetch('exclude') { [] }
+
+      trace "Initialized"
+      trace "  Worker: (#{worker_opts.inspect})"
+      trace "  Sync:   (#{sync_opts.inspect})"
+
+      sync
+    end
+
+    def sync
+      #trace "Synchronizing with #{connect}\n\t#{sync_opts.inspect}"
+      exclude_opts = @exclude_paths.inject(''){|memo, path| memo += "--exclude=#{path} "}
+
+      rsync_command = [
+        'rsync',
+        '-avz',
+        '--delete',
+        exclude_opts,
+        File.expand_path(@local_dir)+'/',
+        "-e \"ssh #{@ssh_opts}\"",
+        "#{@connect}:#{@remote_dir}"
+      ].join(" ")
+      trace rsync_command
+      trace `#{rsync_command}`
+    end
+
+    def self.sync_many opts
+      opts.stringify_keys!
+      config_file = opts.delete('config') { nil }
+      if config_file
+        opts.merge!(YAML.load_file(config_file).stringify_keys!)
+      end
+      @verbose = opts.fetch('verbose') { false }
+      @sync = opts.fetch('sync') { {} }
+
+      workers_opts = opts.fetch('workers') { [] }
+      @remote_worker_opts = []
+      workers_opts.each do |worker_opts|
+        worker_opts.stringify_keys!
+        if worker_opts['type'].to_s == 'ssh'
+          @remote_worker_opts << worker_opts
+        end
+      end
+
+      trace "Initialized"
+      trace "  Sync:   (#{@sync.inspect})"
+      trace "  Workers: (#{@remote_worker_opts.inspect})"
+
+      Thread.abort_on_exception = true
+      trace "Processing workers"
+      @listeners = []
+      @remote_worker_opts.each do |worker_opts|
+        @listeners << Thread.new do
+          begin
+            trace "Syncing #{worker_opts.inspect}"
+            Sync.new worker_opts, @sync, @verbose
+          rescue 
+            trace "Syncing failed [#{worker_opts.inspect}]"
+          end
+        end
+      end
+      
+      @listeners.each{|l| l.join}
+    end
+
+  end
+end

data/lib/hydra/tasks.rb

@@ -0,0 +1,256 @@
+require 'open3'
+module Hydra #:nodoc:
+  # Hydra Task Common attributes and methods
+  class Task
+    # Name of the task. Default 'hydra'
+    attr_accessor :name
+
+    # Files to test.
+    # You can add files manually via:
+    #   t.files << [file1, file2, etc]
+    #
+    # Or you can use the add_files method
+    attr_accessor :files
+
+    # True if you want to see Hydra's message traces
+    attr_accessor :verbose
+
+    # Path to the hydra config file.
+    # If not set, it will check 'hydra.yml' and 'config/hydra.yml'
+    attr_accessor :config
+
+    # Automatically sort files using their historical runtimes.
+    # Defaults to true
+    # To disable:
+    #   t.autosort = false
+    attr_accessor :autosort
+
+    # Event listeners. Defaults to the MinimalOutput listener.
+    # You can add additional listeners if you'd like. For example,
+    # on linux (with notify-send) you can add the notifier listener:
+    #   t.listeners << Hydra::Listener::Notifier.new
+    attr_accessor :listeners
+
+    #
+    # Search for the hydra config file
+    def find_config_file
+      @config ||= 'hydra.yml'
+      return @config if File.exists?(@config)
+      @config = File.join('config', 'hydra.yml')
+      return @config if File.exists?(@config)
+      @config = nil
+    end
+    
+    # Add files to test by passing in a string to be run through Dir.glob.
+    # For example:
+    #
+    #   t.add_files 'test/units/*.rb'
+    def add_files(pattern)
+      @files += Dir.glob(pattern)
+    end
+
+  end
+
+  # Define a test task that uses hydra to test the files.
+  #
+  #   Hydra::TestTask.new('hydra') do |t|
+  #     t.add_files 'test/unit/**/*_test.rb'
+  #     t.add_files 'test/functional/**/*_test.rb'
+  #     t.add_files 'test/integration/**/*_test.rb'
+  #     t.verbose = false # optionally set to true for lots of debug messages
+  #     t.autosort = false # disable automatic sorting based on runtime of tests
+  #   end  
+  class TestTask < Hydra::Task
+
+    # Create a new HydraTestTask
+    def initialize(name = :hydra)
+      @name = name
+      @files = []
+      @verbose = false
+      @autosort = true
+      @listeners = [Hydra::Listener::ProgressBar.new]
+
+      yield self if block_given?
+
+      # Ensure we override rspec's at_exit
+      require 'hydra/spec/autorun_override'
+
+      @config = find_config_file
+
+      @opts = {
+        :verbose => @verbose,
+        :autosort => @autosort,
+        :files => @files,
+        :listeners => @listeners
+      }
+      if @config
+        @opts.merge!(:config => @config)
+      else
+        @opts.merge!(:workers => [{:type => :local, :runners => 1}])
+      end
+
+      define
+    end
+
+    private
+    # Create the rake task defined by this HydraTestTask
+    def define
+      desc "Hydra Tests" + (@name == :hydra ? "" : " for #{@name}")
+      task @name do
+        Hydra::Master.new(@opts)
+        #exit(0) #bypass test on_exit output
+      end
+    end
+  end
+
+  # Define a sync task that uses hydra to rsync the source tree under test to remote workers.
+  #
+  # This task is very useful to run before a remote db:reset task to make sure the db/schema.rb
+  # file is up to date on the remote workers.
+  #
+  #   Hydra::SyncTask.new('hydra:sync') do |t|
+  #     t.verbose = false # optionally set to true for lots of debug messages
+  #   end  
+  class SyncTask < Hydra::Task
+
+    # Create a new SyncTestTask
+    def initialize(name = :sync)
+      @name = name
+      @verbose = false
+
+      yield self if block_given?
+
+      @config = find_config_file
+
+      @opts = {
+        :verbose => @verbose
+      }
+      @opts.merge!(:config => @config) if @config
+
+      define
+    end
+
+    private
+    # Create the rake task defined by this HydraSyncTask
+    def define
+      desc "Hydra Tests" + (@name == :hydra ? "" : " for #{@name}")
+      task @name do
+        Hydra::Sync.sync_many(@opts)
+      end
+    end
+  end
+
+  # Setup a task that will be run across all remote workers
+  #   Hydra::RemoteTask.new('db:reset')
+  #
+  # Then you can run:
+  #   rake hydra:remote:db:reset
+  class RemoteTask < Hydra::Task
+    include Open3
+    # Create a new hydra remote task with the given name.
+    # The task will be named hydra:remote:<name>
+    def initialize(name)
+      @name = name
+      yield self if block_given?
+      @config = find_config_file
+      if @config
+        define
+      else
+        task "hydra:remote:#{@name}" do ; end
+      end
+    end
+
+    private
+    def define
+      desc "Run #{@name} remotely on all workers"
+      task "hydra:remote:#{@name}" do
+        config = YAML.load_file(@config)
+        environment = config.fetch('environment') { 'test' }
+        workers = config.fetch('workers') { [] }
+        workers = workers.select{|w| w['type'] == 'ssh'}
+
+        $stdout.write "==== Hydra Running #{@name} ====\n"
+        Thread.abort_on_exception = true
+        @listeners = []
+        @results = {}
+        workers.each do |worker|
+          @listeners << Thread.new do
+            begin
+              @results[worker] = if run_task(worker, environment)
+                "==== #{@name} passed on #{worker['connect']} ====\n"
+              else
+                "==== #{@name} failed on #{worker['connect']} ====\nPlease see above for more details.\n"
+              end
+            rescue 
+              @results[worker] = "==== #{@name} failed for #{worker['connect']} ====\n#{$!.inspect}\n#{$!.backtrace.join("\n")}"
+            end
+          end
+        end
+        @listeners.each{|l| l.join}
+        $stdout.write "\n==== Hydra Running #{@name} COMPLETE ====\n\n"
+        $stdout.write @results.values.join("\n")
+      end
+    end
+
+    def run_task worker, environment
+      $stdout.write "==== Hydra Running #{@name} on #{worker['connect']} ====\n"
+      ssh_opts = worker.fetch('ssh_opts') { '' }
+      writer, reader, error = popen3("ssh -tt #{ssh_opts} #{worker['connect']} ")
+      writer.write("cd #{worker['directory']}\n")
+      writer.write "echo BEGIN HYDRA\n"
+      writer.write("RAILS_ENV=#{environment} rake #{@name}\n")
+      writer.write "echo END HYDRA\n"
+      writer.write("exit\n")
+      writer.close
+      ignoring = true
+      passed = true
+      while line = reader.gets
+        line.chomp!
+        if line =~ /^rake aborted!$/
+          passed = false
+        end
+        if line =~ /echo END HYDRA$/
+          ignoring = true
+        end
+        $stdout.write "#{worker['connect']}: #{line}\n" unless ignoring
+        if line == 'BEGIN HYDRA'
+          ignoring = false
+        end
+      end
+      passed
+    end
+  end
+
+  # A Hydra global task is a task that is run both locally and remotely.
+  #
+  # For example:
+  #
+  #   Hydra::GlobalTask.new('db:reset')
+  #
+  # Allows you to run:
+  #   
+  #   rake hydra:db:reset
+  #
+  # Then, db:reset will be run locally and on all remote workers. This
+  # makes it easy to setup your workers and run tasks all in a row.
+  #
+  # For example:
+  #
+  #   rake hydra:db:reset hydra:factories hydra:tests
+  #
+  # Assuming you setup hydra:db:reset and hydra:db:factories as global
+  # tasks and hydra:tests as a Hydra::TestTask for all your tests
+  class GlobalTask < Hydra::Task
+    def initialize(name)
+      @name = name
+      define
+    end
+
+    private
+    def define
+      Hydra::RemoteTask.new(@name)
+      desc "Run #{@name.to_s} Locally and Remotely across all Workers"
+      task "hydra:#{@name.to_s}" => [@name.to_s, "hydra:remote:#{@name.to_s}"] 
+    end
+  end
+end