ngauthier-hydra 0.24.0

data/lib/hydra/runner_listener/abstract.rb

@@ -0,0 +1,23 @@
+module Hydra #:nodoc:
+  module RunnerListener #:nodoc:
+    # Abstract listener that implements all the events
+    # but does nothing.
+    class Abstract
+      # Create a new listener.
+      #
+      # Output: The IO object for outputting any information.
+      # Defaults to STDOUT, but you could pass a file in, or STDERR
+      def initialize(output = $stdout)
+        @output = output
+      end
+
+      # Fired by the runner just before requesting the first file
+      def runner_begin( runner )
+      end
+
+      # Fired by the runner just after stoping
+      def runner_end( runner )
+      end
+    end
+  end
+end

data/lib/hydra/safe_fork.rb

@@ -0,0 +1,31 @@
+class SafeFork
+  def self.fork
+    begin
+      # remove our connection so it doesn't get cloned
+      connection = ActiveRecord::Base.remove_connection if defined?(ActiveRecord)
+      # fork a process
+      child = Process.fork do
+        begin
+          # create a new connection and perform the action
+          begin
+          ActiveRecord::Base.establish_connection((connection || {}).merge({:allow_concurrency => true})) if defined?(ActiveRecord)
+          rescue ActiveRecord::AdapterNotSpecified
+            # AR was defined but we didn't have a connection
+          end
+          yield
+        ensure
+          # make sure we remove the connection before we're done
+          ActiveRecord::Base.remove_connection if defined?(ActiveRecord)
+        end      
+      end
+    ensure
+      # make sure we re-establish the connection before returning to the main instance
+      begin
+        ActiveRecord::Base.establish_connection((connection || {}).merge({:allow_concurrency => true})) if defined?(ActiveRecord)
+      rescue ActiveRecord::AdapterNotSpecified
+        # AR was defined but we didn't have a connection
+      end
+    end
+    return child
+  end
+end

data/lib/hydra/spec/autorun_override.rb

@@ -0,0 +1,3 @@
+if defined?(RSpec)
+  RSpec::Core::Runner.disable_autorun!
+end

data/lib/hydra/spec/hydra_formatter.rb

@@ -0,0 +1,26 @@
+require 'rspec/core/formatters/progress_formatter'
+module RSpec
+  module Core
+    module Formatters
+      class HydraFormatter < ProgressFormatter
+        def example_passed(example)
+        end
+
+        def example_pending(example)
+        end
+
+        def example_failed(example)
+        end
+
+        # Stifle the post-test summary
+        def dump_summary(duration, example, failure, pending)
+        end
+
+        # Stifle pending specs
+        def dump_pending
+        end
+      end
+    end
+  end
+end
+

data/lib/hydra/ssh.rb

@@ -0,0 +1,41 @@
+require 'open3'
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write with an ssh connection. For example:
+  #   @ssh = Hydra::SSH.new(
+  #     'localhost', # connect to this machine
+  #     '/home/user', # move to the home directory
+  #     "ruby hydra/test/echo_the_dolphin.rb" # run the echo script
+  #   )
+  #   @message = Hydra::Messages::TestMessage.new("Hey there!")
+  #   @ssh.write @message
+  #   puts @ssh.gets.text
+  #     => "Hey there!"
+  #
+  # Note that what ever process you run should respond with Hydra messages.
+  class SSH
+    include Open3
+    include Hydra::MessagingIO
+
+    # Initialize new SSH connection.
+    # The first parameter is passed directly to ssh for starting a connection.
+    # The second parameter is the directory to CD into once connected.
+    # The third parameter is the command to run
+    # So you can do:
+    #   Hydra::SSH.new('-p 3022 user@server.com', '/home/user/Desktop', 'ls -l')
+    # To connect to server.com as user on port 3022, then CD to their desktop, then
+    # list all the files.
+    def initialize(connection_options, directory, command)
+      @writer, @reader, @error = popen3("ssh -tt #{connection_options}")
+      @writer.write("mkdir -p #{directory}\n")
+      @writer.write("cd #{directory}\n")
+      @writer.write(command+"\n")
+    end
+
+    # Close the SSH connection
+    def close
+      @writer.write "exit\n"
+      super
+    end
+  end
+end

data/lib/hydra/stdio.rb

@@ -0,0 +1,16 @@
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write via stdout and stdin.
+  class Stdio
+    include Hydra::MessagingIO
+
+    # Initialize new Stdio
+    def initialize()
+      @reader = $stdin
+      @writer = $stdout
+      @reader.sync = true
+      @writer.sync = true
+    end
+  end
+end
+

data/lib/hydra/sync.rb

@@ -0,0 +1,99 @@
+require 'yaml'
+module Hydra #:nodoc:
+  # Hydra class responsible for delegate work down to workers.
+  #
+  # The Sync is run once for each remote worker.
+  class Sync
+    traceable('SYNC')
+    self.class.traceable('SYNC MANY')
+
+    attr_reader :connect, :ssh_opts, :remote_dir
+
+    # Create a new Sync instance to rsync source from the local machine to a remote worker
+    #
+    # Arguments:
+    # * :worker_opts
+    #   * A hash of the configuration options for a worker.
+    # * :sync
+    #   * A hash of settings specifically for copying the source directory to be tested
+    #     to the remote worked
+    # * :verbose
+    #   * Set to true to see lots of Hydra output (for debugging)
+    def initialize(worker_opts, sync_opts, verbose = false)
+      worker_opts ||= {}
+      worker_opts.stringify_keys!
+      @verbose = verbose
+      @connect = worker_opts.fetch('connect') { raise "You must specify an SSH connection target" }
+      @ssh_opts = worker_opts.fetch('ssh_opts') { "" }
+      @remote_dir = worker_opts.fetch('directory') { raise "You must specify a remote directory" }
+
+      return unless sync_opts
+      sync_opts.stringify_keys!
+      @local_dir = sync_opts.fetch('directory') { raise "You must specify a synchronization directory" }
+      @exclude_paths = sync_opts.fetch('exclude') { [] }
+
+      trace "Initialized"
+      trace "  Worker: (#{worker_opts.inspect})"
+      trace "  Sync:   (#{sync_opts.inspect})"
+
+      sync
+    end
+
+    def sync
+      #trace "Synchronizing with #{connect}\n\t#{sync_opts.inspect}"
+      exclude_opts = @exclude_paths.inject(''){|memo, path| memo += "--exclude=#{path} "}
+
+      rsync_command = [
+        'rsync',
+        '-avz',
+        '--delete',
+        exclude_opts,
+        File.expand_path(@local_dir)+'/',
+        "-e \"ssh #{@ssh_opts}\"",
+        "#{@connect}:#{@remote_dir}"
+      ].join(" ")
+      trace rsync_command
+      trace `#{rsync_command}`
+    end
+
+    def self.sync_many opts
+      opts.stringify_keys!
+      config_file = opts.delete('config') { nil }
+      if config_file
+        opts.merge!(YAML.load_file(config_file).stringify_keys!)
+      end
+      @verbose = opts.fetch('verbose') { false }
+      @sync = opts.fetch('sync') { {} }
+
+      workers_opts = opts.fetch('workers') { [] }
+      @remote_worker_opts = []
+      workers_opts.each do |worker_opts|
+        worker_opts.stringify_keys!
+        if worker_opts['type'].to_s == 'ssh'
+          @remote_worker_opts << worker_opts
+        end
+      end
+
+      trace "Initialized"
+      trace "  Sync:   (#{@sync.inspect})"
+      trace "  Workers: (#{@remote_worker_opts.inspect})"
+
+      Thread.abort_on_exception = true
+      trace "Processing workers"
+      @listeners = []
+      @remote_worker_opts.each do |worker_opts|
+        @listeners << Thread.new do
+          begin
+            trace "Syncing #{worker_opts.inspect}"
+            Sync.new worker_opts, @sync, @verbose
+          rescue 
+            trace "Syncing failed [#{worker_opts.inspect}]"
+          end
+        end
+      end
+      
+      @listeners.each{|l| l.join}
+    end
+
+  end
+end

data/lib/hydra/tasks.rb

@@ -0,0 +1,375 @@
+require 'open3'
+module Hydra #:nodoc:
+  # Hydra Task Common attributes and methods
+  class Task
+    include Rake::DSL if defined?(Rake::DSL)
+
+    # Name of the task. Default 'hydra'
+    attr_accessor :name
+
+    # Command line options
+    attr_accessor :options
+
+    # Files to test.
+    # You can add files manually via:
+    #   t.files << [file1, file2, etc]
+    #
+    # Or you can use the add_files method
+    attr_accessor :files
+
+    # True if you want to see Hydra's message traces
+    attr_accessor :verbose
+
+    # Path to the hydra config file.
+    # If not set, it will check 'hydra.yml' and 'config/hydra.yml'
+    attr_accessor :config
+
+    # Automatically sort files using their historical runtimes.
+    # Defaults to true
+    # To disable:
+    #   t.autosort = false
+    attr_accessor :autosort
+
+    # Event listeners. Defaults to the MinimalOutput listener.
+    # You can add additional listeners if you'd like. For example,
+    # on linux (with notify-send) you can add the notifier listener:
+    #   t.listeners << Hydra::Listener::Notifier.new
+    attr_accessor :listeners
+
+    # Set to true if you want to run this task only on the local
+    # machine with one runner. A "Safe Mode" for some test
+    # files that may not play nice with others.
+    attr_accessor :serial
+
+    attr_accessor :environment
+
+    # Set to false if you don't want to show the total running time
+    attr_accessor :show_time
+
+    # Set to a valid file path if you want to save the output of the runners
+    # in a log file
+    attr_accessor :runner_log_file
+
+    #
+    # Search for the hydra config file
+    def find_config_file
+      @config ||= 'hydra.yml'
+      return @config if File.exists?(@config)
+      @config = File.join('config', 'hydra.yml')
+      return @config if File.exists?(@config)
+      @config = nil
+    end
+
+    # Add files to test by passing in a string to be run through Dir.glob.
+    # For example:
+    #
+    #   t.add_files 'test/units/*.rb'
+    def add_files(pattern)
+      @files += Dir.glob(pattern)
+    end
+
+  end
+
+  # Define a test task that uses hydra to test the files.
+  #
+  #   Hydra::TestTask.new('hydra') do |t|
+  #     t.add_files 'test/unit/**/*_test.rb'
+  #     t.add_files 'test/functional/**/*_test.rb'
+  #     t.add_files 'test/integration/**/*_test.rb'
+  #     t.verbose = false # optionally set to true for lots of debug messages
+  #     t.autosort = false # disable automatic sorting based on runtime of tests
+  #   end
+  class TestTask < Hydra::Task
+
+    # Create a new HydraTestTask
+    def initialize(name = :hydra)
+      @name = name
+      @files = []
+      @verbose = false
+      @autosort = true
+      @serial = false
+      @listeners = [Hydra::Listener::ProgressBar.new]
+      @show_time = true
+      @options = ''
+
+      yield self if block_given?
+
+      # Ensure we override rspec's at_exit
+      if defined?(RSpec)
+        RSpec::Core::Runner.disable_autorun!
+      end
+
+      unless @serial
+        @config = find_config_file
+      end
+
+      @opts = {
+        :verbose => @verbose,
+        :autosort => @autosort,
+        :files => @files,
+        :listeners => @listeners,
+        :environment => @environment,
+        :runner_log_file => @runner_log_file,
+        :options => @options
+      }
+      if @config
+        @opts.merge!(:config => @config)
+      else
+        @opts.merge!(:workers => [{:type => :local, :runners => 1}])
+      end
+
+      define
+    end
+
+    private
+    # Create the rake task defined by this HydraTestTask
+    def define
+      desc "Hydra Tests" + (@name == :hydra ? "" : " for #{@name}")
+      task @name do
+        if Object.const_defined?('Rails') && Rails.env == 'development'
+          $stderr.puts %{WARNING: Rails Environment is "development". Make sure to set it properly (ex: "RAILS_ENV=test rake hydra")}
+        end
+
+        start = Time.now if @show_time
+
+        puts '********************'
+        puts @options.inspect
+        master = Hydra::Master.new(@opts)
+
+        $stdout.puts "\nFinished in #{'%.6f' % (Time.now - start)} seconds." if @show_time
+
+        unless master.failed_files.empty?
+          raise "Hydra: Not all tests passes"
+        end
+      end
+    end
+  end
+
+  # Define a test task that uses hydra to profile your test files
+  #
+  #  Hydra::ProfileTask.new('hydra:prof') do |t|
+  #    t.add_files 'test/unit/**/*_test.rb'
+  #    t.add_files 'test/functional/**/*_test.rb'
+  #    t.add_files 'test/integration/**/*_test.rb'
+  #    t.generate_html = true # defaults to false
+  #    t.generate_text = true # defaults to true
+  #  end
+  class ProfileTask < Hydra::Task
+    # boolean: generate html output from ruby-prof
+    attr_accessor :generate_html
+    # boolean: generate text output from ruby-prof
+    attr_accessor :generate_text
+
+    # Create a new Hydra ProfileTask
+    def initialize(name = 'hydra:profile')
+      @name = name
+      @files = []
+      @verbose = false
+      @generate_html = false
+      @generate_text = true
+
+      yield self if block_given?
+
+      # Ensure we override rspec's at_exit
+      require 'hydra/spec/autorun_override'
+
+      @config = find_config_file
+
+      @opts = {
+        :verbose => @verbose,
+        :files => @files
+      }
+      define
+    end
+
+    private
+    # Create the rake task defined by this HydraTestTask
+    def define
+      desc "Hydra Test Profile" + (@name == :hydra ? "" : " for #{@name}")
+      task @name do
+        require 'ruby-prof'
+        RubyProf.start
+
+        runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
+        @files.each do |file|
+          $stdout.write runner.run_file(file)
+          $stdout.flush
+        end
+
+        $stdout.write "\nTests complete. Generating profiling output\n"
+        $stdout.flush
+
+        result = RubyProf.stop
+
+        if @generate_html
+          printer = RubyProf::GraphHtmlPrinter.new(result)
+          out = File.new("ruby-prof.html", 'w')
+          printer.print(out, :min_self => 0.05)
+          out.close
+          $stdout.write "Profiling data written to [ruby-prof.html]\n"
+        end
+
+        if @generate_text
+          printer = RubyProf::FlatPrinter.new(result)
+          out = File.new("ruby-prof.txt", 'w')
+          printer.print(out, :min_self => 0.05)
+          out.close
+          $stdout.write "Profiling data written to [ruby-prof.txt]\n"
+        end
+      end
+    end
+  end
+
+  # Define a sync task that uses hydra to rsync the source tree under test to remote workers.
+  #
+  # This task is very useful to run before a remote db:reset task to make sure the db/schema.rb
+  # file is up to date on the remote workers.
+  #
+  #   Hydra::SyncTask.new('hydra:sync') do |t|
+  #     t.verbose = false # optionally set to true for lots of debug messages
+  #   end  
+  class SyncTask < Hydra::Task
+
+    # Create a new SyncTestTask
+    def initialize(name = :sync)
+      @name = name
+      @verbose = false
+
+      yield self if block_given?
+
+      @config = find_config_file
+
+      @opts = {
+        :verbose => @verbose
+      }
+      @opts.merge!(:config => @config) if @config
+
+      define
+    end
+
+    private
+    # Create the rake task defined by this HydraSyncTask
+    def define
+      desc "Hydra Tests" + (@name == :hydra ? "" : " for #{@name}")
+      task @name do
+        Hydra::Sync.sync_many(@opts)
+      end
+    end
+  end
+
+  # Setup a task that will be run across all remote workers
+  #   Hydra::RemoteTask.new('db:reset')
+  #
+  # Then you can run:
+  #   rake hydra:remote:db:reset
+  class RemoteTask < Hydra::Task
+    include Open3
+    # Create a new hydra remote task with the given name.
+    # The task will be named hydra:remote:<name>
+    def initialize(name, command=nil)
+      @name = name
+      @command = command
+      yield self if block_given?
+      @config = find_config_file
+      if @config
+        define
+      else
+        task "hydra:remote:#{@name}" do ; end
+      end
+    end
+
+    private
+    def define
+      desc "Run #{@name} remotely on all workers"
+      task "hydra:remote:#{@name}" do
+        config = YAML.load_file(@config)
+        environment = config.fetch('environment') { 'test' }
+        workers = config.fetch('workers') { [] }
+        workers = workers.select{|w| w['type'] == 'ssh'}
+        @command = "RAILS_ENV=#{environment} rake #{@name}" unless @command
+
+        $stdout.write "==== Hydra Running #{@name} ====\n"
+        Thread.abort_on_exception = true
+        @listeners = []
+        @results = {}
+        workers.each do |worker|
+          @listeners << Thread.new do
+            begin
+              @results[worker] = if run_command(worker, @command)
+                "==== #{@name} passed on #{worker['connect']} ====\n"
+              else
+                "==== #{@name} failed on #{worker['connect']} ====\nPlease see above for more details.\n"
+              end
+            rescue 
+              @results[worker] = "==== #{@name} failed for #{worker['connect']} ====\n#{$!.inspect}\n#{$!.backtrace.join("\n")}"
+            end
+          end
+        end
+        @listeners.each{|l| l.join}
+        $stdout.write "\n==== Hydra Running #{@name} COMPLETE ====\n\n"
+        $stdout.write @results.values.join("\n")
+      end
+    end
+
+    def run_command worker, command
+      $stdout.write "==== Hydra Running #{@name} on #{worker['connect']} ====\n"
+      ssh_opts = worker.fetch('ssh_opts') { '' }
+      writer, reader, error = popen3("ssh -tt #{ssh_opts} #{worker['connect']} ")
+      writer.write("cd #{worker['directory']}\n")
+      writer.write "echo BEGIN HYDRA\n"
+      writer.write(command + "\r")
+      writer.write "echo END HYDRA\n"
+      writer.write("exit\n")
+      writer.close
+      ignoring = true
+      passed = true
+      while line = reader.gets
+        line.chomp!
+        if line =~ /^rake aborted!$/
+          passed = false
+        end
+        if line =~ /echo END HYDRA$/
+          ignoring = true
+        end
+        $stdout.write "#{worker['connect']}: #{line}\n" unless ignoring
+        if line == 'BEGIN HYDRA'
+          ignoring = false
+        end
+      end
+      passed
+    end
+  end
+
+  # A Hydra global task is a task that is run both locally and remotely.
+  #
+  # For example:
+  #
+  #   Hydra::GlobalTask.new('db:reset')
+  #
+  # Allows you to run:
+  #
+  #   rake hydra:db:reset
+  #
+  # Then, db:reset will be run locally and on all remote workers. This
+  # makes it easy to setup your workers and run tasks all in a row.
+  #
+  # For example:
+  #
+  #   rake hydra:db:reset hydra:factories hydra:tests
+  #
+  # Assuming you setup hydra:db:reset and hydra:db:factories as global
+  # tasks and hydra:tests as a Hydra::TestTask for all your tests
+  class GlobalTask < Hydra::Task
+    def initialize(name)
+      @name = name
+      define
+    end
+
+    private
+    def define
+      Hydra::RemoteTask.new(@name)
+      desc "Run #{@name.to_s} Locally and Remotely across all Workers"
+      task "hydra:#{@name.to_s}" => [@name.to_s, "hydra:remote:#{@name.to_s}"]
+    end
+  end
+end