tddium-old 0.4.7

data/lib/tddium/instance.rb

@@ -0,0 +1,212 @@
+=begin
+Copyright (c) 2010 tddium.com All Rights Reserved
+=end
+
+#
+# tddium support methods
+#
+#
+
+require 'rubygems'
+require 'fog'
+require 'net/http'
+require 'uri'
+AMI_NAME = 'ami-da13e3b3'
+DEV_SESSION_KEY='dev'
+
+def setup_environment(server)
+  if !$tunnel_pid.nil?
+    ENV['SELENIUM_RC_HOST'] = 'localhost'
+  else
+    ENV['SELENIUM_RC_HOST'] = server.dns_name
+  end
+  ENV['TDDIUM'] = '1'
+end
+
+# Start and setup an EC2 instance to run a selenium-grid node.  Set the
+# tddium_session tag to session_key, if it's specified.
+def start_instance(session_key=nil)
+  conf = read_config
+
+  if session_key.nil?
+    @tddium_session = rand(2**64-1).to_s(36)
+  else
+    @tddium_session = session_key
+  end
+
+  key_file = get_keyfile
+
+  @ec2pool = Fog::AWS::Compute.new(:aws_access_key_id => conf[:aws_key],
+                                   :aws_secret_access_key => conf[:aws_secret])
+
+  server = @ec2pool.servers.create(:flavor_id => 'm1.large',
+                                   :groups => ['selenium-grid'],
+                                   :image_id => AMI_NAME,
+                                   :name => 'sg-server',
+                                   :key_name => conf[:key_name])
+
+  server.wait_for { ready? }
+  server.reload
+
+  @ec2pool.tags.create(:key => 'tddium_session', 
+                       :value => @tddium_session,
+                       :resource_id => server.id)
+
+  if conf.include?(:server_tag) then
+    server_tag = conf[:server_tag].split('=')
+
+    @ec2pool.tags.create(:key => server_tag[0],
+                         :value => server_tag[1],
+                         :resource_id => server.id)
+  end
+
+
+  puts "started instance #{server.id} #{server.dns_name} in group #{server.groups} with tags #{server.tags.inspect}"
+
+  if conf[:ssh_tunnel] == "true"
+    make_ssh_tunnel(key_file, server)
+  end
+
+  setup_environment(server)
+
+  uri = wait_for_selenium(ENV['SELENIUM_RC_HOST'])
+
+  puts "Selenium Console:"
+  puts "#{uri}"
+
+  if !key_file.nil?
+    STDERR.puts "You can login via \"ssh -i #{key_file} ec2-user@#{server.dns_name}\""
+    STDERR.puts "Making /var/log/messages world readable"
+    remote_cmd(server.dns_name, "sudo chmod 644 /var/log/messages")
+  else
+    # TODO: Remove when /var/log/messages bug is fixed
+    STDERR.puts "No key_file provided.  /var/log/messages may not be readable by ec2-user."
+  end
+
+  server
+end
+
+def wait_for_selenium(hostname)
+  uri = URI.parse("http://#{hostname}:4444/console")
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.open_timeout = 60
+  http.read_timeout = 60
+
+  rc_up = false
+  tries = 0
+  while !rc_up && tries < 5
+    begin
+      http.request(Net::HTTP::Get.new(uri.request_uri))
+      rc_up = true
+    rescue Errno::ECONNREFUSED
+      sleep 5
+    rescue Timeout::Error
+    ensure
+      tries += 1
+    end
+  end
+  raise "Couldn't connect to #{uri.request_uri}" unless rc_up
+  uri
+end
+
+def recycle_server(dns_name)
+  conf = read_config
+  keyfile = get_keyfile
+
+  if keyfile.nil?
+    raise "No keyfile.  Can't execute remote commands"
+  end
+
+  remote_cmd(dns_name, "sudo killall -9 java")
+  remote_cmd(dns_name, "bin/launch-hub.sh")
+  wait_for_selenium(dns_name)
+  remote_cmd(dns_name, "bin/launch-rc.sh")
+  sleep 5
+  STDERR.puts "Recycled Selenium on instance: #{dns_name}"
+end
+
+def recycle_servers
+  servers = find_instances
+  servers.each do |s|
+    recycle_server(s.dns_name)
+  end
+end 
+
+def recycle_dev
+  dev_servers = session_instances(DEV_SESSION_KEY)
+  
+  if dev_servers.length > 0 then
+    recycle_server(dev_servers[0].dns_name)
+  else
+    STDERR.puts "No dev servers found"
+  end
+end
+
+def checkstart_dev_instance
+  conf = read_config
+  dev_servers = session_instances(DEV_SESSION_KEY)
+  if dev_servers.length > 0
+    STDERR.puts "Using existing server #{dev_servers[0].dns_name}."
+    setup_environment(dev_servers[0])
+    return dev_servers[0]
+  else
+    STDERR.puts "Starting EC2 Instance"
+    server = start_instance(DEV_SESSION_KEY)
+    sleep 30
+    return server
+  end
+end
+
+# Find all instances running the tddium AMI
+def find_instances
+  conf = read_config
+  @ec2pool = Fog::AWS::Compute.new(:aws_access_key_id => conf[:aws_key],
+                              :aws_secret_access_key => conf[:aws_secret])
+
+  @ec2pool.servers.select do |s|
+    s.image_id == AMI_NAME &&
+      !%w{terminated stopped shutting-down}.include?(s.state)
+  end
+end
+
+def session_instances(session_key)
+  servers = find_instances
+  if servers.nil?
+    return nil
+  else
+    session_servers = []
+    servers.each do |s|
+      # in Fog 0.3.33, :filters is buggy and won't accept resourceId or resource_id
+      tags = @ec2pool.tags(:filters => {:key => 'tddium_session'}).select{|t| t.resource_id == s.id}
+      if tags.first and tags.first.value == session_key then
+        STDERR.puts "selecting instance #{s.id} #{s.dns_name} from our session"
+        session_servers << s
+      else
+        STDERR.puts "skipping instance #{s.id} #{s.dns_name} created in another session"
+      end
+    end
+    return session_servers
+  end
+end
+
+def stop_all_instances
+  servers = find_instances
+  servers.each do |s|
+    STDERR.puts "stopping instance #{s.id} #{s.dns_name}"
+    s.destroy
+  end
+end
+
+# Stop the instance created by start_instance
+def stop_instance(session_key=nil)
+  conf = read_config
+
+  kill_tunnel
+
+  servers = session_instances(session_key ? session_key : @tddium_session)
+  servers.each do |s|
+    STDERR.puts "stopping instance #{s.id} #{s.dns_name} from our session"
+    s.destroy
+  end
+  nil
+end

data/lib/tddium/parallel.rb

@@ -0,0 +1,87 @@
+=begin
+Copyright (c) 2010 tddium.com All Rights Reserved
+=end
+
+require 'parallel'
+require 'tddium/config'
+
+def make_spec_cmd(tests, environment, result_path)
+  if !result_path
+    raise "result_path must be specified"
+  end
+
+  environ = { 
+    "RAILS_ENV" => environment,
+    'RSPEC_COLOR' => $stdout.tty? ? 1 : nil
+  }
+
+  env_str = environ.map{|k,v| "#{k}=#{v}" if v}.join(' ')
+  cmd = "env #{env_str} "
+  cmd << "spec "
+  cmd << "#{tests.map{|x| "\"#{x}\""}.join(' ')} "
+  cmd << spec_opts(result_path).join(' ')
+  cmd
+end
+
+def test_batches(num_batches)
+  tests = find_test_files
+  STDERR.puts "\t#{tests.size} test files"
+
+  chunk_size = tests.size / num_batches
+  remainder = tests.size % num_batches
+  batches = []
+  num_batches.times do |c|
+    if c < remainder
+      batches << tests[((c*chunk_size)+c),(chunk_size+1)]
+    else
+      batches << tests[((c*chunk_size)+remainder),chunk_size]
+    end
+  end
+  
+  STDERR.puts batches.inspect
+  batches
+end
+
+def parallel_task(args)
+  args.with_defaults(:threads => 5, :environment => "selenium", :result_directory => '.')
+  threads = args.threads.to_i
+
+  STDERR.puts args.inspect
+
+  latest = args.result_directory
+  puts "Running tests. Results will be in #{latest}"
+  puts "Started at #{Time.now.inspect}"
+
+  output = {}
+
+  batches = test_batches(threads)
+  num_threads = batches.size
+
+  Parallel.in_threads(num_threads) do |i|
+    if batches[i].size > 0
+      result_path = File.join(latest, "#{i}-#{REPORT_FILENAME}")
+      cmd = make_spec_cmd(batches[i], args.environment, result_path)
+      output.merge!({"#{batches[i].inspect}" => execute_command( cmd )})
+      #puts "Running results: #{output.inspect}"
+    end
+  end
+
+  puts "Results:"
+  output.each do |key, value|
+    puts ">>>>>>>> #{key}"
+    puts value
+  end
+  puts "Finished at #{Time.now.inspect}"
+end
+
+def self.execute_command(cmd)
+  STDERR.puts "Running '#{cmd}'"
+  f = open("|#{cmd}")
+  all = ''
+  while out = f.gets(".")
+    all+=out
+    print out
+    STDOUT.flush
+  end
+  all
+end

data/lib/tddium/rails.rb

@@ -0,0 +1,71 @@
+=begin
+Copyright (c) 2010 tddium.com All Rights Reserved
+=end
+
+require 'tddium/config'
+require 'digest/sha1'
+require 'tddium_helper'
+require 'ftools'
+
+# Portions of this file derived from spec_storm, under the following license:
+#
+# Copyright (c) 2010 Sauce Labs Inc
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+def prepare_task(args)
+  args.with_defaults(:environment => "selenium")
+
+  tests = find_test_files()
+  puts "\t#{tests.size} test files"
+  first = true
+
+  tests.each do |test|
+    prefix = SpecStorm::db_prefix_for(test)
+    puts "Migrating another set of tables..."
+    puts "Generating DB_PREFIX: #{test} -> #{prefix}"
+
+    if first == true
+      ["export DB_PREFIX=#{prefix}; rake db:drop RAILS_ENV=#{args.environment} --trace",
+       "export DB_PREFIX=#{prefix}; rake db:create RAILS_ENV=#{args.environment} --trace"].each do |command|
+        IO.popen( command ).close 
+      end
+    end
+    
+    ["export DB_PREFIX=#{prefix}; rake db:migrate RAILS_ENV=#{args.environment} --trace"].each do |cmd|
+      IO.popen( cmd ).close
+    end
+    
+    first = false
+  end
+end
+
+def setup_task(args)
+  args.with_defaults(:environment => "selenium")
+
+  File.copy(File.join(RAILS_ROOT, 'config', 'environments', 'test.rb'),
+            File.join(RAILS_ROOT, 'config', 'environments', "#{args.environment}.rb"))
+
+  open(File.join(RAILS_ROOT, 'config', 'environments', "#{args.environment}.rb"), 'a') do |f|
+    f.puts "\nmodule SpecStorm"
+    f.puts "  USE_NAMESPACE_HACK = true"
+    f.puts "end"
+  end
+end

data/lib/tddium/reporting.rb

@@ -0,0 +1,42 @@
+=begin
+Copyright (c) 2010 tddium.com All Rights Reserved
+=end
+#
+# Prepare the result directory, as specified by config[:result_directory].
+#
+# If the directory doesn't exist create it, and a latest subdirectory.
+#
+# If the latest subdirectory exists, rotate it and create a new empty latest.
+#
+def result_directory
+  conf = read_config
+  latest = File.join(conf[:result_directory], 'latest')
+
+  if File.directory?(latest) then
+    mtime = File.stat(latest).mtime.strftime("%Y%m%d-%H%M%S")
+    archive = File.join(conf[:result_directory], mtime)
+    FileUtils.mv(latest, archive)
+  end
+  FileUtils.mkdir_p latest
+  latest
+end
+
+REPORT_FILENAME = "selenium_report.html"
+
+def default_report_path
+  File.join(read_config[:result_directory], 'latest', REPORT_FILENAME)
+end
+
+def collect_syslog(target_directory='.')
+  keyfile = get_keyfile
+  if keyfile.nil?
+    raise "No ssh keyfile configured.  Can't connect to remote"
+  end
+  instances = session_instances(@tddium_session ? @tddium_session : DEV_SESSION_KEY)
+  instances.each do |inst|
+    %w(selenium-hub selenium-rc).each do |log|
+      remote_cp(inst.dns_name, "/var/log/#{log}.log",
+                File.join(target_directory, "#{log}.#{inst.dns_name}"))
+    end
+  end
+end

data/lib/tddium/ssh.rb

@@ -0,0 +1,49 @@
+=begin
+Copyright (c) 2010 tddium.com All Rights Reserved
+=end
+
+# Subprocess main body to create an ssh tunnel to hostname for selenium, binding
+# remote:4444 to local:4444. Authenticate with the private key in key_file.
+# 
+# The ssh tunnel will auto-accept the remote host key.
+def ssh_tunnel(hostname)
+  ssh_up = false
+  tries = 0
+  while !ssh_up && tries < 3
+    sleep 3
+    ssh_up = remote_cmd(hostname, "-L 4444:#{hostname}:4444 -N")
+    tries += 1
+  end
+end
+
+def make_ssh_tunnel(key_file, server)
+  $tunnel_pid = nil
+  if !key_file.nil? then
+    $tunnel_pid = Process.fork do
+      ssh_tunnel(server.dns_name)
+    end
+
+    STDERR.puts "Created ssh tunnel to #{server.dns_name}:4444 at localhost:4444 [pid #{$tunnel_pid}]"
+  end
+end
+
+def kill_tunnel
+  if !$tunnel_pid.nil?
+    Process.kill("TERM", $tunnel_pid)
+    Process.waitpid($tunnel_pid)
+    $tunnel_pid = nil
+  end
+end
+
+
+def remote_cmd(host, cmd)
+  key_file = get_keyfile
+
+  system("ssh -o 'StrictHostKeyChecking no' -i #{key_file} ec2-user@#{host} '#{cmd}'")
+end
+
+def remote_cp(host, remote_file, local_file)
+  key_file = get_keyfile
+  system("scp -o 'StrictHostKeyChecking no' -i #{key_file} ec2-user@#{host}:#{remote_file} #{local_file}")
+end
+