lobot 1.0.pre → 2.0.0pre

data/lib/lobot/sobo.rb

@@ -0,0 +1,35 @@
+require 'net/ssh'
+
+module Lobot
+  class Sobo
+    attr_reader :ip, :key
+    attr_writer :timeout
+
+    def initialize(ip, key)
+      @ip = ip
+      @key = key
+    end
+
+    def timeout
+      @timeout || 10000
+    end
+
+    def exec(command)
+      Net::SSH.start(ip, "ubuntu", :keys => [key], :timeout => timeout) do |ssh|
+        ssh.exec(command)
+      end
+    end
+
+    def exec!(command)
+      output = nil
+      Net::SSH.start(ip, "ubuntu", :keys => [key], :timeout => timeout) do |ssh|
+        output = ssh.exec!(command)
+      end
+      output
+    end
+
+    def upload(from, to, opts = "--exclude .git")
+      system("rsync -avz --delete #{from} ubuntu@#{ip}:#{to} #{opts}")
+    end
+  end
+end

data/lib/lobot/tasks/ci.rake

@@ -1,130 +1,4 @@
 namespace :ci do
-  desc "Spin up CI server on amazon"
-  task :create_server do
-    require 'fog'
-    require 'yaml'
-    require 'socket'
-
-    aws_conf_location = File.join(Dir.pwd, 'config', 'ci.yml')
-    aws_conf = YAML.load_file(aws_conf_location)
-    aws_credentials = aws_conf['credentials']
-    ec2_server_access = aws_conf['ec2_server_access']
-    server_config = aws_conf['server']
-
-    security_group_name = server_config['security_group']
-
-    aws_connection = Fog::Compute.new(
-      :provider => aws_credentials['provider'],
-      :aws_access_key_id => aws_credentials['aws_access_key_id'],
-      :aws_secret_access_key => aws_credentials['aws_secret_access_key']
-    )
-
-    security_group = aws_connection.security_groups.get(security_group_name)
-
-    if security_group.nil?
-      puts "Could not find security group named '#{security_group_name}'.  Creating..."
-      security_group = aws_connection.security_groups.new(:name => security_group_name, :description => 'ci servers group auto-created by lobot')
-      security_group.save
-      security_group.reload
-      p security_group
-    end
-
-    PORTS_TO_OPEN = [22, 443, 80, 8111 ] + (9000...9010).to_a
-    PORTS_TO_OPEN.each do |port|
-      is_in_security_group = !!security_group.ip_permissions.detect{|group| (group['fromPort']..group['toPort']).include?(port) && group['ipRanges'].detect{|range| range["cidrIp"]== "0.0.0.0/0" } }
-
-      unless is_in_security_group
-        puts "Allowing port #{port} into '#{security_group_name}' security group"
-        security_group.authorize_port_range(port..port)
-      end
-    end
-
-    ec2_key_pair_name = ec2_server_access['key_pair_name'] || "ci"
-    public_key_local_path = File.expand_path("#{ec2_server_access['id_rsa_path']}.pub")
-
-    current_key_pair = aws_connection.key_pairs.get(ec2_key_pair_name)
-    if current_key_pair
-      puts "Using existing '#{ec2_key_pair_name}' keypair"
-    else
-      raise "Unable to upload keypair, missing #{public_key_local_path}!" unless File.exist?(public_key_local_path)
-      puts "Creating '#{ec2_key_pair_name}' keypair, uploading #{public_key_local_path} to aws"
-
-      aws_connection.key_pairs.new(
-        :name => ec2_key_pair_name,
-        :public_key => File.read("#{public_key_local_path}")
-      ).save
-    end
-
-    number_of_servers = aws_connection.servers.select{ |server| server.state == 'running' }.length
-    puts "you have #{number_of_servers} server(s) already running in this account" if number_of_servers > 0
-
-    puts "Launching server... (this costs money until you stop it)"
-    server = aws_connection.servers.create(
-      :image_id => 'ami-a29943cb',
-      :flavor_id =>  server_config['flavor_id'] || 'c1.medium',
-      :key_name => ec2_key_pair_name,
-      :groups => [security_group_name]
-    )
-
-    unless aws_conf['server']['elastic_ip'] =~ /\d.\.\d.\.\d.\.\d./
-      elastic_ip = aws_connection.addresses.create
-      aws_conf['server']['elastic_ip'] = elastic_ip.public_ip
-      puts "Allocated elastic IP address #{aws_conf['server']['elastic_ip']}"
-    end
-
-    server.wait_for { ready? }
-
-    aws_connection.associate_address(server.id, aws_conf['server']['elastic_ip'])
-
-    socket = false
-    Timeout.timeout(180) do
-      print "Server booted, waiting for SSH to come up on #{aws_conf['server']['elastic_ip']}: "
-      until socket
-        begin
-          Timeout.timeout(5) do
-            socket = TCPSocket.open(aws_conf['server']['elastic_ip'], 22)
-          end
-        rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Timeout::Error, Errno::EHOSTUNREACH
-        end
-        putc "."
-        sleep 1
-      end
-    end
-    puts ""
-
-    puts "Server is ready:"
-    p server
-
-    puts "Writing server instance_id(#{server.id}) and elastic IP(#{aws_conf['server']['elastic_ip']}) to ci.yml"
-    aws_conf["server"].merge!("instance_id" => server.id)
-
-    f = File.open(aws_conf_location, "w")
-    f.write(aws_conf.to_yaml)
-    f.close
-  end
-
-  desc "terminate the CI Server and release IP"
-  task :destroy_server do
-    puts "Terminating the CI Server and releasing IP..."
-    require 'fog'
-    require 'yaml'
-    require 'socket'
-
-    aws_conf_location = File.join(Dir.pwd, 'config', 'ci.yml')
-    aws_conf = YAML.load_file(aws_conf_location)
-    aws_credentials = aws_conf['credentials']
-    server_config = aws_conf['server']
-
-    aws_connection = Fog::Compute.new(
-      :provider => aws_credentials['provider'],
-      :aws_access_key_id => aws_credentials['aws_access_key_id'],
-      :aws_secret_access_key => aws_credentials['aws_secret_access_key']
-    )
-
-    aws_connection.release_address(aws_conf['server']['elastic_ip'])
-    aws_connection.servers.new(:id => server_config['instance_id']).destroy
-  end
-
   desc "stop(suspend) the CI Server"
   task :stop_server do
     puts "Stopping (suspending) the CI Server..."
@@ -170,25 +44,6 @@ namespace :ci do
     aws_connection.associate_address(server_config['instance_id'], server_config['elastic_ip']) if server_config['elastic_ip']
   end
 
-  desc "open the CI interface in a browser"
-  task :open do
-    aws_conf_location = File.join(Dir.pwd, 'config', 'ci.yml')
-    aws_conf = YAML.load_file(aws_conf_location)
-    server_config = aws_conf['server']
-    exec "open https://#{server_config['elastic_ip']}"
-  end
-
-  desc "ssh to CI"
-  task :ssh do
-    aws_conf_location = File.join(Dir.pwd, 'config', 'ci.yml')
-    aws_conf = YAML.load_file(aws_conf_location)
-    server_config = aws_conf['server']
-    ssh_port = server_config['ssh_port'] || 22
-    cmd = "ssh -i #{aws_conf['ec2_server_access']['id_rsa_path']} ubuntu@#{server_config['elastic_ip']} -p #{ssh_port}"
-    puts cmd
-    exec cmd
-  end
-
   desc "Get build status"
   task :status do
     require 'nokogiri'
@@ -235,37 +90,4 @@ namespace :ci do
       puts "EC2 instance information not available. Did you run rake ci:server_start?"
     end
   end
-
-  desc "Run a command with a virtual frame buffer"
-  task :headlessly, :command do |task, args|
-    # headless is your friend on linux - http://www.aentos.com/blog/easy-setup-your-cucumber-scenarios-using-headless-gem-run-selenium-your-ci-server
-    begin
-      Headless
-    rescue NameError
-      puts "Headless not available, did you add it to your Gemfile?"
-      exit 1
-    end
-    unless args[:command]
-      puts "Usage: rake ci:headlessly[command] <additional options>"
-      exit 1
-    end
-    exit_code = 1
-    Headless.ly(:display => 42) do |headless|
-      begin
-        command = args[:command].gsub(/^['"](.*)['"]$/, "\\1")
-        system(command)
-        exit_code = $?.exitstatus
-      ensure
-        headless.destroy
-      end
-    end
-    exit exit_code
-  end
-
-  #aliases
-  desc "maybe"
-  task "server:create" => "ci:create_server"
-  task "server:start" => "ci:start_server"
-  task "server:destroy" => "ci:destroy_server"
-  task "server:stop" => "ci:stop_server"
 end

data/lib/lobot/version.rb

@@ -1,3 +1,3 @@
 module Lobot
-  VERSION = "1.0.pre"
+  VERSION = "2.0.0pre"
 end

data/lobot.gemspec

@@ -15,21 +15,25 @@ Gem::Specification.new do |s|
   s.rubyforge_project = "lobot"
 
   s.files         = `git ls-files`.split("\n") + `cd chef/travis-cookbooks && git ls-files`.split("\n").map { |f| "chef/travis-cookbooks/#{f}" }
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.test_files    = `git ls-files -- {test,spec,features}`.split("\n")
+  s.executables   = `git ls-files -- bin`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency('fog', '>=0.9.0')
-  s.add_dependency('capistrano')
-  s.add_dependency('capistrano-ext')
-  s.add_dependency('rvm')
-  s.add_dependency('rvm-capistrano')
-  s.add_dependency('nokogiri', '>=1.4.4')
-  s.add_dependency('ci_reporter', '>=1.7.0')
-  s.add_dependency('headless')
-  s.add_development_dependency('cucumber')
-  s.add_development_dependency('rspec')
-  s.add_development_dependency('generator_spec')
-  s.add_development_dependency('rails')
-  s.add_development_dependency('jasmine')
+  s.add_dependency "fog", "~> 1.6"
+  s.add_dependency "ci_reporter", "~> 1.7"
+  s.add_dependency "thor"
+  s.add_dependency "hashie"
+  s.add_dependency "net-ssh"
+
+  s.add_development_dependency "godot"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "jasmine"
+  s.add_development_dependency "guard-rspec"
+  s.add_development_dependency "guard-bundler"
+  s.add_development_dependency "test-kitchen"
+  s.add_development_dependency "pry"
+
+  s.add_development_dependency "terminal-notifier-guard"
+  s.add_development_dependency "rb-fsevent"
+  s.add_development_dependency "vagrant", "~> 1.0"
 end

data/script/bootstrap_server.sh

@@ -2,11 +2,14 @@
 
 set -e
 
+sudo apt-get update > /dev/null
+
 packages="git build-essential openssl libreadline6 libreadline6-dev libreadline5 curl git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev autoconf libc6-dev libncurses5-dev automake libtool bison subversion pkg-config"
+selections=`dpkg --get-selections`
 
 for package in $packages
 do
-  if ! dpkg --get-selections | grep "^$package\s" > /dev/null
+  if ! echo "$selections" | grep "^$package\s" > /dev/null
   then
     to_install="$to_install $package"
   fi

data/spec/lib/lobot/amazon_spec.rb

@@ -0,0 +1,127 @@
+require "spec_helper"
+
+describe Lobot::Amazon, :slow => true do
+  let(:tempdir) { Dir.mktmpdir }
+  let(:amazon) { Lobot::Amazon.new(ENV["EC2_KEY"], ENV["EC2_SECRET"]) }
+  let(:fog) { amazon.send(:fog) }
+
+  describe "#create_security_group" do
+    context "when there is no existing security group" do
+      it "creates a security group" do
+        amazon.create_security_group("totally_not_a_honeypot")
+        amazon.security_groups.map(&:name).should include "totally_not_a_honeypot"
+      end
+    end
+
+    context "when the security group already exists" do
+      before { amazon.create_security_group("bart_police") }
+
+      it "does not complain" do
+        expect { amazon.create_security_group("bart_police") }.not_to raise_error
+      end
+    end
+  end
+
+  describe "#open_port" do
+    before { amazon.create_security_group("bag_of_weasels") }
+
+    let(:group) { amazon.security_groups.get("bag_of_weasels") }
+
+    def includes_port?(permissions, ports)
+      permissions.any? { |p| (p["fromPort"]..p["toPort"]).include?(80) }
+    end
+
+    it "opens a port for business" do
+      group.ip_permissions.should_not include "80"
+      amazon.open_port("bag_of_weasels", 80)
+      includes_port?(group.reload.ip_permissions, 80).should be_true
+    end
+
+    it "takes a bunch of ports" do
+      amazon.open_port("bag_of_weasels", 22, 443)
+      includes_port?(group.reload.ip_permissions, 22).should be_true
+      includes_port?(group.reload.ip_permissions, 443).should be_true
+    end
+  end
+
+  describe "#add_key_pair" do
+    let(:key_pair_path) { "#{tempdir}/supernuts" }
+
+    before do
+      system "ssh-keygen -q -f #{key_pair_path} -P ''"
+      amazon.delete_key_pair("is_supernuts")
+    end
+
+    it "uploads the key" do
+      amazon.add_key_pair("is_supernuts", "#{key_pair_path}.pub")
+      amazon.key_pairs.map(&:name).should include "is_supernuts"
+    end
+
+    context "when the key is already there" do
+      before { amazon.add_key_pair("is_supernuts", "#{key_pair_path}.pub") }
+
+      it "doesn't reupload" do
+        expect do
+          amazon.add_key_pair("is_supernuts", "#{key_pair_path}.pub")
+        end.not_to raise_error
+      end
+    end
+  end
+
+  describe "things which launch instances" do
+    let(:key_pair_path) { "#{tempdir}/cookie" }
+
+    before do
+      system "ssh-keygen -q -f #{key_pair_path} -P ''"
+      amazon.add_key_pair("eating_my_cookie", "#{key_pair_path}.pub")
+      amazon.create_security_group("chump_of_change")
+    end
+
+    let(:freshly_launched_server) { amazon.launch_server("eating_my_cookie", "chump_of_change", "t1.micro") }
+
+    describe "#launch_instance" do
+      it "creates an instance" do
+        expect { freshly_launched_server }.to change { amazon.servers.reload.count }.by(1)
+
+        freshly_launched_server.availability_zone.should == "us-east-1a"
+        freshly_launched_server.flavor_id.should == "t1.micro"
+        freshly_launched_server.tags.should == {"lobot"=>Lobot::VERSION, "Name"=>"Lobot"}
+        freshly_launched_server.key_name.should == "eating_my_cookie"
+        freshly_launched_server.groups.should == ["chump_of_change"]
+        freshly_launched_server.public_ip_address.should == amazon.elastic_ip_address.public_ip
+
+        freshly_launched_server.destroy
+        amazon.elastic_ip_address.destroy
+      end
+    end
+
+    describe "#destroy_ec2" do
+      let!(:server_ip) { freshly_launched_server.public_ip_address }
+
+      it "stops all the instances" do
+        expect do
+          amazon.destroy_ec2
+        end.to change { freshly_launched_server.reload.state }.from("running")
+        fog.addresses.get(server_ip).should_not be
+      end
+    end
+  end
+
+  describe "#elastic_ip_address" do
+    it "allocates an ip address" do
+      expect { amazon.elastic_ip_address }.to change { fog.addresses.reload.count }.by(1)
+      amazon.elastic_ip_address.public_ip.should =~ /\d+\.\d+\.\d+\.\d+/
+      amazon.elastic_ip_address.destroy
+    end
+  end
+
+  describe "#release_elastic_ip" do
+    let!(:elastic_ip) { amazon.elastic_ip_address }
+
+    it "releases the ip" do
+      expect do
+        amazon.release_elastic_ip(elastic_ip.public_ip)
+      end.to change { fog.addresses.reload.count }.by(-1)
+    end
+  end
+end

data/spec/lib/lobot/cli_spec.rb

@@ -0,0 +1,212 @@
+require "spec_helper"
+
+describe Lobot::CLI do
+  let(:tempfile) do
+    Tempfile.new('lobot-config').tap do |file|
+      file.write YAML.dump({})
+      file.close
+    end
+  end
+
+  let(:lobot_config) { Lobot::Config.new(:aws_key => ENV["EC2_KEY"], :aws_secret => ENV["EC2_SECRET"]) }
+  let(:cli) { subject }
+  let(:sobo) { Lobot::Sobo.new(lobot_config.master, lobot_config.server_ssh_key) }
+
+  before do
+    cli.stub(:lobot_config).and_return(lobot_config)
+  end
+
+  describe "#ssh" do
+    it "starts an ssh session to the lobot host" do
+      cli.should_receive(:exec).with("ssh -i #{cli.lobot_config.server_ssh_key} ubuntu@#{cli.lobot_config.master} -p #{cli.lobot_config.ssh_port}")
+      cli.ssh
+    end
+  end
+
+  describe "#open" do
+    let(:lobot_config) do
+      Lobot::Config.new(:node_attributes => {
+        :nginx => {
+          :basic_auth_user => "ci",
+          :basic_auth_password => "secret"
+        }
+      })
+    end
+
+    it "opens a web browser with the lobot page" do
+      cli.should_receive(:exec).with("open https://#{cli.lobot_config.node_attributes.nginx.basic_auth_user}:#{cli.lobot_config.node_attributes.nginx.basic_auth_password}@#{cli.lobot_config.master}/")
+      cli.open
+    end
+  end
+
+  describe "#add_build" do
+    let(:name) { "bob" }
+    let(:repository) { "http://github.com/mkocher/soloist.git" }
+    let(:branch) { "master" }
+    let(:command) { "script/ci_build.sh" }
+
+    it "adds a build to the node attributes" do
+      cli.add_build(name, repository, branch, command)
+      lobot_config.node_attributes.jenkins.builds.should =~ [{
+        "name" => "bob",
+        "repository" => "http://github.com/mkocher/soloist.git",
+        "command" => "script/ci_build.sh",
+        "branch" => "master"
+      }]
+    end
+
+    it "does not add a build twice with identical parameters" do
+      cli.add_build(name, repository, branch, command)
+      cli.add_build(name, repository, branch, command)
+      lobot_config.node_attributes.jenkins.builds.should =~ [{
+        "name" => "bob",
+        "repository" => "http://github.com/mkocher/soloist.git",
+        "command" => "script/ci_build.sh",
+        "branch" => "master"
+      }]
+    end
+
+    context "with persisted configuration data" do
+      let(:lobot_config) { Lobot::Config.from_file(tempfile.path) }
+
+      def builds
+        cli.lobot_config.reload.node_attributes.jenkins.builds
+      end
+
+      it "persists a build" do
+        cli.add_build(name, repository, branch, command)
+        builds.should_not be_nil
+        builds.should_not be_empty
+      end
+    end
+  end
+
+  shared_examples_for "a start command that updates known_hosts" do
+    let(:key) { "AAAAB3NzaC1yc2EAAAADAQABAAABAQDjhJ/xZCgVhq9Xk+3DKJZ6tcgyIHcIXKSzu6Z/EK1uykyHeP/i7CwwKgiAv7lAV7B4UiUMHUm2nEiguog9VtYc6mc0g1N829lnuMhPRyOTb0SSYTNEN7Uuwy10cuq3Rd/9QAdxNV/voQW3Rl60BFzZvzp8UxJzCXFT1NmB+0W45X7Ypstv0oVV/EdyJJUuoPijQ097A4kHt6KUThKzxhagh1UrVTCE6eccscxuuRPX3yCEf8cUaVrKtuSE3vZnBcmSOY92zA4NV/YdJYNPIrKyCvWb/R+nC4R0pQNqv1gSEqPT51wYxKnvmIPFGntKaJSN2qmMlvs/AlFnFOeUsUFN" }
+
+    def known_hosts_contents
+      File.read(File.expand_path('~/.ssh/known_hosts'))
+    end
+
+    it "clears out the entry in knownhosts as this is a new box but the ip may be recycled" do
+      system "echo '#{ip_address} ssh-rsa #{key}' >> ~/.ssh/known_hosts"
+      action
+      known_hosts_contents.should_not include(key)
+      known_hosts_contents.should include(ip_address)
+    end
+
+    it "doesn't mess with other entries" do
+      expect { action }.not_to change { known_hosts_contents }
+    end
+  end
+
+  describe "#create", :slow => true do
+    it "launches an instance and associates elastic ip" do
+      cli.lobot_config.instance_size = 't1.micro'
+      expect { cli.create }.to change { lobot_config.master }.from(nil)
+      expect { cli.destroy_ec2 }.to change { lobot_config.master }.to(nil)
+    end
+
+    context "with a fake amazon" do
+      let(:ip_address) { "192.168.33.10" }
+      let(:server) { double("server", :public_ip_address => ip_address).as_null_object }
+      let(:amazon) { double("AMZN", :launch_server => server).as_null_object }
+
+      before { cli.stub(:amazon).and_return(amazon) }
+
+      def action
+        cli.create
+      end
+
+      it_behaves_like "a start command that updates known_hosts"
+
+      context "with a custom instance size", :slow => false do
+        before { cli.lobot_config.instance_size = 'really_big_instance' }
+
+        it "launches the instance with the configured instance size" do
+          amazon.should_receive(:launch_server).with(anything, anything, 'really_big_instance')
+          cli.create
+        end
+      end
+    end
+  end
+
+  describe "#create_vagrant" do
+    before do
+      File.open(tempfile.path, "w") do |f|
+        f.write(YAML.dump(
+          "node_attributes" => {
+            "nginx" => {
+              "basic_auth_user" => "ci",
+              "basic_auth_password" => "secret"
+            }
+          }
+        ))
+      end
+    end
+
+    def lobot_config
+      Lobot::Config.from_file(tempfile.path)
+    end
+
+    it "starts a virtual machine" do
+      cli.create_vagrant
+      Godot.wait('192.168.33.10', 22).should be
+    end
+
+    it "updates the config master ip address" do
+      expect { cli.create_vagrant }.to change { lobot_config.master }.to('192.168.33.10')
+    end
+
+    context "known_hosts" do
+      let(:ip_address) { "192.168.33.10" }
+
+      def action
+        cli.create_vagrant
+      end
+
+      it_behaves_like "a start command that updates known_hosts"
+    end
+  end
+
+  describe "#bootstrap", :slow => true do
+    before { cli.create_vagrant }
+
+    it "installs all necessary packages, installs rvm and sets up the user" do
+      cli.bootstrap
+      sobo.exec!("dpkg --get-selections").should include("libncurses5-dev")
+      sobo.exec!("ls /usr/local/rvm/").should_not be_empty
+      sobo.exec!("groups ubuntu").should include("rvm")
+    end
+  end
+
+  describe "#chef", :slow => true do
+    let(:name) { "Bob" }
+    let(:repository) { "http://github.com/mkocher/soloist.git" }
+    let(:branch) { "master" }
+    let(:command) { "exit 0" }
+
+    let(:godot) { Godot.new(cli.master_server.ip, 8080) }
+    let(:jenkins) { Lobot::Jenkins.new(lobot_config) }
+
+    before do
+      cli.create_vagrant
+      cli.bootstrap
+      cli.add_build(name, repository, branch, command)
+    end
+
+    it "runs chef" do
+      cli.lobot_config.recipes = ["pivotal_ci::jenkins", "pivotal_ci::id_rsa", "pivotal_ci::git_config", "sysctl", "pivotal_ci::jenkins_config"]
+      cli.chef
+
+      sobo.exec!("ls /var/lib/").should include "jenkins"
+      sobo.exec!("grep 'kernel.shmmax=' /etc/sysctl.conf").should_not be_empty
+      sobo.exec!("sudo cat /var/lib/jenkins/.ssh/id_rsa").should == File.read(lobot_config.github_ssh_key)
+
+      godot.wait!
+      godot.match!(/Bob/, 'api/json')
+
+      jenkins.jobs.first.name.should == "Bob"
+    end
+  end
+end