lobot 2.0.0pre → 2.1.0

data/chef/travis-cookbooks/worker_host/travis_worker/metadata.rb

@@ -1,6 +1,6 @@
 maintainer       "Travis CI"
 maintainer_email "mathias@travis-ci.org"
 license          "Apache 2.0"
-description      "Installs/Configures Travis Worker"
-version          "0.2.0"
+description      "Installs/Configures the Travis Worker"
+version          "0.5.1"
 depends "runit"

data/chef/travis-cookbooks/worker_host/travis_worker/recipes/bluebox.rb

@@ -0,0 +1,97 @@
+require_recipe 'runit'
+require_recipe 'jruby'
+
+users = if Chef::Config[:solo]
+          node[:users]
+        else
+          search(:users)
+        end
+
+execute "monit-reload" do
+  action :nothing
+  command "monit reload"
+end
+
+package "ruby1.9.3" do
+  action :install
+end
+
+cookbook_file "/usr/local/bin/signal_wrapper" do
+  source "signal_wrapper.rb"
+  mode "0755"
+  owner "root"
+  backup false
+end
+
+1.upto(node[:travis][:worker][:workers]) do |worker| 
+  app = "worker-#{worker}"
+  worker_name = "#{app}.#{node[:fqdn]}"
+  home = "#{node[:travis][:worker][:home]}/#{app}"
+  service_name = "travis-worker-#{worker}"
+
+  service service_name do
+    action :nothing
+  end
+
+  directory home do
+    action :create
+    recursive true
+    owner "travis"
+    group "travis"
+    mode "0755"
+  end
+
+  git home do
+    repository node[:travis][:worker][:repository]
+    reference node[:travis][:worker][:ref]
+    action :sync
+    user "travis"
+    group "travis"
+    notifies :restart, resources(:service => service_name)
+  end
+
+  directory "#{home}/log" do
+    action :create
+    owner "travis"  
+    group "travis"
+    mode "0755"
+  end
+
+  bash "bundle gems" do
+    code "#{File.dirname(node[:jruby][:bin])}/bundle install --deployment --binstubs"
+    user "travis"
+    group "travis"
+    cwd home
+  end
+
+  template "#{home}/config/worker.yml" do
+    source "worker-bluebox.yml.erb"
+    owner "travis"
+    group "travis"
+    mode "0600"
+    variables :amqp => node[:travis][:worker][:amqp],
+              :worker => node[:travis][:worker],
+              :hostname => worker_name,
+              :bluebox => node[:bluebox],
+              :librato => node[:collectd_librato]
+
+    notifies :restart, resources(:service => service_name)
+  end
+
+  runit_service "travis-worker-#{worker}" do
+    options :jruby => node[:jruby][:bin],
+            :worker_home => home,
+            :user => "travis",
+            :group => "travis"
+    template_name "travis-worker"
+  end
+
+  template "/etc/monit/conf.d/travis-worker-#{worker}.monitrc" do
+    source "travis-worker.monitrc.erb"
+    owner "root"
+    group "root"
+    mode "0644"
+    variables :home => home
+    notifies :run, resources(:execute => 'monit-reload')
+  end
+end

data/chef/travis-cookbooks/worker_host/travis_worker/recipes/default.rb

@@ -60,18 +60,8 @@ link "#{node[:travis][:worker][:home]}/.VirtualBox/VirtualBox.xml" do
   }
 end
 
-if not node[:travis][:worker][:post_checkout].empty?
-  bash "run post checkout hook (#{node[:travis][:worker][:post_checkout][:command]})" do
-    code node[:travis][:worker][:post_checkout][:command]
-    user "travis"
-    not_if "cd #{node[:travis][:worker][:home]} && #{node[:travis][:worker][:post_checkout][:condition]}"
-    cwd node[:travis][:worker][:home]
-    notifies :restart, resources(:service => 'travis-worker')
-  end
-end
-
 bash "bundle gems" do
-  code "#{File.dirname(node[:jruby][:bin])}/bundle install --path vendor/bundle --binstubs"
+  code "#{File.dirname(node[:jruby][:bin])}/bundle install --deployment --binstubs"
   user "travis"
   group "travis"
   cwd node[:travis][:worker][:home]
@@ -89,17 +79,37 @@ template "#{node[:travis][:worker][:home]}/config/worker.yml" do
 end
 
 bash "download VirtualBox images" do
-  code "#{node[:jruby][:bin]} ./bin/thor travis:vms:download 2>/dev/null"
+  if node[:travis][:worker][:box]
+    code "#{node[:jruby][:bin]} ./bin/thor travis:vms:download #{node[:travis][:worker][:box]} 2>/dev/null"
+    not_if {
+      File.exists?("#{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}-pro.box")
+    }
+  else
+    code "#{node[:jruby][:bin]} ./bin/thor travis:vms:download 2>/dev/null"
+    not_if {
+      File.exists?("#{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}.box")
+    }
+  end
+
   user "travis"
   group "travis"
   cwd node[:travis][:worker][:home]
-  not_if {
-    File.exists?("#{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}.box")
-  }
   notifies :restart, resources(:service => 'travis-worker')
   environment({"HOME" => home})
 end
 
+bash "copy box image" do
+  code "cp #{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}-pro.box #{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}.box"
+  user "travis"
+  group "travis"
+  cwd node[:travis][:worker][:home]
+  not_if {
+    File.exists?("#{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}.box")
+  }
+  only_if {
+    File.exists?("#{node[:travis][:worker][:home]}/boxes/travis-#{node[:travis][:worker][:env]}-pro.box")
+  }
+end
 
 bash "create VirtualBox images" do
   code "#{node[:jruby][:bin]} ./bin/thor travis:vms:create &>/tmp/vbox-create.log"

data/chef/travis-cookbooks/worker_host/travis_worker/templates/default/sv-travis-worker-run.erb

@@ -1,4 +1,4 @@
 #!/bin/bash
 exec 2>&1
 cd <%= @options[:worker_home] %>
-exec chpst -u <%= @options[:user] %>:<%= @options[:group] %> <%= @options[:jruby] %> ./bin/thor travis:worker:boot
+exec chpst -u <%= @options[:user] %>:<%= @options[:group] %> /usr/local/bin/signal_wrapper <%= @options[:jruby] %> ./bin/thor travis:worker:boot

data/chef/travis-cookbooks/worker_host/travis_worker/templates/default/worker-bluebox.yml.erb

@@ -0,0 +1,21 @@
+env: <%= @worker[:env] %>
+linux:
+  host: <%= @hostname %>
+  log_level: <%= @worker[:log_level] %>
+  amqp:
+  <% @amqp.each do |key, value| -%>
+    <%= key %>: <%= value %>
+  <% end -%>
+  vms:
+    provider: <%= @worker[:provider] %>
+    count: <%= @worker[:vms] %>
+  blue_box:
+    customer_id: <%= @bluebox[:customer_id] %>
+    api_key: <%= @bluebox[:api_key] %>
+    flavor_id: <%= @bluebox[:flavor_id] %>
+    location_id: <%= @bluebox[:location_id] %>
+  librato:
+    email: <%= @librato[:email] %>
+    token: <%= @librato[:token] %>
+  language_mappings:
+    haskell: jvm

data/chef/travis-cookbooks/worker_host/travis_worker/templates/default/worker.yml.erb

@@ -8,7 +8,3 @@ env: <%= @env %>
   vms:
     count: <%= @vms %>
     vbox_home: '/usr/lib/virtualbox'
-timeouts:
-  install: 600
-  before_script: 480
-  script: 1500

data/chef/travis-cookbooks/worker_host/travis_worker_collectd/metadata.rb

@@ -1 +1,2 @@
+version "0.1.0"
 depends "collectd"

data/chef/travis-cookbooks/worker_host/travis_worker_collectd/recipes/bluebox.rb

@@ -0,0 +1,15 @@
+require_recipe 'collectd'
+
+collectd_plugin "load"
+
+collectd_plugin "memory"
+
+collectd_plugin "interface" do
+  options :interface => "lo", :ignore_selected => true
+end
+
+collectd_plugin "df" do
+  options :mount_point => "/",
+          :report_reserved => false,
+          :report_inodes => false
+end

data/chef/travis-cookbooks/worker_host/travis_worker_collectd/recipes/default.rb

@@ -9,3 +9,4 @@ collectd_plugin "cpu"
 collectd_plugin "interface" do
   options :interface => "lo", :ignore_selected => true
 end
+

data/chef/travis-cookbooks/worker_host/users/recipes/default.rb

@@ -42,6 +42,11 @@ users.each do |user|
     notifies :reload, resources(:ohai => "reload_passwd"), :immediately
   end
 
+  file "#{user[:home]}/.zshrc" do
+    action :create_if_missing
+    content "# Automatically generated by Chef."
+  end
+
   (user[:groups] || []).each do |group_name|
     group_id = groups.find {|group| group[:id] == group_name}[:gid]
 

data/cookbooks/.gitkeep

@@ -0,0 +1,3 @@
+This directory should remain empty, it is used by consumers of
+Lobot, so when we deploy a lobot CI instance for testing, this
+matches an end user's experience.

data/lib/lobot.rb

@@ -1,4 +1,8 @@
-require File.expand_path("../lobot/version", __FILE__)
-
-module Lobot
-end
+require "lobot/version"
+require "lobot/config"
+require "lobot/sobo"
+require "lobot/amazon"
+require "lobot/keychain"
+require "lobot/configuration_wizard"
+require "lobot/cli"
+require "lobot/password"

data/lib/lobot/amazon.rb

@@ -11,15 +11,15 @@ module Lobot
       @secret = secret
     end
 
-    def security_groups
+    def fog_security_groups
       fog.security_groups
     end
 
-    def key_pairs
+    def fog_key_pairs
       fog.key_pairs
     end
 
-    def servers
+    def fog_servers
       fog.servers
     end
 
@@ -28,13 +28,13 @@ module Lobot
     end
 
     def create_security_group(group_name)
-      unless security_groups.get(group_name)
-        security_groups.create(:name => group_name, :description => 'Lobot-generated group')
+      unless fog_security_groups.get(group_name)
+        fog_security_groups.create(:name => group_name, :description => 'Lobot-generated group')
       end
     end
 
     def open_port(group_name, *ports)
-      group = security_groups.get(group_name)
+      group = fog_security_groups.get(group_name)
       ports.each do |port|
         unless group.ip_permissions.any? { |p| (p["fromPort"]..p["toPort"]).include?(port) }
           group.authorize_port_range(port..port)
@@ -43,20 +43,26 @@ module Lobot
     end
 
     def delete_key_pair(key_pair_name)
-      key_pairs.new(:name => key_pair_name).destroy
+      fog_key_pairs.new(:name => key_pair_name).destroy
     end
 
-    def add_key_pair(key_pair_name, key_path)
-      unless key_pairs.get(key_pair_name)
-        key_pairs.create(:name => key_pair_name, :public_key => File.read("#{key_path}"))
-      end
+    def add_key_pair(key_pair_name, public_key)
+      fog_key_pairs.create(:name => key_pair_name, :public_key => public_key) unless fog_key_pairs.get(key_pair_name)
+    end
+
+    def with_key_pair(pubkey)
+      unique_key_pair_name = "LOBOT-#{Time.now.to_i}"
+      add_key_pair(unique_key_pair_name, pubkey)
+      yield unique_key_pair_name if block_given?
+    ensure
+      delete_key_pair(unique_key_pair_name)
     end
 
     def launch_server(key_pair_name, security_group_name, instance_type = "m1.medium")
-      servers.create(
+      fog_servers.create(
         :image_id => "ami-a29943cb",
         :flavor_id => instance_type,
-        :availability_zone => "us-east-1a",
+        :availability_zone => "us-east-1b",
         :tags => {"Name" => "Lobot", "lobot" => Lobot::VERSION},
         :key_name => key_pair_name,
         :groups => [security_group_name]
@@ -67,11 +73,19 @@ module Lobot
       end
     end
 
-    def destroy_ec2
-      servers = fog.servers.select { |s| s.tags.keys.include?("lobot") && s.state == "running" }
-      ips = servers.map(&:public_ip_address)
-      servers.map(&:destroy)
-      ips.each { |ip| release_elastic_ip(ip) }
+    def destroy_ec2(confirm_proc, *args)
+      servers.each do |server|
+        next unless (args == [:all]) || args.include?(server.id)
+        next unless confirm_proc.call(server)
+        ip = server.public_ip_address
+        server.destroy
+        release_elastic_ip(ip)
+        yield(server) if block_given?
+      end
+    end
+
+    def servers
+      fog_servers.select { |s| s.tags.keys.include?("lobot") && s.state == "running" }
     end
 
     def release_elastic_ip(ip)
@@ -88,4 +102,4 @@ module Lobot
       )
     end
   end
-end
+end

data/lib/lobot/cli.rb

@@ -1,50 +1,55 @@
 require "thor"
-require "lobot/config"
-require "lobot/sobo"
-require "lobot/amazon"
-require "lobot/jenkins"
+require "thor/group"
 require "pp"
 require "tempfile"
 require "json"
+require "lobot/configuration_wizard"
+require "godot"
 
 module Lobot
-  class CLI < Thor
+  class CLI < ::Thor
+    register(ConfigurationWizard, "setup", "setup", ConfigurationWizard::DESCRIPTION_TEXT)
+
     desc "ssh", "SSH into Lobot"
     def ssh
-      exec("ssh -i #{lobot_config.server_ssh_key} ubuntu@#{lobot_config.master} -p #{lobot_config.ssh_port}")
+      exec("ssh -i #{lobot_config.server_ssh_key_path} ubuntu@#{lobot_config.master} -p #{lobot_config.ssh_port}")
     end
 
     desc "open", "Open a browser to Lobot"
     def open
-      exec("open https://#{lobot_config.node_attributes.nginx.basic_auth_user}:#{lobot_config.node_attributes.nginx.basic_auth_password}@#{lobot_config.master}/")
+      exec("open #{lobot_config.jenkins_url}/")
     end
 
     desc "create", "Create a new Lobot server using EC2"
     def create
-      ssh_key_path = File.expand_path("#{lobot_config.server_ssh_key}.pub")
-
-      amazon.add_key_pair("lobot", ssh_key_path)
-      amazon.create_security_group("lobot")
-      amazon.open_port("lobot", 22, 443)
-      server = amazon.launch_server("lobot", "lobot", lobot_config.instance_size)
+      server = amazon.with_key_pair(lobot_config.server_ssh_pubkey) do |keypair_name|
+        amazon.create_security_group("lobot")
+        amazon.open_port("lobot", 22, 443)
+        amazon.launch_server(keypair_name, "lobot", lobot_config.instance_size)
+      end
+      wait_for_server(server)
 
-      puts "Writing ip address for ec2: #{server.public_ip_address}"
-      lobot_config.master = server.public_ip_address
-      lobot_config.instance_id = server.id
-      lobot_config.save
+      say("Writing ip address for ec2: #{server.public_ip_address}")
 
-      update_known_hosts
+      lobot_config.update(master: server.public_ip_address, instance_id: server.id)
     end
 
-    desc "destroy_ec2", "Destroys all the lobot resources that we can find on ec2.  Be Careful!"
+    desc "destroy_ec2", "Destroys all the lobot resources on EC2"
+    method_option :all, default: false
+    method_option :force, default: false
     def destroy_ec2
-      amazon.destroy_ec2
-      lobot_config.master = nil
+      instance = (options['all'] ? :all : lobot_config.instance_id)
+
+      amazon.destroy_ec2(confirmation_proc(options['force']), instance) do |server|
+        say("Clearing ip address for ec2: #{server.public_ip_address}")
+
+        lobot_config.update(master: nil, instance_id: nil)
+      end
     end
 
-    desc "create_vagrant", "Lowers the price of heroin to reasonable levels"
+    desc "create_vagrant", "Creates a vagrant instance"
     def create_vagrant
-      spawn_env = {"LOBOT_SSH_KEY" => File.expand_path("#{lobot_config.server_ssh_key}.pub"),
+      spawn_env = {"LOBOT_SSH_KEY" => lobot_config.server_ssh_pubkey_path,
                    "VAGRANT_HOME" => File.expand_path("~")}
       spawn_options = {chdir: lobot_root_path}
 
@@ -53,22 +58,25 @@ module Lobot
 
       vagrant_ip = "192.168.33.10"
 
-      puts "Writing ip address for vagrant: #{vagrant_ip}"
-      lobot_config.master = vagrant_ip
-      lobot_config.save
+      say("Writing ip address for vagrant: #{vagrant_ip}")
 
-      update_known_hosts
+      lobot_config.update(master: vagrant_ip)
     end
 
     desc "config", "Dumps all configuration data for Lobot"
     def config
-      pp lobot_config.to_hash
+      say lobot_config.display
+    end
+
+    desc "certificate", "Dump the certificate"
+    def certificate
+      say(keychain.fetch_remote_certificate("https://#{lobot_config.master}"))
     end
 
     desc "bootstrap", "Configures Lobot's master node"
     def bootstrap
       sync_bootstrap_script
-      master_server.exec("bash -l script/bootstrap_server.sh")
+      master_server.system!("bash -l script/bootstrap_server.sh")
     rescue Errno::ECONNRESET
       sleep 1
     end
@@ -78,28 +86,28 @@ module Lobot
       sync_chef_recipes
       upload_soloist
       sync_github_ssh_key
-      master_server.exec("bash -l -c 'rvm use 1.9.3; gem list | grep soloist || gem install --no-ri --no-rdoc soloist; soloist'")
-    rescue Errno::ECONNRESET
+      master_server.upload(File.expand_path('../../../templates/Gemfile-remote', __FILE__), 'Gemfile')
+      master_server.system!("bash -l -c 'rvm use 1.9.3; bundle install; soloist'")    rescue Errno::ECONNRESET
       sleep 1
     end
 
-    desc "add_build(name, repository, branch, command)", "Adds a build to Lobot"
+    desc "add_build <name> <repository> <branch> <command>", "Adds a build to Lobot"
     def add_build(name, repository, branch, command)
-      build = {
-        "name" => name,
-        "repository" => repository,
-        "branch" => branch,
-        "command" => command
-      }
-      lobot_config.node_attributes = lobot_config.node_attributes.tap do |config|
-        config.jenkins.builds << build unless config.jenkins.builds.include?(build)
-      end
+      raise lobot_config.errors.join(" and ") unless lobot_config.valid?
+
+      lobot_config.add_build(name, repository, branch, command)
       lobot_config.save
     end
 
+    desc "trust_certificate", "Adds the current master's certificate to your OSX keychain"
+    def trust_certificate
+      certificate_contents = keychain.fetch_remote_certificate("https://#{lobot_config.master}/")
+      keychain.add_certificate(certificate_contents)
+    end
+
     no_tasks do
       def master_server
-        @master_server ||= Lobot::Sobo.new(lobot_config.master, lobot_config.server_ssh_key)
+        @master_server ||= Lobot::Sobo.new(lobot_config.master, lobot_config.server_ssh_key_path)
       end
 
       def lobot_config
@@ -110,16 +118,21 @@ module Lobot
         @amazon ||= Lobot::Amazon.new(lobot_config.aws_key, lobot_config.aws_secret)
       end
 
+      def keychain
+        @keychain ||= Lobot::Keychain.new("/Library/Keychains/System.keychain")
+      end
+
       def sync_bootstrap_script
         master_server.upload(File.join(lobot_root_path, "script/"), "script/")
       end
 
       def sync_github_ssh_key
-        master_server.upload(lobot_config.github_ssh_key, "~/.ssh/id_rsa")
+        master_server.upload(lobot_config.github_ssh_key_path, "~/.ssh/id_rsa")
       end
 
       def sync_chef_recipes
         master_server.upload(File.join(lobot_root_path, "chef/"), "chef/")
+        master_server.upload("cookbooks/", "chef/project-cookbooks/")
       end
 
       def upload_soloist
@@ -132,6 +145,7 @@ module Lobot
     end
 
     private
+
     def lobot_root_path
       File.expand_path('../../..', __FILE__)
     end
@@ -140,9 +154,19 @@ module Lobot
       File.expand_path("config/lobot.yml", Dir.pwd)
     end
 
-    def update_known_hosts
-      raise "failed to remove old host key from known_hosts" unless system "ssh-keygen -R #{lobot_config.master} 2> /dev/null"
-      raise "failed to add host key to known_hosts" unless system "ssh-keyscan #{lobot_config.master} 2> /dev/null >> ~/.ssh/known_hosts"
+    def wait_for_server(server)
+      Godot.new(server.public_ip_address, 22, :timeout => 180).wait!
+    end
+
+    # The proc is given a Fog server object and must return true/false
+    def confirmation_proc(force)
+      if force
+        ->(_) { true }
+      else
+        ->(server) {
+          yes?("DESTROY #{server.id} (#{server.public_ip_address})?")
+        }
+      end
     end
   end
 end