lobot 1.0.pre → 2.0.0pre

data/chef/cookbooks/pivotal_ci/templates/default/nginx-htaccess.erb

@@ -1,3 +1,3 @@
-<% node["nginx_settings"]["basic_auth_users"].each do |user| %>
-<%= "#{user["username"]}:#{user["password"].crypt("AA")}" %>
-<% end %>
+<% if node['nginx']['basic_auth_user'] && node['nginx']['basic_auth_password'] %>
+<%= "#{node['nginx']['basic_auth_user']}:#{node['nginx']['basic_auth_password'].crypt("AA")}" %>
+<% end %>

data/chef/cookbooks/pivotal_ci/templates/default/org.jenkinsci.plugins.xvfb.XvfbBuildWrapper.xml.erb

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<org.jenkinsci.plugins.xvfb.XvfbBuildWrapper_-XvfbBuildWrapperDescriptor plugin="xvfb@1.0.4">
+  <helpRedirect/>
+  <installations>
+    <org.jenkinsci.plugins.xvfb.XvfbInstallation>
+      <name>system</name>
+      <home>/usr/bin</home>
+      <properties/>
+    </org.jenkinsci.plugins.xvfb.XvfbInstallation>
+  </installations>
+</org.jenkinsci.plugins.xvfb.XvfbBuildWrapper_-XvfbBuildWrapperDescriptor>

data/chef/cookbooks/pivotal_ci/test/kitchen/Kitchenfile

@@ -0,0 +1,4 @@
+cookbook "pivotal_ci" do
+  configuration "default"
+  lint(:ignore => ["FC007", "FC009"])
+end

data/chef/cookbooks/pivotal_ci/test/kitchen/cookbooks/pivotal_ci_test/attributes/default.rb

@@ -0,0 +1,3 @@
+node['nginx']['basic_auth_user'] = "wheelbarrow"
+node['nginx']['basic_auth_password'] = "full-of-garbage"
+node['jenkins']['builds'] = []

data/chef/cookbooks/pivotal_ci/test/kitchen/cookbooks/pivotal_ci_test/metadata.rb

@@ -0,0 +1,10 @@
+name              "pivotal_ci_test"
+maintainer        "Pivotal Labs"
+maintainer_email  "commoncode+lobot@pivotallabs.com"
+license           "MIT"
+description       "Sets up Lobot"
+version           "0.1.0"
+
+recipe "pivotal_ci_test::default", "Bootstrap Lobot"
+
+supports "ubuntu", "12.04"

data/chef/cookbooks/pivotal_ci/test/kitchen/cookbooks/pivotal_ci_test/recipes/default.rb

@@ -0,0 +1,41 @@
+bash "bootstrap" do
+  cwd "/home/vagrant"
+  code <<-BOOTSTRAP
+#!/bin/bash
+
+set -e
+
+sudo apt-get update > /dev/null
+
+packages="git build-essential openssl libreadline6 libreadline6-dev libreadline5 curl git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev autoconf libc6-dev libncurses5-dev automake libtool bison subversion pkg-config"
+selections=`dpkg --get-selections`
+
+for package in $packages
+do
+  if ! echo "$selections" | grep "^$package\s" > /dev/null
+  then
+    to_install="$to_install $package"
+  fi
+done
+
+if [ ! -z "$to_install" ]
+then
+  sudo apt-get install -y $to_install
+fi
+
+test -d /usr/local/rvm || curl -L https://get.rvm.io | sudo bash -s stable
+
+sudo tee /etc/profile.d/rvm.sh > /dev/null <<RVMSH_CONTENT
+[[ -s "/usr/local/rvm/scripts/rvm" ]] && source "/usr/local/rvm/scripts/rvm"
+RVMSH_CONTENT
+
+sudo tee /etc/rvmrc > /dev/null <<RVMRC_CONTENTS
+rvm_install_on_use_flag=1
+rvm_trust_rvmrcs_flag=1
+rvm_gemset_create_on_use_flag=1
+RVMRC_CONTENTS
+
+sudo usermod vagrant -a -G rvm
+
+BOOTSTRAP
+end

data/lib/generators/lobot/templates/soloistrc

@@ -18,5 +18,6 @@ env_variable_switches:
 node_attributes:
   travis_build_environment:
     user: jenkins
-    group: jenkins
+    group: nogroup
     home: /var/lib/jenkins
+

data/lib/lobot.rb

@@ -1,15 +1,4 @@
-require File.expand_path('lobot/version', File.dirname(__FILE__))
-require 'headless'
+require File.expand_path("../lobot/version", __FILE__)
 
 module Lobot
-  def self.rails3?
-    return Rails.version.split(".").first.to_i == 3 if defined? Rails
-    begin
-      Gem::Specification::find_by_name "rails", ">= 3.0"
-    rescue
-      Gem.available? "rails", ">= 3.0"
-    end
-  end
 end
-
-require File.expand_path(File.join('lobot', "railtie"), File.dirname(__FILE__)) if Lobot.rails3?

data/lib/lobot/amazon.rb

@@ -0,0 +1,91 @@
+require "fog"
+
+module Lobot
+  class Amazon
+    PORTS_TO_OPEN = [22, 443] + (9000...9010).to_a
+
+    attr_reader :key, :secret
+
+    def initialize(key, secret)
+      @key = key
+      @secret = secret
+    end
+
+    def security_groups
+      fog.security_groups
+    end
+
+    def key_pairs
+      fog.key_pairs
+    end
+
+    def servers
+      fog.servers
+    end
+
+    def elastic_ip_address
+      @elastic_ip_address ||= fog.addresses.create
+    end
+
+    def create_security_group(group_name)
+      unless security_groups.get(group_name)
+        security_groups.create(:name => group_name, :description => 'Lobot-generated group')
+      end
+    end
+
+    def open_port(group_name, *ports)
+      group = security_groups.get(group_name)
+      ports.each do |port|
+        unless group.ip_permissions.any? { |p| (p["fromPort"]..p["toPort"]).include?(port) }
+          group.authorize_port_range(port..port)
+        end
+      end
+    end
+
+    def delete_key_pair(key_pair_name)
+      key_pairs.new(:name => key_pair_name).destroy
+    end
+
+    def add_key_pair(key_pair_name, key_path)
+      unless key_pairs.get(key_pair_name)
+        key_pairs.create(:name => key_pair_name, :public_key => File.read("#{key_path}"))
+      end
+    end
+
+    def launch_server(key_pair_name, security_group_name, instance_type = "m1.medium")
+      servers.create(
+        :image_id => "ami-a29943cb",
+        :flavor_id => instance_type,
+        :availability_zone => "us-east-1a",
+        :tags => {"Name" => "Lobot", "lobot" => Lobot::VERSION},
+        :key_name => key_pair_name,
+        :groups => [security_group_name]
+      ).tap do |server|
+        server.wait_for { ready? }
+        fog.associate_address(server.id, elastic_ip_address.public_ip) # needs to be running
+        server.reload
+      end
+    end
+
+    def destroy_ec2
+      servers = fog.servers.select { |s| s.tags.keys.include?("lobot") && s.state == "running" }
+      ips = servers.map(&:public_ip_address)
+      servers.map(&:destroy)
+      ips.each { |ip| release_elastic_ip(ip) }
+    end
+
+    def release_elastic_ip(ip)
+      fog.addresses.get(ip).destroy if fog.addresses.get(ip)
+    end
+
+    private
+
+    def fog
+      @fog ||= Fog::Compute.new(
+        :provider => "aws",
+        :aws_access_key_id => key,
+        :aws_secret_access_key => secret
+      )
+    end
+  end
+end

data/lib/lobot/cli.rb

@@ -0,0 +1,148 @@
+require "thor"
+require "lobot/config"
+require "lobot/sobo"
+require "lobot/amazon"
+require "lobot/jenkins"
+require "pp"
+require "tempfile"
+require "json"
+
+module Lobot
+  class CLI < Thor
+    desc "ssh", "SSH into Lobot"
+    def ssh
+      exec("ssh -i #{lobot_config.server_ssh_key} ubuntu@#{lobot_config.master} -p #{lobot_config.ssh_port}")
+    end
+
+    desc "open", "Open a browser to Lobot"
+    def open
+      exec("open https://#{lobot_config.node_attributes.nginx.basic_auth_user}:#{lobot_config.node_attributes.nginx.basic_auth_password}@#{lobot_config.master}/")
+    end
+
+    desc "create", "Create a new Lobot server using EC2"
+    def create
+      ssh_key_path = File.expand_path("#{lobot_config.server_ssh_key}.pub")
+
+      amazon.add_key_pair("lobot", ssh_key_path)
+      amazon.create_security_group("lobot")
+      amazon.open_port("lobot", 22, 443)
+      server = amazon.launch_server("lobot", "lobot", lobot_config.instance_size)
+
+      puts "Writing ip address for ec2: #{server.public_ip_address}"
+      lobot_config.master = server.public_ip_address
+      lobot_config.instance_id = server.id
+      lobot_config.save
+
+      update_known_hosts
+    end
+
+    desc "destroy_ec2", "Destroys all the lobot resources that we can find on ec2.  Be Careful!"
+    def destroy_ec2
+      amazon.destroy_ec2
+      lobot_config.master = nil
+    end
+
+    desc "create_vagrant", "Lowers the price of heroin to reasonable levels"
+    def create_vagrant
+      spawn_env = {"LOBOT_SSH_KEY" => File.expand_path("#{lobot_config.server_ssh_key}.pub"),
+                   "VAGRANT_HOME" => File.expand_path("~")}
+      spawn_options = {chdir: lobot_root_path}
+
+      pid = Process.spawn(spawn_env, "vagrant up", spawn_options)
+      Process.wait(pid)
+
+      vagrant_ip = "192.168.33.10"
+
+      puts "Writing ip address for vagrant: #{vagrant_ip}"
+      lobot_config.master = vagrant_ip
+      lobot_config.save
+
+      update_known_hosts
+    end
+
+    desc "config", "Dumps all configuration data for Lobot"
+    def config
+      pp lobot_config.to_hash
+    end
+
+    desc "bootstrap", "Configures Lobot's master node"
+    def bootstrap
+      sync_bootstrap_script
+      master_server.exec("bash -l script/bootstrap_server.sh")
+    rescue Errno::ECONNRESET
+      sleep 1
+    end
+
+    desc "chef", "Uploads chef recipes and runs them"
+    def chef
+      sync_chef_recipes
+      upload_soloist
+      sync_github_ssh_key
+      master_server.exec("bash -l -c 'rvm use 1.9.3; gem list | grep soloist || gem install --no-ri --no-rdoc soloist; soloist'")
+    rescue Errno::ECONNRESET
+      sleep 1
+    end
+
+    desc "add_build(name, repository, branch, command)", "Adds a build to Lobot"
+    def add_build(name, repository, branch, command)
+      build = {
+        "name" => name,
+        "repository" => repository,
+        "branch" => branch,
+        "command" => command
+      }
+      lobot_config.node_attributes = lobot_config.node_attributes.tap do |config|
+        config.jenkins.builds << build unless config.jenkins.builds.include?(build)
+      end
+      lobot_config.save
+    end
+
+    no_tasks do
+      def master_server
+        @master_server ||= Lobot::Sobo.new(lobot_config.master, lobot_config.server_ssh_key)
+      end
+
+      def lobot_config
+        @lobot_config ||= Lobot::Config.from_file(lobot_config_path)
+      end
+
+      def amazon
+        @amazon ||= Lobot::Amazon.new(lobot_config.aws_key, lobot_config.aws_secret)
+      end
+
+      def sync_bootstrap_script
+        master_server.upload(File.join(lobot_root_path, "script/"), "script/")
+      end
+
+      def sync_github_ssh_key
+        master_server.upload(lobot_config.github_ssh_key, "~/.ssh/id_rsa")
+      end
+
+      def sync_chef_recipes
+        master_server.upload(File.join(lobot_root_path, "chef/"), "chef/")
+      end
+
+      def upload_soloist
+        Tempfile.open("lobot-soloistrc") do |file|
+          file.write(YAML.dump(JSON.parse(JSON.dump(lobot_config.soloistrc))))
+          file.close
+          master_server.upload(file.path, "soloistrc")
+        end
+      end
+    end
+
+    private
+    def lobot_root_path
+      File.expand_path('../../..', __FILE__)
+    end
+
+    def lobot_config_path
+      File.expand_path("config/lobot.yml", Dir.pwd)
+    end
+
+    def update_known_hosts
+      raise "failed to remove old host key from known_hosts" unless system "ssh-keygen -R #{lobot_config.master} 2> /dev/null"
+      raise "failed to add host key to known_hosts" unless system "ssh-keyscan #{lobot_config.master} 2> /dev/null >> ~/.ssh/known_hosts"
+    end
+  end
+end

data/lib/lobot/config.rb

@@ -0,0 +1,80 @@
+require "hashie"
+
+module Lobot
+  class Config < Hashie::Dash
+    property :path
+    property :aws_key
+    property :aws_secret
+    property :instance_id
+    property :master
+
+    property :instance_size, :default => 'c1.medium'
+    property :ssh_port, :default => 22
+    property :server_ssh_key, :default => File.expand_path("~/.ssh/id_rsa")
+    property :github_ssh_key, :default => File.expand_path("~/.ssh/id_rsa")
+    property :recipes, :default => ["pivotal_ci::jenkins", "pivotal_ci::limited_travis_ci_environment", "pivotal_ci"]
+    property :cookbook_paths, :default => ['./chef/cookbooks/', './chef/travis-cookbooks/ci_environment']
+    property :node_attributes, :default => {
+      :travis_build_environment => {
+        :user => "jenkins",
+        :group => "nogroup",
+        :home => "/var/lib/jenkins"
+      },
+      :nginx => {
+        :basic_auth_user => "ci",
+      },
+      :jenkins => {
+        :builds => []
+      }
+    }
+
+    def initialize(attributes = {})
+      super
+      self["node_attributes"] = Hashie::Mash.new(node_attributes)
+    end
+
+    def node_attributes=(attributes)
+      self["node_attributes"] = Hashie::Mash.new(attributes)
+    end
+
+    def soloistrc
+      {
+        "recipes" => recipes,
+        "cookbook_paths" => cookbook_paths,
+        "node_attributes" => node_attributes.to_hash
+      }
+    end
+
+    def save
+      return self unless path
+      File.open(path, "w+") { |file| file.write(YAML.dump(JSON.parse(JSON.dump(to_hash)))) }
+      self
+    end
+
+    def reload
+      self.class.from_file(path)
+    end
+
+    def to_hash
+      hash = super
+      hash.delete("path")
+      {
+        "ssh_port" => ssh_port,
+        "master" => master,
+        "server_ssh_key" => server_ssh_key,
+        "recipes" => recipes,
+        "cookbook_paths" => cookbook_paths,
+        "node_attributes" => node_attributes
+      }.merge(hash)
+    end
+
+    def self.from_file(yaml_file)
+      config = read_config(yaml_file)
+      self.new(config.merge(:path => yaml_file))
+    end
+
+    def self.read_config(yaml_file)
+      File.open(yaml_file, "r") { |file| YAML.load(file.read) }
+    end
+  end
+end

data/lib/lobot/jenkins.rb

@@ -0,0 +1,17 @@
+module Lobot
+  class Jenkins
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def jobs
+      Hashie::Mash.new(api_json).jobs
+    end
+
+    def api_json
+      JSON.parse(`curl -s http://#{config.master}:8080/api/json`)
+    end
+  end
+end