lobot 0.9.0

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+features/config/secrets.yml
+.idea
+spec/tmp

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2-p180@lobot

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+gem "rspec"
+gem "generator_spec"
+gem "rails"
+gem "cucumber"
+
+# Specify your gem's dependencies in lobot.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Pivotal Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+Lobot: Your Chief Administrative Aide on Cloud City
+============================
+
+![Lobot](http://i.imgur.com/QAkd7.jpg)
+###A "one click" solution for deploying CI to EC2
+
+Lando Calrissian relies on Lobot to keep Cloud City afloat, and now you can rely on Lobot to keep your continuous integration server running in the cloud. Lobot is a gem that will help you spin-up, bootstrap, and install Jenkins for CI for your Rails app on Amazon EC2.
+
+# What do I get?
+
+* rake tasks for starting a CI instance
+* capistrano tasks for bootstrapping and deploying to an EC2 instance
+* chef recipes for configuring a Centos server to run Jenkins and build Rails projects.
+
+all you'll need to do is:
+
+    rails g lobot:install
+    edit config/ci.yml
+    rake ci:server_start && cap ci bootstrap && cap ci chef
+
+## Install
+
+Add lobot to your Gemfile
+
+    gem "lobot"
+
+## Generate
+Lobot is a Rails 3 generator.  Rails 2 can be made to work, but you will need to copy the template files into your project.
+
+    rails g lobot:install
+
+## Setup
+
+Edit config/ci.yml
+
+    --- 
+    app_name: # a short name for your application
+    app_user: # the user created to run your CI process
+    git_location: # The location of your remote git repository which Jenkins will poll and pull from on changes.
+    basic_auth: 
+    - username: # The username you will use to access the Jenkins web interface
+      password: # The password you will use to access the Jenkins web interface
+    credentials: 
+      aws_access_key_id: # The Access Key for your Amazon AWS account
+      aws_secret_access_key: The Secret Access Key for your Amazon AWS account
+      provider: AWS # leave this one alone
+    server: 
+      name: run 'rake ci:server_start to populate'
+      instance_id: run 'rake ci:server_start to populate'
+    build_command: ./cruise_build.sh
+    ec2_server_access: 
+      key_pair_name: myapp_ci
+      id_rsa_path: ~/.ssh/id_rsa
+    id_rsa_for_github_access: |-
+      -----BEGIN RSA PRIVATE KEY-----
+      SSH KEY WITH ACCESS TO GITHUB GOES HERE
+      -----END RSA PRIVATE KEY-----
+      
+For security, you can add ci.yml to your .gitignore file and store a ci.yml.example without authentication credentials in your repository
+
+## Dependencies
+
+* fog
+* capistrano
+* capistrano-ext
+* rvm (the gem - it configures capistrano to use RVM on the server)
+
+# Tests
+
+Lobot is tested using rspec, generator_spec and cucumber.  Cucumber provides a full integration test which can generate a rails application, push it to github, start a server and bring up CI for the generated project.  You'll need a git repository(which should not have any code you care about) and an AWS account to run the spec.  It costs about $0.50 and takes about half an hour.  It does not clean up after itself, so be sure to terminate the server when you're done, or it will cost substantially more than $0.50.  Use the secrets.yml.example to create a secrets.yml file with your accounts.
+
+# Contributing
+
+Lobot is in its infancy and we welcome pull requests.  Pull requests should have test coverage for quick consideration.
+
+# License
+
+Lobot is MIT Licensed and © Pivotal Labs.  See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/features/ci.feature

@@ -0,0 +1,18 @@
+Feature: CI 
+
+  Background:
+    Given the temp directory is clean
+    Given I am in the temp directory
+  
+  Scenario: Install CI on Amazon AWS using new Rails template
+    When I create a new Rails project using a Rails template
+    And I vendor Lobot
+    And I put Lobot in the Gemfile
+    And I run bundle install
+    And I run the Lobot generator
+    And I enter my info into the ci.yml file
+    And I push to git
+    And I run the server setup
+    And I bootstrap
+    And I deploy
+    Then CI is green

data/features/config/secrets.yml.example

@@ -0,0 +1,6 @@
+--- 
+aws_access_key_id: 
+aws_secret_access_key: 
+id_rsa: |-
+  -----BEGIN RSA PRIVATE KEY-----
+  -----END RSA PRIVATE KEY-----

data/features/step_definitions/ci_steps.rb

@@ -0,0 +1,98 @@
+
+def system!(str)
+  raise "Command Failed: #{str}" unless system(str)
+end
+
+Given /^the temp directory is clean$/ do
+  system!("rm -rf /tmp/lobot-test")
+  system!("mkdir -p /tmp/lobot-test")
+end
+
+Given /^I am in the temp directory$/ do
+  Dir.chdir('/tmp/lobot-test')
+end
+
+When /^I create a new Rails project using a Rails template$/ do
+  system!("echo -e '\nyes\nno\nno\nno\nno\nno\nno' | rails new testapp -m https://github.com/pivotal/guiderails/raw/master/main.rb")
+end
+
+When /^I vendor Lobot$/ do
+  lobot_dir = File.expand_path('../../', File.dirname(__FILE__))
+  system! "cd #{lobot_dir} && rake build"
+  system! "mkdir -p testapp/vendor/cache/"
+  system! "cp #{lobot_dir}/pkg/lobot-#{Lobot::VERSION}.gem testapp/vendor/cache/"
+end
+
+When /^I put Lobot in the Gemfile$/ do
+  lobot_path = File.expand_path('../../', File.dirname(__FILE__))
+  system!(%{echo "gem 'lobot'" >> testapp/Gemfile})
+end
+
+When /^I run bundle install$/ do
+  system("cd testapp && gem uninstall lobot")
+  system!("cd testapp && bundle install")
+  system!('cd testapp && bundle exec gem list | grep lobot')
+end
+
+When /^I run the Lobot generator$/ do
+  system!('cd testapp && rails generate lobot:install')
+  system!('ls testapp | grep -s soloistrc')
+end
+
+When /^I enter my info into the ci\.yml file$/ do
+  secrets = YAML.load_file(File.expand_path('../config/secrets.yml', File.dirname(__FILE__)))
+  
+  ci_conf_location = 'testapp/config/ci.yml'
+  ci_yml = YAML.load_file(ci_conf_location)
+  ci_yml.merge!(
+  'app_name' => 'testapp',
+  'app_user' => 'testapp-user',
+  'git_location' => 'git@github.com:pivotalprivate/ci-smoke.git',
+  'basic_auth' => [{ 'username' => 'testapp', 'password' => 'testpass' }],
+  'credentials' => { 'aws_access_key_id' => secrets['aws_access_key_id'], 'aws_secret_access_key' => secrets['aws_secret_access_key'], 'provider' => 'AWS' },
+  'ec2_server_access' => {'key_pair_name' => 'lobot_cucumber_key_pair', 'id_rsa_path' => '~/.ssh/id_rsa'},
+  'id_rsa_for_github_access' => secrets['id_rsa_for_github_access']
+  )
+  # ci_yml['server']['name]  = '' # This can be used to merge in a server which is already running if you want to skip the setup steps while iterating on a test
+  File.open(ci_conf_location, "w") do |f|
+    YAML.dump(ci_yml, f)
+  end
+end
+
+When /^I push to git$/ do
+  system! "echo 'config/ci.yml' >> testapp/.gitignore"
+  system! "cd testapp && git add ."
+  system! "cd testapp && git commit -m'initial commit'"
+  system "cd testapp && git remote rm origin"
+  system! "cd testapp && git remote add origin git@github.com:pivotalprivate/ci-smoke.git"
+  system! "cd testapp && git push --force -u origin master"
+end
+
+When /^I run the server setup$/ do
+  system! "cd testapp && rake ci:server_start"
+end
+
+When /^I bootstrap$/ do
+  server_is_available = false
+  iterations = 0
+  until server_is_available
+    server_is_available = system("cd testapp && cap ci check_for_server_availability")
+    puts "Sleeping for 3 seconds"
+    sleep 3
+    iterations += 1
+    raise "server is not available" if iterations > 10
+  end
+  system! "cd testapp && cap ci bootstrap"
+end
+
+When /^I deploy$/ do
+  system! "cd testapp && cap ci chef"
+end
+
+Then /^CI is green$/ do
+  Timeout::timeout(300) do
+    until system("cd testapp && rake ci:status")
+      sleep 5
+    end
+  end
+end

data/features/support/env.rb

@@ -0,0 +1,3 @@
+require "rubygems"
+
+require File.expand_path('../../lib/lobot', File.dirname(__FILE__))

data/lib/generators/lobot/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Explain the generator
+
+Example:
+    rails generate install Thing
+
+    This will create:
+        what/will/it/create

data/lib/generators/lobot/install_generator.rb

@@ -0,0 +1,24 @@
+module Lobot
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path(File.join(File.dirname(__FILE__), 'templates'))
+    
+    def create_ci_files
+      template 'ci.yml', 'config/ci.yml'
+      template 'ci.rake', 'lib/tasks/ci.rake'
+      template 'bootstrap_server.sh', 'script/bootstrap_server.sh'
+      template 'deploy-ci.rb', 'config/deploy/ci.rb'
+      template 'capistrano-ci.rb', 'config/capistrano/ci.rb'
+      template 'soloistrc', 'soloistrc'
+    end
+    
+    def add_load_path_to_capfile
+      template 'Capfile', 'Capfile' unless File.exists?("#{destination_root}/Capfile")
+      prepend_to_file 'Capfile', "load 'config/capistrano/ci'\n"
+    end
+    
+    def create_chef_cookbooks
+      directory 'chef', 'chef'
+    end
+    
+  end
+end

data/lib/generators/lobot/templates/Capfile

@@ -0,0 +1,6 @@
+# Lobot provides the default Capfile for you in case you want to use more cap later
+
+# load 'deploy' if respond_to?(:namespace) # cap2 differentiator
+# Dir['vendor/gems/*/recipes/*.rb','vendor/plugins/*/recipes/*.rb'].each { |plugin| load(plugin) }
+# 
+# load 'config/deploy' # remove this line to skip loading any of the default tasks

data/lib/generators/lobot/templates/bootstrap_server.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -e
+
+env | grep -q "APP_USER=" || echo "Please set APP_USER environment variable"
+
+# perl -e 'print crypt("password", "salt"),"\n"'
+getent passwd $APP_USER >/dev/null 2>&1 || useradd $APP_USER -p DEADBEEFSCRYPTEDPASSWORD #sa3tHJ3/KuYvI would be password
+
+# copy root's authorized keys to APP_USER
+mkdir -p  /home/$APP_USER/.ssh
+touch /home/$APP_USER/.ssh/authorized_keys
+chmod 700 /home/$APP_USER/.ssh
+chmod 600 /home/$APP_USER/.ssh/authorized_keys
+chown -R $APP_USER /home/$APP_USER/.ssh
+
+authorized_keys_string=`cat /root/.ssh/authorized_keys`
+grep -sq "$authorized_keys_string" /home/$APP_USER/.ssh/authorized_keys || cat /root/.ssh/authorized_keys >> /home/$APP_USER/.ssh/authorized_keys
+
+
+## enable ssh password auth
+perl -p -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config 
+/etc/init.d/sshd reload
+
+# install epel
+rpm -q epel-release-5-4.noarch || rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
+
+# install git
+yum -y install git
+
+# rvm prereqs
+yum install -y gcc-c++ patch readline readline-devel zlib zlib-devel libffi-devel openssl-devel iconv-devel java
+
+# passwordless sudo
+sudo_string='ALL            ALL = (ALL) NOPASSWD: ALL'
+grep "$sudo_string" /etc/sudoers || echo "$sudo_string" >> /etc/sudoers
+
+cat <<'BOOTSTRAP_AS_USER' > /home/$APP_USER/bootstrap_as_user.sh
+set -e
+
+export APP_USER=$1
+
+mkdir -p /home/$APP_USER/rvm/src
+curl -Lskf http://github.com/wayneeseguin/rvm/tarball/156d0b42feba4922ad04 | tar xvz -C/home/$APP_USER/rvm/src --strip 1
+cd "/home/$APP_USER/rvm/src" && ./install
+
+rvm_include_string='[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"'
+grep "$rvm_include_string" ~/.bashrc || echo "$rvm_include_string" >> ~/.bashrc
+
+cat <<'RVMRC_CONTENTS' > ~/.rvmrc
+rvm_install_on_use_flag=1
+rvm_trust_rvmrcs_flag=1
+rvm_gemset_create_on_use_flag=1
+RVMRC_CONTENTS
+BOOTSTRAP_AS_USER
+
+chmod a+x /home/$APP_USER/bootstrap_as_user.sh
+su - $APP_USER /home/$APP_USER/bootstrap_as_user.sh $APP_USER
+rm /home/$APP_USER/bootstrap_as_user.sh

data/lib/generators/lobot/templates/capistrano-ci.rb

@@ -0,0 +1,68 @@
+require 'capistrano/ext/multistage'
+require "rvm/capistrano"  # Use the gem, don't unshift RVM onto the load path, that's crazy.
+
+task :ci_setup do
+  ci_conf_location ||= File.expand_path('../../../config/ci.yml', __FILE__)
+  ci_conf ||= YAML.load_file(ci_conf_location)
+  
+  raise "rvm_ruby_string not set - are you using RVM" unless ENV['rvm_ruby_string']
+  set :rvm_ruby_string, ENV['rvm_ruby_string']
+  set :rvm_type, :user
+  set :app_name, ci_conf['app_name']
+  set(:app_dir) { "/var/#{stage}/#{app_name}" }
+  set :user, ci_conf['app_user']
+  ssh_options[:keys] = [ci_conf['ec2_server_access']['id_rsa_path']]
+  default_run_options[:pty] = true
+end
+
+desc "check for server availability - run this task in a loop to see if the slice is ready to connect to"
+task :check_for_server_availability do
+  ci_setup
+  set :user, "root"
+  run "echo Server is available"
+end
+
+desc "bootstrap"
+task :bootstrap do
+  ci_setup
+  app_user = user
+  set :user, "root"
+  set :default_shell, "/bin/bash"
+  upload "script/bootstrap_server.sh", "/root/bootstrap_server.sh"
+  run "chmod a+x /root/bootstrap_server.sh"
+  run "APP_USER=#{app_user} /root/bootstrap_server.sh"
+end
+
+desc "setup and run chef"
+task :chef do
+  ci_setup
+  install_base_gems
+  upload_cookbooks
+  run_soloist
+end
+
+desc "Install gems that are needed for a chef run"
+task :install_base_gems do
+  ci_setup
+  p rvm_ruby_string
+  run "gem list | grep soloist || gem install soloist --no-rdoc --no-ri"
+  run "gem list | grep bundler || gem install bundler --no-rdoc --no-ri"
+end
+
+desc "Upload cookbooks"
+task :upload_cookbooks do
+  ci_setup
+  run "sudo mkdir -p #{app_dir}"
+  run "sudo chown -R #{user} #{app_dir}"
+  run "rm #{app_dir}/soloistrc || true"
+  run "rm -r #{app_dir}/chef || true"
+  upload("soloistrc", "#{app_dir}/soloistrc")
+  upload("config/ci.yml", "#{app_dir}/ci.yml")
+  upload("chef/", "#{app_dir}/chef/", :via => :scp, :recursive => true)
+end
+
+desc "Run soloist"
+task :run_soloist do
+  ci_setup
+  run "cd #{app_dir} && PATH=/usr/sbin:$PATH APP_NAME=#{fetch(:app_name)} APP_DIR=#{fetch(:app_dir)} LOG_LEVEL=debug soloist"
+end

data/lib/generators/lobot/templates/chef/cookbooks/pivotal_ci/attributes/nginx.rb

@@ -0,0 +1,2 @@
+node["nginx_settings"] ||= {}
+node["nginx_settings"]["basic_auth_users"] = CI_CONFIG['basic_auth']

data/lib/generators/lobot/templates/chef/cookbooks/pivotal_ci/libraries/ci_config.rb

@@ -0,0 +1,2 @@
+ci_conf_location = File.expand_path('../../../../../ci.yml', __FILE__)
+::CI_CONFIG = YAML.load_file(ci_conf_location)

data/lib/generators/lobot/templates/chef/cookbooks/pivotal_ci/recipes/default.rb

@@ -0,0 +1,7 @@
+include_recipe "pivotal_server::imagemagick"
+include_recipe "pivotal_server::postgres"
+include_recipe "pivotal_server::sqlite"
+include_recipe "pivotal_server::libxml_prereqs"
+include_recipe "pivotal_server::nginx"
+include_recipe "pivotal_ci::id_rsa"
+include_recipe "pivotal_ci::jenkins"