ciborg 0.0.1 → 3.0.0

data/jenkins_plugin/src/main/java/com/pivotallabs/CiborgDecorator.java

@@ -0,0 +1,22 @@
+package com.pivotallabs;
+
+import hudson.Extension;
+import hudson.model.PageDecorator;
+import net.sf.json.JSONObject;
+
+import org.kohsuke.stapler.StaplerRequest;
+
+@Extension
+public class CiborgDecorator extends PageDecorator {
+  public CiborgDecorator() {
+    super(CiborgDecorator.class);
+    load();
+  }
+
+  @Override
+  public boolean configure(StaplerRequest req, JSONObject formData)
+  throws FormException {
+    save();
+    return super.configure(req, formData);
+  }
+}

data/jenkins_plugin/src/main/resources/com/pivotallabs/CiborgDecorator/header.jelly

@@ -0,0 +1,3 @@
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
+  <link type="text/css" rel="stylesheet" href="/plugin/jenkins-ciborg-plugin/ciborg.css" />
+</j:jelly>

data/jenkins_plugin/src/main/resources/index.jelly

@@ -0,0 +1,6 @@
+<!--
+  This view is used to render the installed plugins page.
+-->
+<div>
+  Ciborg is your administrative aide in Cloud City.
+</div>

data/jenkins_plugin/src/main/webapp/ciborg.css

@@ -0,0 +1,5 @@
+#main-table {
+  background-repeat: no-repeat !important;
+  background-position: 10px 100% !important;
+  background-image: url(/plugin/jenkins-ciborg-plugin/ciborg.png) !important;
+}

data/lib/ciborg.rb

@@ -1,5 +1,8 @@
 require "ciborg/version"
-
-module Ciborg
-  # Your code goes here...
-end
+require "ciborg/config"
+require "ciborg/sobo"
+require "ciborg/amazon"
+require "ciborg/keychain"
+require "ciborg/configuration_wizard"
+require "ciborg/cli"
+require "ciborg/password"

data/lib/ciborg/amazon.rb

@@ -0,0 +1,105 @@
+require "fog"
+
+module Ciborg
+  class Amazon
+    PORTS_TO_OPEN = [22, 443] + (9000...9010).to_a
+
+    attr_reader :key, :secret
+
+    def initialize(key, secret)
+      @key = key
+      @secret = secret
+    end
+
+    def fog_security_groups
+      fog.security_groups
+    end
+
+    def fog_key_pairs
+      fog.key_pairs
+    end
+
+    def fog_servers
+      fog.servers
+    end
+
+    def elastic_ip_address
+      @elastic_ip_address ||= fog.addresses.create
+    end
+
+    def create_security_group(group_name)
+      unless fog_security_groups.get(group_name)
+        fog_security_groups.create(:name => group_name, :description => 'Ciborg-generated group')
+      end
+    end
+
+    def open_port(group_name, *ports)
+      group = fog_security_groups.get(group_name)
+      ports.each do |port|
+        unless group.ip_permissions.any? { |p| (p["fromPort"]..p["toPort"]).include?(port) }
+          group.authorize_port_range(port..port)
+        end
+      end
+    end
+
+    def delete_key_pair(key_pair_name)
+      fog_key_pairs.new(:name => key_pair_name).destroy
+    end
+
+    def add_key_pair(key_pair_name, public_key)
+      fog_key_pairs.create(:name => key_pair_name, :public_key => public_key) unless fog_key_pairs.get(key_pair_name)
+    end
+
+    def with_key_pair(pubkey)
+      unique_key_pair_name = "CIBORG-#{Time.now.to_i}"
+      add_key_pair(unique_key_pair_name, pubkey)
+      yield unique_key_pair_name if block_given?
+    ensure
+      delete_key_pair(unique_key_pair_name)
+    end
+
+    def launch_server(key_pair_name, security_group_name, instance_type = "m1.medium", availability_zone = "us-east-1b")
+      fog_servers.create(
+        :image_id => "ami-a29943cb",
+        :flavor_id => instance_type,
+        :availability_zone => availability_zone,
+        :tags => {"Name" => "Ciborg", "ciborg" => Ciborg::VERSION},
+        :key_name => key_pair_name,
+        :groups => [security_group_name]
+      ).tap do |server|
+        server.wait_for { ready? }
+        fog.associate_address(server.id, elastic_ip_address.public_ip) # needs to be running
+        server.reload
+      end
+    end
+
+    def destroy_ec2(confirm_proc, *args)
+      servers.each do |server|
+        next unless (args == [:all]) || args.include?(server.id)
+        next unless confirm_proc.call(server)
+        ip = server.public_ip_address
+        server.destroy
+        release_elastic_ip(ip)
+        yield(server) if block_given?
+      end
+    end
+
+    def servers
+      fog_servers.select { |s| s.tags.keys.include?("ciborg") && s.state == "running" }
+    end
+
+    def release_elastic_ip(ip)
+      fog.addresses.get(ip).destroy if fog.addresses.get(ip)
+    end
+
+    private
+
+    def fog
+      @fog ||= Fog::Compute.new(
+        :provider => "aws",
+        :aws_access_key_id => key,
+        :aws_secret_access_key => secret
+      )
+    end
+  end
+end

data/lib/ciborg/cli.rb

@@ -0,0 +1,172 @@
+require "thor"
+require "thor/group"
+require "pp"
+require "tempfile"
+require "json"
+require "ciborg/configuration_wizard"
+require "godot"
+
+module Ciborg
+  class CLI < ::Thor
+    register(ConfigurationWizard, "setup", "setup", ConfigurationWizard::DESCRIPTION_TEXT)
+
+    desc "ssh", "SSH into Ciborg"
+    def ssh
+      exec("ssh -i #{ciborg_config.server_ssh_key_path} ubuntu@#{ciborg_config.master} -p #{ciborg_config.ssh_port}")
+    end
+
+    desc "open", "Open a browser to Ciborg"
+    def open
+      exec("open #{ciborg_config.jenkins_url}/")
+    end
+
+    desc "create", "Create a new Ciborg server using EC2"
+    def create
+      server = amazon.with_key_pair(ciborg_config.server_ssh_pubkey) do |keypair_name|
+        amazon.create_security_group(ciborg_config.security_group)
+        amazon.open_port(ciborg_config.security_group, 22, 443)
+        amazon.launch_server(keypair_name, ciborg_config.security_group, ciborg_config.instance_size, ciborg_config.availability_zone)
+      end
+      wait_for_server(server)
+
+      say("Writing ip address for ec2: #{server.public_ip_address}")
+
+      ciborg_config.update(master: server.public_ip_address, instance_id: server.id)
+    end
+
+    desc "destroy_ec2", "Destroys all the ciborg resources on EC2"
+    method_option :all, default: false
+    method_option :force, default: false
+    def destroy_ec2
+      instance = (options['all'] ? :all : ciborg_config.instance_id)
+
+      amazon.destroy_ec2(confirmation_proc(options['force']), instance) do |server|
+        say("Clearing ip address for ec2: #{server.public_ip_address}")
+
+        ciborg_config.update(master: nil, instance_id: nil)
+      end
+    end
+
+    desc "create_vagrant", "Creates a vagrant instance"
+    def create_vagrant
+      spawn_env = {"CIBORG_SSH_KEY" => ciborg_config.server_ssh_pubkey_path,
+                   "VAGRANT_HOME" => File.expand_path("~")}
+      spawn_options = {chdir: ciborg_root_path}
+
+      pid = Process.spawn(spawn_env, "vagrant up", spawn_options)
+      Process.wait(pid)
+
+      vagrant_ip = "192.168.33.10"
+
+      say("Writing ip address for vagrant: #{vagrant_ip}")
+
+      ciborg_config.update(master: vagrant_ip)
+    end
+
+    desc "config", "Dumps all configuration data for Ciborg"
+    def config
+      say ciborg_config.display
+    end
+
+    desc "certificate", "Dump the certificate"
+    def certificate
+      say(keychain.fetch_remote_certificate("https://#{ciborg_config.master}"))
+    end
+
+    desc "bootstrap", "Configures Ciborg's master node"
+    def bootstrap
+      sync_bootstrap_script
+      master_server.system!("bash -l script/bootstrap_server.sh")
+    rescue Errno::ECONNRESET
+      sleep 1
+    end
+
+    desc "chef", "Uploads chef recipes and runs them"
+    def chef
+      sync_chef_recipes
+      upload_soloist
+      sync_github_ssh_key
+      master_server.upload(File.expand_path('../../../templates/Gemfile-remote', __FILE__), 'Gemfile')
+      master_server.system!("rvm autolibs enable; bash -l -c 'rvm use 1.9.3; bundle install; soloist'")    rescue Errno::ECONNRESET
+      sleep 1
+    end
+
+    desc "add_build <name> <repository> <branch> <command>", "Adds a build to Ciborg"
+    def add_build(name, repository, branch, command)
+      raise ciborg_config.errors.join(" and ") unless ciborg_config.valid?
+
+      ciborg_config.add_build(name, repository, branch, command)
+      ciborg_config.save
+    end
+
+    desc "trust_certificate", "Adds the current master's certificate to your OSX keychain"
+    def trust_certificate
+      certificate_contents = keychain.fetch_remote_certificate("https://#{ciborg_config.master}/")
+      keychain.add_certificate(certificate_contents)
+    end
+
+    no_tasks do
+      def master_server
+        @master_server ||= Ciborg::Sobo.new(ciborg_config.master, ciborg_config.server_ssh_key_path)
+      end
+
+      def ciborg_config
+        @ciborg_config ||= Ciborg::Config.from_file(ciborg_config_path)
+      end
+
+      def amazon
+        @amazon ||= Ciborg::Amazon.new(ciborg_config.aws_key, ciborg_config.aws_secret)
+      end
+
+      def keychain
+        @keychain ||= Ciborg::Keychain.new("/Library/Keychains/System.keychain")
+      end
+
+      def sync_bootstrap_script
+        master_server.upload(File.join(ciborg_root_path, "script/"), "script/")
+      end
+
+      def sync_github_ssh_key
+        master_server.upload(ciborg_config.github_ssh_key_path, "~/.ssh/id_rsa")
+      end
+
+      def sync_chef_recipes
+        master_server.upload(File.join(ciborg_root_path, "chef/"), "chef/")
+        master_server.upload("cookbooks/", "chef/project-cookbooks/")
+      end
+
+      def upload_soloist
+        Tempfile.open("ciborg-soloistrc") do |file|
+          file.write(YAML.dump(JSON.parse(JSON.dump(ciborg_config.soloistrc))))
+          file.close
+          master_server.upload(file.path, "soloistrc")
+        end
+      end
+    end
+
+    private
+
+    def ciborg_root_path
+      File.expand_path('../../..', __FILE__)
+    end
+
+    def ciborg_config_path
+      File.expand_path("config/ciborg.yml", Dir.pwd)
+    end
+
+    def wait_for_server(server)
+      Godot.new(server.public_ip_address, 22, :timeout => 180).wait!
+    end
+
+    # The proc is given a Fog server object and must return true/false
+    def confirmation_proc(force)
+      if force
+        ->(_) { true }
+      else
+        ->(server) {
+          yes?("DESTROY #{server.id} (#{server.public_ip_address})?")
+        }
+      end
+    end
+  end
+end

data/lib/ciborg/config.rb

@@ -0,0 +1,214 @@
+# encoding: UTF-8
+require "hashie"
+
+module Ciborg
+  class Config < Hashie::Dash
+    property :path
+    property :aws_key
+    property :aws_secret
+    property :instance_id
+    property :master
+
+    property :instance_size, :default => 'c1.medium'
+    property :ssh_port, :default => 22
+    property :server_ssh_key, :default => Proc.new { default_ssh_key }
+    property :github_ssh_key, :default => Proc.new { default_ssh_key }
+    property :recipes, :default => ["pivotal_ci::jenkins", "pivotal_ci::limited_travis_ci_environment", "pivotal_ci"]
+    property :cookbook_paths, :default => ['./chef/cookbooks/', './chef/travis-cookbooks/ci_environment', './chef/project-cookbooks']
+    property :node_attributes, :default => Proc.new { default_node_attributes }
+    property :security_group, :default => 'ciborg'
+    property :availability_zone, :default => "us-east-1b"
+
+    def initialize(attributes = {})
+      super
+      self["node_attributes"] = Hashie::Mash.new(node_attributes)
+    end
+
+    def add_build(name, repository, branch, command)
+      build = {
+        "name" => name,
+        "repository" => repository,
+        "branch" => branch,
+        "command" => command,
+        "junit_publisher" => true
+      }
+      self.node_attributes = self.node_attributes.tap do |config|
+        config.jenkins.builds << build unless config.jenkins.builds.include?(build)
+      end
+    end
+
+    def github_ssh_key_path
+      File.expand_path(self["github_ssh_key"]) if self["github_ssh_key"]
+    end
+
+    def github_ssh_pubkey_path
+      github_ssh_key_path + ".pub" if self["github_ssh_key"]
+    end
+
+    def github_ssh_key
+      File.read(github_ssh_key_path)
+    end
+
+    def github_ssh_pubkey
+      File.read(github_ssh_pubkey_path)
+    end
+
+    def server_ssh_key_path
+      File.expand_path(self["server_ssh_key"]) if self["server_ssh_key"]
+    end
+
+    def server_ssh_pubkey_path
+      server_ssh_key_path + ".pub" if self["server_ssh_key"]
+    end
+
+    def server_ssh_key
+      File.read(server_ssh_key_path)
+    end
+
+    def server_ssh_pubkey
+      File.read(server_ssh_pubkey_path)
+    end
+
+    def master_url
+      "https://#{master}" if master
+    end
+
+    def jenkins_url
+      "https://#{CGI.escape(basic_auth_user)}:#{CGI.escape(basic_auth_password)}@#{master}" if master
+    end
+
+    def cc_menu_url
+      "#{jenkins_url}/cc.xml" if master
+    end
+
+    def rss_url(job_name)
+      master_url + "/job/#{job_name}/rssAll" if master_url
+    end
+
+    def node_attributes=(attributes)
+      self["node_attributes"] = Hashie::Mash.new(attributes)
+    end
+
+    def valid?
+      errors.empty?
+    end
+
+    def errors
+      messages = []
+      if node_attributes.has_key?("jenkins")
+        unless node_attributes.jenkins.has_key?("builds")
+          messages << "[:node_attributes][:jenkins][:builds]"
+        end
+      else
+        messages << "[:node_attributes][:jenkins]"
+      end
+      messages.map{ |path| "your config file does not have a #{path} key" }
+    end
+
+    def soloistrc
+      {
+        "recipes" => recipes,
+        "cookbook_paths" => cookbook_paths,
+        "node_attributes" => node_attributes.to_hash
+      }
+    end
+
+    def save
+      return self unless path
+      File.open(path, "w+") { |file| file.write(YAML.dump(JSON.parse(JSON.dump(to_hash)))) }
+      self
+    end
+
+    def update(options = {})
+      options.each_pair do |attr, value|
+        self.send("#{attr}=", value)
+      end
+      save
+    end
+
+    def reload
+      self.class.from_file(path)
+    end
+
+    def to_hash
+      hash = super
+      hash.delete("path")
+      {
+        "ssh_port" => ssh_port,
+        "master" => master,
+        "server_ssh_key" => server_ssh_key_path,
+        "recipes" => recipes,
+        "cookbook_paths" => cookbook_paths,
+        "node_attributes" => node_attributes
+      }.merge(hash)
+    end
+
+    def display
+      <<OOTPÜT.gsub(/(\S)\s+$/, '\1').gsub(/^\./, '')
+-- ciborg configuration --
+  Instance ID:        #{instance_id}
+  IP Address:         #{master}
+  Instance size:      #{instance_size}
+.
+  Builds:
+#{builds}
+.
+  Web URL:            #{master_url}
+  User name:          #{basic_auth_user}
+  User password:      #{basic_auth_password}
+.
+  CC Menu URL:        #{cc_menu_url}
+.
+OOTPÜT
+    end
+
+    def self.from_file(yaml_file)
+      config = {:path => yaml_file}
+      config.merge!(read_config(yaml_file)) if File.exists?(yaml_file)
+      config.delete('keypair_name')
+      self.new(config)
+    end
+
+    def self.read_config(yaml_file)
+      File.open(yaml_file, "r") { |file| YAML.load(file.read) }
+    end
+
+    def basic_auth_user
+      node_attributes[:nginx][:basic_auth_user]
+    end
+
+    def basic_auth_password
+      node_attributes[:nginx][:basic_auth_password]
+    end
+
+    def builds
+      node_attributes[:jenkins][:builds].
+        map { |build| build[:name] }    .
+        map { |build| "    %-17s %s" % [build, rss_url(build)] }  .
+        join("\n")
+    end
+
+    private
+
+    def self.default_ssh_key
+      "~/.ssh/id_rsa" if File.exists?(File.expand_path("~/.ssh/id_rsa"))
+    end
+
+    def self.default_node_attributes
+      {
+        :travis_build_environment => {
+          :user => "jenkins",
+          :group => "nogroup",
+          :home => "/var/lib/jenkins"
+        },
+        :nginx => {
+          :basic_auth_user => "ci",
+          :basic_auth_password => Ciborg::Password.generate
+        },
+        :jenkins => {
+          :builds => []
+        }
+      }
+    end
+  end
+end