hubcap 0.0.1

data/lib/hubcap/recipes/puppet.rb

@@ -0,0 +1,176 @@
+Capistrano::Configuration.instance(:must_exist).load do
+
+  namespace(:puppet) do
+
+    unless exists?(:puppet_repository)
+      set(:puppet_repository) { raise "Required variable: puppet_repository" }
+    end
+    set(:puppet_branch, 'master')
+    set(:puppet_path, '/var/www/provision/puppet')
+    set(:puppet_git_password) { Capistrano::CLI.password_prompt }
+    set(:puppet_manifest_path) { "#{puppet_path}/puppet/host.pp" }
+    set(:puppet_modules_path) { "#{puppet_path}/puppet/modules" }
+    set(:puppet_yaml_path) { "#{puppet_path}/puppet/host.yaml" }
+    set(:puppet_enc_path) { "#{puppet_path}/puppet/enc" }
+    set(:puppet_enc) { "#!/bin/sh\ncat '#{puppet_yaml_path}'" }
+    set(:puppet_parameters, '--no-report')
+
+
+    desc <<-DESC
+      Calls 'check' and 'update' and 'properties', which means it performs a
+      check for the necessary puppet dependencies, deploys the puppet scripts
+      via git, then pushes up a special yaml file describing the properties of
+      this particular server.
+    DESC
+    task(:freshen) do
+      check
+      update
+      properties
+    end
+
+
+    desc <<-DESC
+      Looks for Ruby 1.9 on the server. Installs it (and git-core) if not found.
+      Also looks for Puppet 3.0 gem on the server, and installs it if not found.
+    DESC
+    task(:check) do
+      unless exists?(:hubcap_agnostic)
+        raise "Hubcap has not configured this Capistrano instance."
+      end
+      unless hubcap_agnostic
+        raise "Puppet tasks are not available in Hubcap application mode"
+      end
+
+      apt_cmd = [
+        "env",
+        "DEBCONF_TERSE='yes'",
+        "DEBIAN_PRIORITY='critical'",
+        "DEBIAN_FRONTEND=noninteractive",
+        "apt-get --force-yes -qyu"
+      ].join(" ")
+
+      # Because Ubuntu is weird, the 1.9.1 package installs Ruby 1.9.3-p0.
+      sudo_bash([
+        'if [[ `which ruby` && (`ruby -v` =~ "ruby 1.9") ]]; then',
+          'echo "Ruby 1.9 verified";',
+        'else',
+          "#{apt_cmd} update;",
+          "#{apt_cmd} install ruby1.9.1 git-core;",
+        'fi'
+      ].join(' '))
+
+      # 3.0.0-rc5 is the last version that a) runs on Ruby 1.9 and b) works.
+      ppt_ver = '3.0.0.rc5'
+      sudo_bash([
+        "if [[ `gem q -i -n \"^puppet$\" -v #{ppt_ver}` =~ \"true\" ]]; then",
+          'echo "Puppet verified";',
+        'else',
+          "gem install puppet -v #{ppt_ver} --pre --no-rdoc --no-ri;",
+        'fi'
+      ].join(' '))
+
+      # Workaround for: http://projects.puppetlabs.com/issues/9862
+      sudo_bash([
+        'if [[ `egrep -i "^puppet" /etc/group` =~ "puppet" ]]; then',
+          'echo "Puppet group exists";',
+        'else',
+          'groupadd puppet;',
+        'fi'
+      ].join(' '))
+    end
+
+
+    desc <<-DESC
+      Basically, this pulls down your puppet scripts so you can run them.
+      It does this by cloning the Hubcap repository to each server (if necessary),
+      then fetching the latest code and resetting to the HEAD of the
+      specified branch.
+    DESC
+    task(:update) do
+      handle_data = lambda { |channel, stream, text|
+        host = channel[:host]
+        logger.info "[#{host} :: #{stream}] #{text}"
+        out = case text
+        when /\bpassword.*:/i, /passphrase/i  # Git password or SSH passphrase.
+          "#{puppet_git_password}\n"
+        when %r{\(yes/no\)}                   # Should git connect?
+          "yes\n"
+        when /accept \(t\)emporarily/         # Should git accept certificate?
+          "t\n"
+        end
+        channel.send_data(out)  if out
+      }
+      sudo("mkdir -p #{File.dirname(puppet_path)}")
+      sudo("chown #{user} #{File.dirname(puppet_path)}")
+      run(
+        "[ -d #{puppet_path} ] || git clone #{puppet_repository} #{puppet_path}",
+        :shell => nil,
+        &handle_data
+      )
+      run(
+        [
+          "cd #{puppet_path}",
+          "git fetch origin",
+          "git reset --hard origin/#{puppet_branch}"
+        ].join(' && '),
+        :shell => nil,
+        &handle_data
+      )
+    end
+
+
+    desc <<-DESC
+      Pushes a YAML file containing all the classes and parameters that Puppet
+      needs to know about in order to provision this server. Each file is
+      server-specific.
+    DESC
+    task(:properties) do
+      hubcap.servers.each { |s|
+        put(s.yaml, puppet_yaml_path, :hosts => s.address)
+      }
+      put(puppet_enc, puppet_enc_path)
+      run("chmod +x #{puppet_enc_path}")
+    end
+
+
+    desc <<-DESC
+      Tells you what would happen if you ran puppet:apply. This is a safe way
+      to test your changes.
+    DESC
+    task(:noop) do
+      run_puppet('--noop')
+    end
+
+
+    desc <<-DESC
+      Runs the puppet scripts on each server. Be careful!
+    DESC
+    task(:apply) do
+      run_puppet
+    end
+
+
+    def sudo_bash(cmd, options = {}, &blk)
+      sudo("/bin/bash -c \'#{cmd}\'", options, &blk)
+    end
+
+
+    def run_puppet(params = nil)
+      sudo([
+        "puppet apply",
+        "--node_terminus exec",
+        "--external_nodes '#{puppet_enc_path}'",
+        "--modulepath '#{puppet_modules_path}'",
+        puppet_parameters,
+        params,
+        "'#{puppet_manifest_path}'"
+      ].compact.join(' '))
+    end
+
+
+    before('puppet:noop', 'puppet:freshen')
+    before('puppet:apply', 'puppet:freshen')
+
+  end
+
+end

data/lib/hubcap/recipes/servers.rb

@@ -0,0 +1,21 @@
+Capistrano::Configuration.instance(:must_exist).load do
+
+  namespace(:servers) do
+
+    desc <<-DESC
+      Lists all the servers that match the filter used when Hubcap was loaded.
+    DESC
+    task(:list) do
+      puts(hubcap.servers.collect(&:tree))
+    end
+
+    desc <<-DESC
+      Show the entire Hubcap configuration tree for the given filter.
+    DESC
+    task(:tree) do
+      puts(hubcap.tree)
+    end
+
+  end
+
+end

data/lib/hubcap/server.rb

@@ -0,0 +1,46 @@
+class Hubcap::Server < Hubcap::Group
+
+  attr_reader(:address)
+
+
+  def initialize(parent, name, options = {}, &blk)
+    @address = options[:address] || name
+    super(parent, name, &blk)
+  end
+
+
+  def application(*args)
+    raise(Hubcap::ServerSubgroupDisallowed, 'application')
+  end
+
+
+  def server(*args)
+    raise(Hubcap::ServerSubgroupDisallowed, 'server')
+  end
+
+
+  def group(*args)
+    raise(Hubcap::ServerSubgroupDisallowed, 'group')
+  end
+
+
+  def application_parent
+    p = self
+    while p && p != hub
+      return p  if p.kind_of?(Hubcap::Application)
+      p = p.instance_variable_get(:@parent)
+    end
+  end
+
+
+  def yaml
+    {
+      'classes' => @puppet_roles.collect(&:to_s),
+      'parameters' => @params
+    }.to_yaml
+  end
+
+
+  class Hubcap::ServerSubgroupDisallowed < StandardError; end
+
+end

data/lib/hubcap/version.rb

@@ -0,0 +1,5 @@
+module Hubcap
+
+  VERSION = '0.0.1'
+
+end

data/lib/hubcap.rb

@@ -0,0 +1,29 @@
+require 'yaml'
+
+module Hubcap
+
+  def self.hub(filter_string = '', &blk)
+    Hubcap::Hub.new(filter_string).tap { |scope| scope.instance_eval(&blk) }
+  end
+
+
+  def self.load(filter_string, *paths)
+    Hubcap::Hub.new(filter_string).tap { |scope|
+      while paths.any?
+        path = paths.shift
+        if File.directory?(path)
+          paths += Dir.glob(File.join(path, '*.rb'))
+        else
+          scope.absorb(path)
+        end
+      end
+    }
+  end
+
+end
+
+
+require 'hubcap/group'
+require 'hubcap/application'
+require 'hubcap/server'
+require 'hubcap/hub'

data/test/data/example.rb

@@ -0,0 +1,68 @@
+application('example', :recipes => 'deploy') {
+
+  # These cap settings will apply to all the servers within this application
+  # group or any subgroups. (The same applies for roles, attributes, etc.)
+  cap_set(:repository, 'git@github.com:example/example.git')
+  cap_set(:branch, 'master')
+
+  # Load some ssh keys into param() from a separate (more secure?) file.
+  # These will be handed to your puppet scripts.
+  #absorb('nodes/private/admin_ssh_keys')
+
+  # Just a normal Ruby hash var we can reuse throughout the config.
+  common_env = {
+    'PGUSER' => 'example',
+    'RABBIT_URI' => 'amqp://example:password@localhost:54322'
+  }
+
+  # A local dev simulation.
+  group('vagrant') {
+    param('env' => common_env.merge('PGPASSWORD' => 'password'))
+    server('127.0.0.1:2222') {
+      role(:app, :db, :queue)
+    }
+  }
+
+  # Your staging environment.
+  group('staging') {
+    param('env' => common_env.merge(
+      'PGPASSWORD' => 'pa55w3rd',
+      'PGHOST' => '10.10.10.20',
+      'RABBIT_URI' => 'amqp://example:pa55w3rd@10.10.10.20'
+    ))
+    server('app', :address => '10.10.10.10') {
+      role(:app)
+    }
+    server('db', :address => '10.10.10.15') {
+      role(:db)
+    }
+    server('queue', :address => '10.10.10.20') {
+      role(:queue)
+    }
+  }
+
+  # Your production environment.
+  group('production') {
+    prod_env = common_env.merge(
+      'PGPASSWORD' => '1391f3a24daef0c78f75cbef9d62eb848c2d454c71a5fd2a',
+      'PGHOST' => '20.20.20.20',
+      'RABBIT_URI' => 'amqp://example:e18edfa58fa6c5d3a9a@20.20.20.30'
+    )
+    param('env' => prod_env)
+    group('app') {
+      role(:app)
+      param('env' => prod_env.merge('FORCE_SSL' => '1'))
+      server('app-1.example.com')
+      server('app-2.example.com')
+      server('app-3.example.com')
+    }
+    group('db') {
+      role(:db)
+      server('db-1.example.com')
+      server('db-2.example.com')
+    }
+    server('queue-1.example.com') {
+      role(:queue)
+    }
+  }
+}

data/test/data/parts/foo_param.rb

@@ -0,0 +1 @@
+param(:foo => 'foo')

data/test/data/readme.rb

@@ -0,0 +1,47 @@
+# An application called 'readme' that uses Cap's default deployment recipe.
+application('readme', :recipes => 'deploy') {
+  # Set a capistrano variable.
+  cap_set('repository', 'git@github.com:joseph/readme.git')
+
+  # Declare that all servers will have the 'baseline' puppet class.
+  role(:puppet => 'baseline')
+
+  group('staging') {
+    # Puppet will have a $::exception_subject_prefix variable on these servers.
+    param('exception_subject_prefix' => '[STAGING] ')
+    # For simple staging, just one server that does everything.
+    server('readme.stage') {
+      role(:cap => [:web, :app, :db], :puppet => ['proxy', 'app', 'db'])
+    }
+  }
+
+  group('production') {
+    # Puppet will have these top-scope variables on all these servers.
+    param(
+      'exception_subject_prefix' => '[PRODUCTION] ',
+      'env' => {
+        "FORCE_SSL" => true,
+        "S3_KEY" => "AKIAKJRK23943202JK",
+        "S3_SECRET" => "KDJkaddsalkjfkawjri32jkjaklvjgakljkj"
+      }
+    )
+
+    group('proxy') {
+      # Servers will have the :web role and the 'proxy' puppet class.
+      role(:cap => :web, :puppet => 'proxy')
+      server('proxy-1', :address => '10.10.10.5')
+    }
+
+    group('app') {
+      # Servers will have the :app role and the 'app' puppet class.
+      role(:app)
+      server('app-1', :address => '10.10.10.10')
+      server('app-2', :address => '10.10.10.11')
+    }
+
+    group('db') {
+      role(:db)
+      server('db-1', :address => '10.10.10.50')
+    }
+  }
+}

data/test/data/simple.rb

@@ -0,0 +1,4 @@
+group('simple') {
+  role('baseline')
+  server('localhost')
+}

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'hubcap'
+require 'test/unit'

data/test/unit/test_application.rb

@@ -0,0 +1,20 @@
+require 'test_helper'
+
+class Hubcap::TestApplication < Test::Unit::TestCase
+
+  def test_recipe_paths
+    hub = Hubcap.hub { application('test', :recipes => 'foo') }
+    assert_equal(['foo'], hub.applications.first.recipe_paths)
+
+    hub = Hubcap.hub { application('test', :recipes => ['foo', 'bar']) }
+    assert_equal(['foo', 'bar'], hub.applications.first.recipe_paths)
+  end
+
+
+  def test_nested_application_disallowed
+    assert_raises(Hubcap::NestedApplicationDisallowed) {
+      Hubcap.hub { application('test') { application('child') } }
+    }
+  end
+
+end

data/test/unit/test_group.rb

@@ -0,0 +1,176 @@
+require 'test_helper'
+
+class Hubcap::TestGroup < Test::Unit::TestCase
+
+  def test_name
+    hub = Hubcap.hub { group('test') }
+    assert_equal('test', hub.groups.first.name)
+  end
+
+
+  def test_hub
+    hub = Hubcap.hub { group('test') }
+    assert_equal(hub, hub.groups.first.hub)
+  end
+
+
+  def test_must_have_parent_unless_hub
+    assert_raises(Hubcap::GroupWithoutParent) { Hubcap::Group.new(nil, 'foo') }
+  end
+
+
+  def test_history
+    hub = Hubcap.hub { group('test') { group('child') } }
+    assert_equal(['test'], hub.children.first.history)
+    assert_equal(['test', 'child'], hub.children.first.children.first.history)
+  end
+
+
+  def test_absorb
+    hub = Hubcap.hub {
+      group('test') { absorb('test/data/parts/foo_param') }
+    }
+    assert_equal('foo', hub.groups.first.params[:foo])
+  end
+
+
+  def test_processable_and_collectable
+    x = {}
+    hub = Hubcap.hub('g1.a1') {
+      x[:g1] = group('g1') {
+        x[:a1] = application('a1') {
+          x[:s1] = server('s1')
+        }
+        x[:a2] = application('a2') {
+          x[:s2] = server('s2')
+        }
+      }
+      x[:g2] = group('g2') {
+        x[:a3] = application('a3') {
+          x[:s3] = server('s3')
+        }
+      }
+    }
+    assert_equal(true, x[:g1].processable?)
+    assert_equal(false, x[:g1].collectable?)
+    assert_equal(true, x[:a1].processable?)
+    assert_equal(true, x[:a1].collectable?)
+    assert_equal(true, x[:s1].processable?)
+    assert_equal(true, x[:s1].collectable?)
+    assert_equal(false, x[:g2].processable?)
+    assert_equal(false, x[:g2].collectable?)
+    assert_nil(x[:a3])
+    assert_nil(x[:s3])
+  end
+
+
+  def test_cap_set
+    # Single value, as key, val arguments
+    hub = Hubcap.hub { group('test') { cap_set(:foo, 'bar') } }
+    assert_equal('bar', hub.cap_sets[:foo])
+
+    # Single value, as one hash argument
+    hub = Hubcap.hub { group('test') { cap_set('baz' => 'yyy') } }
+    assert_equal('yyy', hub.cap_sets['baz'])
+
+    # Multiple values
+    hub = Hubcap.hub { group('test') { cap_set(:a => 1, :z => 0) } }
+    assert_equal(1, hub.cap_sets[:a])
+    assert_equal(0, hub.cap_sets[:z])
+
+    # Lazily-evaluated block
+    hub = Hubcap.hub { group('test') { cap_set(:blk) { 'garply' } } }
+    assert_equal('garply', hub.cap_sets[:blk].call)
+  end
+
+
+  def test_cap_attribute
+    # Single value as key, val arguments
+    hub = Hubcap.hub { server('test') { cap_attribute(:foo, 'bar') } }
+    assert_equal({ :foo => 'bar' }, hub.servers.first.cap_attributes)
+
+    # Single value as hash
+    hub = Hubcap.hub { server('test') { cap_attribute(:foo => 'bar') } }
+    assert_equal({ :foo => 'bar' }, hub.servers.first.cap_attributes)
+
+    # Multiple values
+    hub = Hubcap.hub { server('test') { cap_attribute(:a => 1, :z => 0) } }
+    assert_equal({ :a => 1, :z => 0 }, hub.servers.first.cap_attributes)
+
+    # Cap attributes are additive down the tree
+    hub = Hubcap.hub {
+      group('g') {
+        cap_attribute(:excellent => true)
+        server('s') { cap_attribute(:modest => false) }
+      }
+    }
+    assert_equal({ :excellent => true }, hub.groups.first.cap_attributes)
+    assert_equal(
+      { :excellent => true, :modest => false },
+      hub.servers.first.cap_attributes
+    )
+  end
+
+
+  def test_role
+    # Single role
+    hub = Hubcap.hub {
+      role(:baseline)
+      server('test')
+    }
+    assert_equal([:baseline], hub.servers.first.cap_roles)
+    assert_equal([:baseline], hub.servers.first.puppet_roles)
+
+    # Multiple roles in a single declaration
+    hub = Hubcap.hub {
+      role(:baseline, :app)
+      server('test')
+    }
+    assert_equal([:baseline, :app], hub.servers.first.cap_roles)
+    assert_equal([:baseline, :app], hub.servers.first.puppet_roles)
+
+    # Multiple declarations are additive
+    hub = Hubcap.hub {
+      role(:baseline)
+      server('test') { role(:db) }
+    }
+    assert_equal([:baseline, :db], hub.servers.first.cap_roles)
+    assert_equal([:baseline, :db], hub.servers.first.puppet_roles)
+
+    # Separate cap and puppet roles
+    hub = Hubcap.hub {
+      role(:cap => :app, :puppet => 'testapp')
+      server('test')
+    }
+    assert_equal([:app], hub.servers.first.cap_roles)
+    assert_equal(['testapp'], hub.servers.first.puppet_roles)
+
+    # Separate cap/puppet roles can be defined with an array
+    # Also shows that multiple role declarations are additive
+    hub = Hubcap.hub {
+      role(:cap => [:app, :db])
+      server('test') { role(:baseline) }
+    }
+    assert_equal([:app, :db], hub.cap_roles)
+    assert_equal([], hub.puppet_roles)
+    assert_equal([:app, :db, :baseline], hub.servers.first.cap_roles)
+    assert_equal([:baseline], hub.servers.first.puppet_roles)
+  end
+
+
+  def test_param
+    # Single key/val.
+    hub = Hubcap.hub {
+      server('test') { param('foo' => 1) }
+    }
+    assert_equal(1, hub.servers.first.params['foo'])
+
+    # Multiple key/vals.
+    hub = Hubcap.hub {
+      server('test') { param('foo' => 1, 'baz' => 2) }
+    }
+    assert_equal(1, hub.servers.first.params['foo'])
+    assert_equal(2, hub.servers.first.params['baz'])
+  end
+
+end