hubcap 0.0.1

data/Capfile.example

@@ -0,0 +1,37 @@
+# This just lets us test Hubcap without installing it as a gem...
+#
+$LOAD_PATH.unshift('lib')
+
+# Force Hubcap into agnostic mode with AGNOSTIC=1, or application mode with 0.
+# Use this environment variable with care!
+#
+# if ag = ENV['AGNOSTIC']
+#   set(:hubcap_agnostic, ag == '1')  if ['0', '1'].include?(ag)
+# end
+
+
+# SSH user (and login password if required).
+#
+set(:user, ENV['AS'] || 'deploy')
+set(:password) {
+  puts
+  warn("A server is prompting for the \"#{user}\" user's login password.")
+  warn("NB: You can run as another user with the AS environment variable.")
+  Capistrano::CLI.password_prompt
+}
+
+
+# This is only processed if we are using cap directly (not bin/hubcap).
+#
+# Load servers and sets from node config. Any recipes loaded after this
+# point will be available only in application mode.
+#
+unless exists?(:hubcap)
+  if (target = ENV['TO']) && !ENV['TO'].empty?
+    target = ''  if target == 'ALL'
+    require('hubcap')
+    Hubcap.load(target, 'test/data').configure_capistrano(self)
+  else
+    warn("NB: No servers specified. Target a Hubcap group or server with TO.")
+  end
+end

data/README.md

@@ -0,0 +1,168 @@
+# Hubcap
+
+Create a hub for your server configuration. Use it with Capistrano,
+Puppet and others.
+
+
+## Meet Hubcap
+
+You want to provision your servers with Puppet. You want to deploy to your
+servers with Capistrano. Where do you define your server infrastructure?
+
+Hubcap lets you define the characteristics of your servers once. Then, when you
+need to use Puppet, Capistrano drives. It deploys your Puppet modules and
+manifests, plus a special host-specific file, and applies it to the server.
+(This is sometimes called "masterless Puppet". It has a lot of benefits that
+derive from decentralization and pushing changes on-demand.)
+
+Here's what your config file might look like:
+
+    # An application called 'readme' that uses Cap's default deployment recipe.
+    application('readme', :recipes => 'deploy') {
+      # Set a capistrano variable.
+      cap_set('repository', 'git@github.com:joseph/readme.git')
+
+      # Declare that all servers will have the 'baseline' puppet class.
+      role(:puppet => 'baseline')
+
+      group('staging') {
+        # Puppet gets a $::exception_subject_prefix variable on these servers.
+        param('exception_subject_prefix' => '[STAGING] ')
+        # For simple staging, just one server that does everything.
+        server('readme.stage') {
+          role(:cap => [:web, :app, :db], :puppet => ['proxy', 'app', 'db'])
+        }
+      }
+
+      group('production') {
+        # Puppet gets these top-scope variables on servers in this group.
+        param(
+          'exception_subject_prefix' => '[PRODUCTION] ',
+          'env' => {
+            "FORCE_SSL" => true,
+            "S3_KEY" => "AKIAKJRK23943202JK",
+            "S3_SECRET" => "KDJkaddsalkjfkawjri32jkjaklvjgakljkj"
+          }
+        )
+
+        group('proxy') {
+          # Servers will have the :web role and the 'proxy' puppet class.
+          role(:cap => :web, :puppet => 'proxy')
+          server('proxy-1', :address => '10.10.10.5')
+        }
+
+        group('app') {
+          # Servers will have the :app role and the 'app' puppet class.
+          role(:app)
+          server('app-1', :address => '10.10.10.10')
+          server('app-2', :address => '10.10.10.11')
+        }
+
+        group('db') {
+          role(:db)
+          server('db-1', :address => '10.10.10.50')
+        }
+      }
+    }
+
+
+Save this as `hub/example.rb`.
+
+Run:
+
+    $ hubcap ALL servers:tree
+
+That's a lot of info. You can filter your server list to target specific
+groups of servers:
+
+    $ `hubcap example.vagrant servers:tree`
+
+You can run `list` in place of `tree` to see just the servers that match
+your filter:
+
+    $ `hubcap example.production.db servers:tree`
+
+
+## Working with Puppet
+
+You should have your Puppet modules in a git repository. The location of this
+repository should be specified in your Capfile with
+`set(:puppet_repository, '...')`. Your site manifest should be within this repo
+at `puppet/host.pp` (but this is also configurable).
+
+When you're ready to provision some servers:
+
+    $ `hubcap example.vagrant puppet:noop`
+    $ `hubcap example.vagrant puppet:apply`
+
+Once that's done, you can deploy your app in the usual way:
+
+    $ `hubcap example.vagrant deploy:setup deploy:cold`
+
+
+
+## The Hubcap DSL
+
+The Hubcap DSL is very simple. This is the basic set of statements:
+
+* `group` - A named set of servers, roles, variables, attributes. Groups
+  can be nested.
+
+* `application` - A special kind of group. You can pass `:recipes => ...`
+  to this declaration. Each recipe path will be loaded into Capistrano only
+  for this application. Applications can't be nested.
+
+* `server` - An actual host that you are managing with Capistrano and
+  Puppet. The first argument is the name, which can be an IP address or domain
+  name if you like. Otherwise, pass `:address => '...'`.
+
+* `cap_set` - Set a Capistrano variable.
+
+* `cap_attribute` - Set a Cap attribute on all the servers within this
+  group, such as `:primary => true` or `:no_release => true`.
+
+* `role` - Add a role to the list of Capistrano roles for servers within
+  this group. By default, these roles are supplied as classes to apply to the 
+  host in Puppet. You can specify that a role is Capistrano-only with
+  `:cap => '...'`, or Puppet-only with :puppet => `'...'`. This is additive:
+  if you have multiple role declarations in your tree, all of them apply.
+
+* `param` - Add to a hash of 'parameters' that will be supplied to Puppet
+  as top-scope variables for servers in this group. Like `role`, this is 
+  additive.
+
+Hubcap uses Puppet's External Node Classifier (ENC) feature to provide the
+list of classes and parameters for a specific host. More info here: 
+http://docs.puppetlabs.com/guides/external_nodes.html
+
+
+## Hubcap as a library
+
+If you'd rather run `cap` than `hubcap`, you can load your hub configuration
+directly in your `Capfile`. Add this to the end of the file:
+
+     require('hubcap')
+     Hubcap.load('', 'hub').configure_capistrano(self)
+
+The two arguments to `Hubcap.load` are the filter (where `''` means no filter),
+and the path to the hub configuration. This will load `*.rb` in the `hub`
+directory (but not subdirectories). You can specify multiple paths as additional
+arguments -- whole directories or specific files.
+
+If you want to simulate the behaviour of the `hubcap` script, you could do it
+with something like this in your `Capfile`.
+
+     # Load servers and sets from node config. Any recipes loaded after this
+     # point will be available only in application mode.
+     if (target = ENV['TO']) && !ENV['TO'].empty?
+       target = ''  if target == 'ALL'
+       require('hubcap')
+       Hubcap.load(target, 'hub').configure_capistrano(self)
+     else
+       warn("NB: No servers specified. Target a Hubcap group with TO.")
+     end
+
+In this set-up, you'd run `cap` like this:
+
+    $ cap TO=example.vagrant servers:tree
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+
+Rake::TestTask.new { |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/unit/test*.rb']
+  t.verbose = true
+}
+
+desc("Run tests")
+task(:default => :test)

data/bin/hubcap

@@ -0,0 +1,98 @@
+#!/usr/bin/env ruby
+
+# 'hubcap' - a convenience script for loading Capistrano with Hubcap config.
+#
+# Usages:
+#
+#   # This will list the available default tasks:
+#   hubcap -T
+#
+#   # This task (or tasks) will run on all defined servers in the hub:
+#   hubcap ALL task:name ...
+#
+#   # This task will run on all servers inside the specified group:
+#   hubcap app_name.group_name task:name ...
+#
+#   # This will run on just the specified server:
+#   hubcap app_name.group_name.server_name task:name ...
+#
+#
+# Note that the hub configuration files are loaded from a subdirectory of the
+# current directory named 'hub'. You can change this to point at another
+# directory, single file or multiple files by setting the HUB_CONFIG env var.
+#
+#   HUB_CONFIG=test/data hubcap ALL servers:list
+#
+#   HUB_CONFIG=test/data/example.rb,test/data/simple.rb hubcap ALL servers:tree
+#
+#
+
+
+require('capistrano/cli')
+require('hubcap')
+
+
+class Hubcap::CLI < Capistrano::CLI
+
+  attr_accessor(:cap)
+
+
+  def self.roll!
+    target = pre_parse_for_target(ARGV)
+
+    if !target
+      puts("Usage: hubcap name.of.target.group task:name")
+      puts("To target all servers: hubcap ALL task:name")
+      exit
+    end
+
+    if target == :skip
+      parse(ARGV).execute!
+    else
+      filter = (target == 'ALL') ? '' : target
+      paths = ['hub']
+      paths = ENV['HUB_CONFIG'].split(',')  if ENV['HUB_CONFIG']
+      hub = Hubcap.load(filter, *paths)
+      unless hub.children.any?
+        puts("Hubcap error: no servers for '#{target}' in [#{paths.join(',')}]")
+        exit
+      end
+      parse(ARGV) { |cap|
+        cap.load('standard')
+        hub.configure_capistrano(cap)
+      }.execute!
+    end
+  end
+
+
+  def self.pre_parse_for_target(args)
+    if args.length == 1 && args.first.match(/^-/)
+      return :skip
+    elsif args.length < 1
+      puts("Error: no servers specified")
+      return nil
+    elsif args.length < 2
+      puts("Error: no tasks specified")
+      return nil
+    end
+    return args.shift
+  end
+
+
+  def self.parse(args)
+    cli = new(args)
+    cli.parse_options!
+    cli.cap = Capistrano::Configuration.new(cli.options)
+    yield(cli.cap)  if block_given?
+    cli
+  end
+
+
+  def instantiate_configuration(options = {})
+    self.cap
+  end
+
+end
+
+
+Hubcap::CLI.roll!

data/hubcap.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/hubcap/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors = ['Joseph Pearson']
+  gem.email = ['joseph@booki.sh']
+  gem.description = 'Unite Capistrano and Puppet config in one Ruby file.'
+  gem.summary = 'Hubcap Capistrano/Puppet extension'
+  gem.homepage = ''
+
+  gem.files = `git ls-files`.split($\)
+  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name = 'hubcap'
+  gem.require_paths = ['lib']
+  gem.version = Hubcap::VERSION
+end

data/lib/hubcap/application.rb

@@ -0,0 +1,24 @@
+class Hubcap::Application < Hubcap::Group
+
+  attr_reader(:recipe_paths)
+
+  def initialize(parent, name, options = {}, &blk)
+    @recipe_paths = [options[:recipes]].flatten.compact
+    super(parent, name, &blk)
+  end
+
+
+  def application(*args)
+    raise(Hubcap::NestedApplicationDisallowed)
+  end
+
+
+  def extend_tree(outs)
+    outs << "Load: #{@recipe_paths.inspect}"  if @recipe_paths.any?
+  end
+
+
+  class Hubcap::NestedApplicationDisallowed < StandardError; end
+
+end
+

data/lib/hubcap/group.rb

@@ -0,0 +1,223 @@
+class Hubcap::Group
+
+  attr_reader(
+    :name,
+    :cap_attributes,
+    :cap_roles,
+    :puppet_roles,
+    :params,
+    :parent,
+    :children
+  )
+
+  # Supply the parent group, the name of this new group and a block of code to
+  # evaluate in the context of this new group.
+  #
+  # Every group must have a parent group, unless it is the top-most group: the
+  # hub. The hub must be a Hubcap::Hub.
+  #
+  def initialize(parent, name, &blk)
+    @name = name.to_s
+    if @parent = parent
+      @cap_attributes = parent.cap_attributes.clone
+      @cap_roles = parent.cap_roles.clone
+      @puppet_roles = parent.puppet_roles.clone
+      @params = parent.params.clone
+    elsif !kind_of?(Hubcap::Hub)
+      raise(Hubcap::GroupWithoutParent, self.inspect)
+    end
+    @children = []
+    instance_eval(&blk)  if blk && processable?
+  end
+
+
+  # Load a Ruby file and evaluate it in the context of this group.
+  # Like Ruby's require(), the '.rb' is optional in the path.
+  #
+  def absorb(path)
+    p = path
+    p += '.rb'  unless File.exists?(p)
+    raise("File not found: #{path}")  unless File.exists?(p)
+    code = IO.read(p)
+    eval(code)
+  end
+
+
+  # Finds the top-level Hubcap::Hub to which this group belongs.
+  #
+  def hub
+    @parent ? @parent.hub : self
+  end
+
+
+  # An array of names, from the oldest ancestor to the parent to self.
+  #
+  def history
+    @parent ? @parent.history + [@name] : []
+  end
+
+
+  # Indicates whether we should process this group. We process all groups that
+  # match the filter or are below the furthest point in the filter.
+  #
+  # "Match the filter" means that this group's history and the filter are
+  # identical to the end of the shortest of the two arrays.
+  #
+  def processable?
+    s = [history.length, hub.filter.length].min
+    history.slice(0,s) == hub.filter.slice(0,s)
+  end
+
+
+  # Indicates whether we should store this group in the hub's array of
+  # groups/applications/servers. We only store groups at the end of the filter
+  # and below.
+  #
+  # That is, this group's history should be the same length or longer than the
+  # filter, but identical at each point in the filter.
+  #
+  def collectable?
+    (history.length >= hub.filter.length) && processable?
+  end
+
+
+  # Sets a variable in the Capistrano instance.
+  #
+  # Note: when Hubcap is in application mode (not executing a default task),
+  # an exception will be raised if a variable is set twice to two different
+  # values.
+  #
+  # Either:
+  #   cap_set(:foo, 'bar')
+  # or:
+  #   cap_set(:foo => 'bar')
+  # and this works too:
+  #   cap_set(:foo => 'bar', :garply => 'grault')
+  # in fact, even this works:
+  #   cap_set(:foo) { bar }
+  #
+  def cap_set(*args, &blk)
+    if args.length == 2
+      hub.cap_set(args.first => args.last)
+    elsif args.length == 1
+      if block_given?
+        hub.cap_set(args.first => blk)
+      elsif args.first.kind_of?(Hash)
+        hub.cap_set(args.first)
+      end
+    else
+      raise ArgumentError('Must be (key, value) or (hash) or (key) { block }.')
+    end
+  end
+
+
+  # Sets an attribute in the Capistrano server() definition for all Hubcap
+  # servers to which it applies.
+  #
+  # For eg, :primary => true or :no_release => true.
+  #
+  # Either:
+  #   cap_attribute(:foo, 'bar')
+  # or:
+  #   cap_attribute(:foo => 'bar')
+  # and this works too:
+  #   cap_attribute(:foo => 'bar', :garply => 'grault')
+  #
+  def cap_attribute(*args)
+    if args.length == 2
+      cap_attribute(args.first => args.last)
+    elsif args.length == 1 && args.first.kind_of?(Hash)
+      @cap_attributes.update(args.first)
+    else
+      raise ArgumentError('Must be (key, value) or (hash).')
+    end
+  end
+
+
+  # Sets the Capistrano role and/or Puppet class for all Hubcap servers to
+  # which it applies.
+  #
+  # When declared multiple times (even in parents), it's additive.
+  #
+  # Either:
+  #   role(:app)
+  # or:
+  #   role(:app, :db)
+  # or:
+  #   role(:cap => :app, :puppet => 'relishapp')
+  # or:
+  #   role(:cap => [:app, :db], :puppet => 'relishapp')
+  #
+  def role(*args)
+    if args.length == 1 && args.first.kind_of?(Hash)
+      h = args.first
+      @cap_roles += [h[:cap]].flatten  if h.has_key?(:cap)
+      @puppet_roles += [h[:puppet]].flatten  if h.has_key?(:puppet)
+    else
+      @cap_roles += args
+      @puppet_roles += args
+    end
+  end
+
+
+  # Adds values to a hash that is supplied to Puppet when it is provisioning
+  # the server.
+  #
+  # If you do this...
+  #   params(:foo => 'bar')
+  # ...then Puppet will have a top-level variable called $foo, containing 'bar'.
+  #
+  def param(hash)
+    @params.update(hash)
+  end
+
+
+  # Instantiate an application as a child of this group.
+  #
+  def application(name, options = {}, &blk)
+    add_child(:applications, Hubcap::Application.new(self, name, options, &blk))
+  end
+
+
+  # Instantiate a server as a child of this group.
+  #
+  def server(name, options = {}, &blk)
+    add_child(:servers, Hubcap::Server.new(self, name, options, &blk))
+  end
+
+
+  # Instantiate a group as a child of this group.
+  #
+  def group(name, &blk)
+    add_child(:groups, Hubcap::Group.new(self, name, &blk))
+  end
+
+
+  # Returns a formatted string of all the key details for this group, and
+  # recurses into each child.
+  #
+  def tree(indent = "  ")
+    outs = [self.class.name.split('::').last.upcase, "Name: #{@name}"]
+    outs << "Atts: #{@cap_attributes.inspect}"  if @cap_attributes.any?
+    outs << "Pram: #{@params.inspect}"  if @params.any?
+    extend_tree(outs)  if respond_to?(:extend_tree)
+    outs << ""
+    if @children.any?
+      @children.each { |child| outs << child.tree(indent+"  ") }
+    end
+    outs.join("\n#{indent}")
+  end
+
+
+  private
+
+    def add_child(category, child)
+      @children << child  if child.processable?
+      hub.send(category) << child  if child.collectable?
+      child
+    end
+
+
+  class Hubcap::GroupWithoutParent < StandardError; end
+
+end

data/lib/hubcap/hub.rb

@@ -0,0 +1,125 @@
+class Hubcap::Hub < Hubcap::Group
+
+  attr_reader(:filter, :applications, :servers, :groups, :cap_sets)
+
+
+  def initialize(filter_string)
+    @filter = filter_string.split('.')
+    @cap_sets = {}
+    @cap_set_clashes = []
+    @cap_attributes = {}
+    @cap_roles = []
+    @puppet_roles = []
+    @params = {}
+    @applications = []
+    @servers = []
+    @groups = []
+    super(nil, '∞')
+  end
+
+
+  def cap_set(hash)
+    hash.each_pair { |k, v|
+      if @cap_sets[k] && @cap_sets[k] != v
+        @cap_set_clashes << { k => v }
+      else
+        @cap_sets[k] = v
+      end
+    }
+  end
+
+
+  def extend_tree(outs)
+    outs << "Sets: #{@cap_sets.inspect}"
+  end
+
+
+  # Does a few things:
+  #
+  # * Sets the :hubcap variable in the Capistrano instance.
+  # * Loads the default Hubcap recipes into the Capistrano instance.
+  # * Defines each server as a Capistrano server().
+  #
+  # If we are in "agnostic mode" - executing a default task - that's all we
+  # do. If we are in "application mode" - executing at least one non-standard
+  # task - then we do a few more things:
+  #
+  # * Load all the recipes for the application into the Capistrano instance.
+  # * Set all the @cap_set variables in the Capistrano instance.
+  #
+  def configure_capistrano(cap)
+    raise(Hubcap::CapistranoAlreadyConfigured)  if cap.exists?(:hubcap)
+    cap.set(:hubcap, self)
+
+    cap.instance_eval {
+      require('hubcap/recipes/servers')
+      require('hubcap/recipes/puppet')
+    }
+
+    # Declare the servers.
+    servers.each { |s|
+      cap.server(s.address, *(s.cap_roles + [s.cap_attributes]))
+    }
+
+    configure_application_mode(cap)  unless capistrano_is_agnostic?(cap)
+  end
+
+
+  # In agnostic mode, Capistrano recipes for specific applications are not
+  # loaded, and cap_set collisions are ignored.
+  #
+  def capistrano_is_agnostic?(cap)
+    return cap.fetch(:hubcap_agnostic)  if cap.exists?(:hubcap_agnostic)
+    ag = true
+    options = cap.logger.instance_variable_get(:@options)
+    if options && options[:actions] && options[:actions].any?
+      tasks = options[:actions].clone
+      while tasks.any?
+        ag = false  unless cap.find_task(tasks.shift)
+      end
+    end
+    cap.set(:hubcap_agnostic, ag)
+    ag
+  end
+
+
+  private
+
+    def configure_application_mode(cap)
+      apps = servers.collect(&:application_parent).compact.uniq
+
+      # A - there should be only one application for all the servers
+      raise(
+        Hubcap::ApplicationModeError::TooManyApplications,
+        apps.collect(&:name).join(', ')
+      )  if apps.size > 1
+
+      # B - there should be no clash of cap sets
+      raise(
+        Hubcap::ApplicationModeError::DuplicateSets,
+        @cap_set_clashes.inspect
+      )  if @cap_set_clashes.any?
+
+      # C - app-specific, but no applications
+      raise(Hubcap::ApplicationModeError::NoApplications)  if !apps.any?
+
+      # Otherwise, load all recipes...
+      cap.set(:application, apps.first.name)
+      apps.first.recipe_paths.each { |rp| cap.load(rp) }
+
+      # ..and declare all cap sets.
+      @cap_sets.each_pair { |key, val|
+        val.kind_of?(Proc) ? cap.set(key, &val) : cap.set(key, val)
+      }
+    end
+
+
+
+  class Hubcap::CapistranoAlreadyConfigured < StandardError; end
+  class Hubcap::ApplicationModeError < StandardError;
+    class TooManyApplications < self; end
+    class NoApplications < self; end
+    class DuplicateSets < self; end
+  end
+
+end