stack-kicker 0.0.10 → 0.0.11

data/.rvmrc

@@ -6,7 +6,7 @@
 # First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
 # Only full ruby name is supported here, for short names use:
 #     echo "rvm use 1.9.3" > .rvmrc
-environment_id="ruby-1.9.3-p286@stack-kicker"
+environment_id="ruby-1.9.3@stack-kicker"
 
 # Uncomment the following lines if you want to verify rvm version per project
 # rvmrc_rvm_version="1.16.17 (stable)" # 1.10.1 seams as a safe start

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+## 0.0.11
+Stack.validate() was still using the role name, not role_details[:security_group]
+Added doc/examples/apache-via-cloud-init, example of just using cloud-init, no chef.
+Fixed some issues when only using cloud-init

data/README.md

@@ -1,14 +1,10 @@
 # stack-kicker
 
-stack-kicker is a simple 'application stack' deployment tool, it's purpose in life 
-is to spin up a set of instances in a repeatable, controlled fashion, and optionally 
+stack-kicker is a simple 'application stack' deployment tool, it's purpose in life
+is to spin up a set of instances in a repeatable, controlled fashion, and optionally
 run post-install scripts after each instance has been started.
 
-stack-kicker has hooks to allow default & custom cloud-init templates to be built & passed to
-your compute provider (we currently use ruby-openstack, so are limited to OpenStack providers, 
-however, a sister project, aws-kicker, uses fog.io, the interaction with the compute provider is
-minimal, so it's on the roadmap to merge aws-kicker & stack-kicker, and use either an internal
-abstraction layer or just fog.io for all compute provisioning requests)
+stack-kicker has hooks to allow default & custom cloud-init templates to be built & passed to your compute provider (we currently use ruby-openstack, so are limited to OpenStack providers, however, a sister project, aws-kicker, uses fog.io, the interaction with the compute provider is minimal, so it's on the roadmap to merge aws-kicker & stack-kicker, and use either an internal abstraction layer or just fog.io for all compute provisioning requests)
 
 ## Stackfile
 Normally, stack configurations are stored in a Stackfile, which is a ruby hash of configuration options.
@@ -24,14 +20,72 @@ stack-kicker sequentially iterates over defined roles, creating the required num
 Hostnames are generated from a customizable template, which is effectively:
 
 	config[:name_template] = '%s-%s-%s%04d'
-	config['global_service_name'] = 'myapp'
+	config[:global_service_name] = 'myapp'
 	site = <derived from region/az, via config[:site_template]>
-	hostname = sprintf(config[:name_template], config['global_service_name'], site, role, position)
+	hostname = sprintf(config[:name_template], config[:global_service_name], site, role, position)
 
-So hostnames will be myapp-az1-chef0001, myapp-az1-web0001, myapp-az1-web0002 etc. 
+So hostnames will be myapp-az1-chef0001, myapp-az1-web0001, myapp-az1-web0002 etc.
 
 post-install scripts are executed from the same host as stack-kicker is being used, using the same credentials as the current user.  They are can be used to retrieve information from a freshly built node (like certificates from a chef server), so block progress until the chef-client run has completed (we use this to block percona/galera & rabbitmq cluster builds so that the first node is up & running correctly before we try and add another node to the cluster)
 
+### [Role Attributes](id:role_attributes)
+Roles have several attributes, which control how & how many nodes are created, and how they are created.  Below shows the default values for these attributes, which can all be overridden.
+
+    :role_name = {
+      :count               => 1,
+      :azs[]               => config['REGION']
+
+      :chef_server         => false,
+      :skip_chef_prereg    => false,
+      :security_group      => :role_name.to_s,
+
+      :cloud_config_yaml   => 'cloud-config.yaml',
+      :bootstrap           => 'chef-client-bootstrap-excl-validation-pem.sh',
+      :data_dir            => '/dummy',
+
+      :floating_ips        => nil
+
+      :post_install_script => nil,
+      :post_install_args   => '',
+      :post_install_cwd    => '/.'
+    }
+
+#### :count
+The number of nodes or instances of this role that will be created.
+#### :azs
+This can be an array of strings, so that nodes will be placed in specific availability zones.  If no array is set, the REGION set in the global section will be used.
+#### :chef_server
+this is flag used to denote if the node created by this role should be used as a chef server for nodes created by other roles.  If this flag is set, we extract the public & private IP address for use later, as well downloading validation.pem & creating a user account in chef & downloading the pen (this is done via chef-post-install.sh, or your own alternative methods)
+#### :skip_chef_prereg
+This is usually only used when :chef_server = true, it stops stack-kicker from attempting to pre-create the chef client & node and applying roles to the node.
+#### :security_group
+security group to be assigned to this node.  (set to default is you don't want to manage security groups for every role)
+#### :cloud_config_yaml
+defaults to a file which contains a simple template (lib/cloud-config.yaml in the github repo) that installs the http://apt.opscode.com repo & gig key, as well as installing the opscode-keyring.  Can be replaced with any filename that complies with cloud-init.
+#### :bootstrap
+Optional filename, the contents of which will get combined with :cloud_config_yaml to form the cloud-init payload (using mime encoding, supported types are #include, ) with some variable substation (chef server ip, environment, validation.pem, roles)  See lib/chef-client-bootstrap-excl-validation-pem.sh as an example.
+
+:cloud_config_yaml & :bootstrap files can be of the following type:
+
+mime-type          | first line
+-------------------|-----------
+text/x-include-url | #include
+text/x-shellscript | #!
+text/cloud-config  | #cloud-config
+text/upstart-job   | #upstart-job
+text/part-handler  | #part-handler
+text/cloud-boothook | #cloud-boothook
+
+#### :data_dir
+data_dir is a hook into the optional :cloud_config_yaml template (lib/cloud-config-w-ephemeral.yaml), which formats & mounts ephemeral0 early in the boot process, allowing it to be used during the rest of the cloud-init. ephemeral0 is mounted as /mnt & then bind mounted to #{data_dir}
+
+#### :floating_ips
+This can be an array of strings, such that node X will be assigned :floating_ips[X-1] via "nova add-floating-ip".  If :floating_ips[X-1].nil?, then no floating ip will be attached.  (the floating ip must already be assigned to a pool in your account)
+
+#### :post_install_script, :post_install_args & :post_install_cwd
+These are used to construct a command to execute, which is executed locally where you executed stack-kicker.  :post_install_args can contain %PUBLIC_IP%, which will be replaced by the public IP of the just created node.  :post_install_script scripts are executed as soon the the instance returns a status='ACTIVE'.  They can be used delay the creation of further nodes of the same role (for example, when creating a rabbitmq cluster, you need to wait for the rabbitmq process to be running before creating the next member of the cluster, or when you are creating a chef-server, you need to wait for the packages to install & daemons to start before attempting to create Chef users & retrieve keys)
+
+
 ## Example workflows/models
 stack-kicker was built with the following workflows in mind:
 
@@ -39,11 +93,11 @@ stack-kicker was built with the following workflows in mind:
 This was the original requirement, a multi-role application stack build that started
 with building a chef-server, uploading roles, environments, cookbooks & databags to it,
 and then building the rest of the application-stack instances, using the freshly built chef-server
-to drop the application on to the instances.  In this setup we used vanilla images (Ubuntu 12.04 LTS, 
+to drop the application on to the instances.  In this setup we used vanilla images (Ubuntu 12.04 LTS,
 but you could use any image, either vanilla or pre-populated with your software).
 
 Here's an example Stackfile for this:
-	
+
 	module StackConfig
 		Stacks = {
 	    'web-w-chef-server' => {
@@ -53,14 +107,14 @@ Here's an example Stackfile for this:
 	      'PASSWORD'    => ENV['OS_PASSWORD'],
 	      'AUTH_URL'    => ENV['OS_AUTH_URL'],
 	      'TENANT_NAME' => ENV['OS_TENANT_NAME'],
-	
+
 	       # generic instance info
 	       'flavor_id'  => 103,
 	       'image_id'   => 75845,
 	       :key_pair => 'ssh-keypair-name',
 	       :key_public => '/path/to/id_rsa.pub',
 	       :global_service_name => 'perconaconf',
-	       
+
 	       # role details
 	       :roles => {
 	         # override the default cloud-init script & default bootstrap (which is a chef-client bootstrap)
@@ -70,13 +124,13 @@ Here's an example Stackfile for this:
 	           # override the default cloud-config with a chef-server template
 	           :cloud_config_yaml => 'chef-cloud-config.yaml',
 	           # skip the default chef-client bootstrap
-	           :bootstrap => '',                                 
+	           :bootstrap => '',
 	           # wait for the chef server to come up & download pem files & generate client account
 	           :post_install_script => 'bootstrap/chef-post-install.sh',
-	            # our post install script dumps out .pem files in the CWD   
-	           :post_install_cwd => '.chef',                    
+	            # our post install script dumps out .pem files in the CWD
+	           :post_install_cwd => '.chef',
 	           # The post-install script needs to know the public IP of the just built instance so that this station can access it
-	           :post_install_args => '%PUBLIC_IP%'               
+	           :post_install_args => '%PUBLIC_IP%'
 	         },
 	         # much simpler role, just build 3 of these, chef-client will do the rest on boot
 	         :web => {  :count => 3 }
@@ -84,34 +138,34 @@ Here's an example Stackfile for this:
 	    }
 	    }
 	end
-           
+
 
 ### simple roles
-There is no requirement that stack-kicker do anything other than spin up your instances, your requirements 
+There is no requirement that stack-kicker do anything other than spin up your instances, your requirements
 may be such that you just need a number of instances started with certain images, region & flavor requirements.
 
 ### masterless puppet
-aws-kicker (a sister project) had an original requirement of starting a simple 2-tier web application in multiple 
-locations/environments (prod, stage, dev etc), to do this we configured the instances by bootrapping puppet, 
+aws-kicker (a sister project) had an original requirement of starting a simple 2-tier web application in multiple
+locations/environments (prod, stage, dev etc), to do this we configured the instances by bootrapping puppet,
 git clonig /etc/puppet and running "puppet apply", a simple pattern used in many places, this was all achievd with a
 carefully crafted cloud-init template (incidentally, this also allowed for simple prototyping using vagrant to
 provide local instances using the exact same '/etc/puppet' git repo.
 
 ### Other workflows
-These are only the workflows I've used, there is no reason a puppet master couldn't be built & used, or 
+These are only the workflows I've used, there is no reason a puppet master couldn't be built & used, or
 hosted/external puppet & chef servers. (pull requests accepted etc, including salt, ansible, cfengine etc..)
 
 ## Installation
 
     $ gem install stack-kicker
-    
+
 ## Requirements
 In addition to the the ruby dependencies which gem will install for you, access to python-novaclient is currently required to attach floating-ips to instances.
 
 ## Usage
 
 	Usage: stack-kicker [options] task
-	
+
 	Options:
 	    -h, --help                       Show command line help
 	        --stackfile Stackfile        Specify an alternative Stackfile
@@ -122,7 +176,7 @@ In addition to the the ruby dependencies which gem will install for you, access
 	        --log-level LEVEL            Set the logging level
 	                                     (debug|info|warn|error|fatal)
 	                                     (Default: info)
-	
+
 	Arguments:
 
     task
@@ -130,13 +184,13 @@ In addition to the the ruby dependencies which gem will install for you, access
 
 ## TODO
 
-1. Clean up provider logic
+1. Clean/Add up provider logic
 2. Remove dependency on python-novaclient for floating-ip attach
 3. Remove dependency on a full chef gem install
 4. Better docs & examples
 5. Support for AWS EC2 (from aws-kicker)
 5. Support for DNS Updates on instance creation (from aws-kicker)
-6. 
+
 ## Contributing
 
 1. Fork it

data/doc/examples/.rvmrc

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p327@stack-kicker-examples"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.17.3 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi

data/doc/examples/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+# source 'http://15.185.118.80/partial-gem-mirror/'
+
+gem 'stack-kicker', '>=0.0.10'

data/doc/examples/apache-via-cloud-init/Stackfile

@@ -0,0 +1,40 @@
+module StackConfig
+    Stacks = Hash.new
+
+    Stacks['apache-cloud-init'] = {
+      # (we can access environment variable via ENV['foo'] instead of hard coding u/p here)
+      'REGION'      => ENV['OS_REGION_NAME'],
+      'USERNAME'    => ENV['OS_USERNAME'],
+      'PASSWORD'    => ENV['OS_PASSWORD'],
+      'AUTH_URL'    => ENV['OS_AUTH_URL'],
+      'TENANT_NAME' => ENV['OS_TENANT_NAME'],
+
+       # generic instance info
+       'flavor_id'  => 103,
+       'image_id'   => 75845, # Ubuntu Precise 12.04 LTS Server 64-bit
+       # per-az image_id's
+       'az-2.region-a.geo-1' => { 'image_id' => 67074 },
+
+       # provisioning info
+       :key_pair => 'dnsaas-dev@hp.com',
+       :key_public => '../credentials/ssh-keys/dnsaas-dev@hp.com.pub',
+
+       :name_template => '%s-%s-%s%04d',   # service-site-role0001
+       :global_service_name => 'webci',
+       :site_template => '%s',
+
+       # role specification
+       # role names & chef roles should match
+       :roles => {
+         :web => {  :count => 1,
+                    # use the default security group
+                    :security_group => 'default',
+                    # don't use chef as the second-stage provisioner
+                    :skip_chef_prereg => true,
+                    :bootstrap => 'cloud-init.sh',
+                    # use the yaml template that supports configuring the ephemeral space early in the boot process
+                    :cloud_config_yaml => 'cloud-init.yaml',
+         }
+       }
+    }
+end

data/doc/examples/apache-via-cloud-init/cloud-init.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cat > /var/www/index.html <<HEREDOC
+<html><body><h1>Stack-Kicker was 'ere!</h1>
+<p>This file is dropped in place by a shell script passed to the host via cloud-init</p>
+<p>The web server software is running but no content has been added, yet.</p>
+</body></html>
+HEREDOC

data/doc/examples/apache-via-cloud-init/cloud-init.yaml

@@ -0,0 +1,28 @@
+#cloud-config
+
+output:
+  all: ">> /var/log/cloud-init.log"
+
+# use the HPCS Ubuntu Mirror for security-updates too
+bootcmd:
+  - echo 127.0.1.1 %HOSTNAME% >> /etc/hosts
+  - echo # force security.ubuntu.com to mirror.clouds.archive.ubuntu.com >> /etc/hosts
+  - echo 15.185.107.200 security.ubuntu.com >> /etc/hosts
+
+# use the HPCS Ubuntu Mirrors
+apt_mirror: http://nova.clouds.archive.ubuntu.com/ubuntu
+
+# Run apt-get update
+package_update: true
+
+# Run apt-get update
+package_upgrade: true
+
+# Install some packages
+packages:
+ - apache2
+ - php5
+
+# Use `sudo -i` to simulate a login shell...
+runcmd:
+ - sudo -i touch /var/log/cloud-init.complete

data/lib/stack-kicker/version.rb

@@ -1,5 +1,5 @@
 module Stack
   module Kicker
-    VERSION = "0.0.10"
+    VERSION = "0.0.11"
   end
 end

data/lib/stack.rb

@@ -28,10 +28,10 @@ require 'tempfile'
 
 #
 # This really needs to be converted into a class....
-# 
+#
 module Stack
 
-  # Shadow the global constant Logger with Stack::Logger 
+  # Shadow the global constant Logger with Stack::Logger
   # (if you want access to the global constant, use ::Logger from inside the Stack module)
   Logger = Logger.new(STDOUT)
   Logger.level = ::Logger::INFO
@@ -60,7 +60,7 @@ module Stack
       Logger.info { "  #{name}" }
     end
   end
- 
+
   def Stack.show_stack(config)
     # syntax_check is a light weight check that doesn't talk to OpenStalk
     Stack.syntax_check(config)
@@ -76,7 +76,7 @@ module Stack
     eval(config_raw)
 
     # if there is only one stack defined in the Stackfile, load it:
-    if StackConfig::Stacks.count == 1 && stack_name.nil? 
+    if StackConfig::Stacks.count == 1 && stack_name.nil?
       stack_name = StackConfig::Stacks.keys[0]
       Logger.info { "Defaulting to #{stack_name} as there is a single stack defined and no stack named" }
     end
@@ -129,9 +129,9 @@ module Stack
 
   # check that all the required config items are set
   def Stack.syntax_check(config)
-    if config['REGION'].nil? || config['USERNAME'].nil? || config['PASSWORD'].nil? || config['AUTH_URL'].nil? || config['TENANT_NAME'].nil? && 
+    if config['REGION'].nil? || config['USERNAME'].nil? || config['PASSWORD'].nil? || config['AUTH_URL'].nil? || config['TENANT_NAME'].nil? &&
        config['REGION'].empty? || config['USERNAME'].empty? || config['PASSWORD'].empty? || config['AUTH_URL'].empty? || config['TENANT_NAME'].empty?
-      Logger.error { "REGION, USERNAME, PASSWORD, AUTH_URL & TENANT_NAME must all be set" } 
+      Logger.error { "REGION, USERNAME, PASSWORD, AUTH_URL & TENANT_NAME must all be set" }
       exit
     end
 
@@ -146,7 +146,7 @@ module Stack
       if !File.directory?(dot_chef_abs)
         Logger.warn "#{dot_chef_abs} doesn't exist"
       end
-      
+
       # Check we have a #{dot_chef_abs}/.chef/knife.rb
       knife_rb_abs = dot_chef_abs + '/knife.rb'
       if File.exists?(knife_rb_abs)
@@ -175,7 +175,7 @@ module Stack
         Logger.error "Couldn't find #{config[:key_pair]} key in the ssh-agent key list! Aborting!"
         Logger.erroLogger.error "ssh_keys_loaded: #{ssh_keys_loaded}"
         exit 2
-      end 
+      end
     end
 
     # populate the config & then walk through the AZs verifying the config
@@ -199,7 +199,7 @@ module Stack
       keypairs = os.keypairs()
       if (keypairs[config[:key_pair]].nil? && keypairs[config[:key_pair].to_sym].nil?)
         Logger.warn "#{config[:key_pair]} isn't available, uploading the key"
-        
+
         # upload the key
         key =  os.create_keypair({:name=> config[:key_pair], :public_key=> File.read(config[:key_public])})
         Logger.warn "#{config[:key_pair]} fingerprint=#{key[:fingerprint]}"
@@ -216,10 +216,10 @@ module Stack
 
       config[:roles].each do |role, role_details|
         # is does the secgroup exist?
-        if sg_names.include?(role.to_s)
-          Logger.info "security group #{role} exists in #{az}"
+        if sg_names.include?(role_details[:security_group])
+          Logger.info "security group #{role_details[:security_group]} exists in #{az}"
         else
-          Logger.error "security group #{role} is missing in #{az}"
+          Logger.error "security group #{role_details[:security_group]} is missing in #{az}"
         end
       end
     end
@@ -251,7 +251,7 @@ module Stack
 
     client_key = dot_chef_abs + '/' + config[:name] + '-' + ENV['USER'] + '.pem'
     validation_key = dot_chef_abs + '/' + config[:name] + '-' + 'validation.pem'
-  
+
     Logger.debug "stackhome: #{config[:stackhome]}"
     Logger.debug "Current user client key: #{client_key}"
     Logger.debug "New Host Validation key: #{validation_key}"
@@ -290,11 +290,11 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
     Logger.debug role_details[:azs]
 
     site = sprintf(config[:site_template], role_details[:azs][position-1].split('.')[0].sub(/-/, ''))
-    
+
     # generate the hostname
     hostname = sprintf(config[:name_template], config[:global_service_name], site, role, position)
 
-    hostname 
+    hostname
   end
 
   def Stack.generate_server_names(config)
@@ -305,7 +305,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
 
   def Stack.populate_config(config)
     # config[:role_details] contains built out role details with defaults filled in from stack defaults
-    # config[:node_details] contains node details built out from role_details 
+    # config[:node_details] contains node details built out from role_details
 
     # set some sensible defaults to the stack-wide defaults if they haven't been set in the Stackfile.
     if config[:provisioner].nil?
@@ -330,7 +330,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
 
     if config[:name_template].nil?
       Logger.warn { "Defaulting to '%s-%s-%s%04d' for config[:name_template]" }
-      config[:name_template] = '%s-%s-%s%04d' 
+      config[:name_template] = '%s-%s-%s%04d'
     end
 
     if config[:site_template].nil?
@@ -343,24 +343,24 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
       config[:site_template] = 'UNKNOWN'
     end
 
-    
+
     if config[:node_details].nil?
       Logger.debug { "Initializing config[:node_details] and config[:azs]" }
       config[:node_details] = Hash.new
       config[:azs] = Array.new
 
-      config[:roles].each do |role,role_details| 
+      config[:roles].each do |role,role_details|
         Logger.debug { "Setting defaults for #{role}" }
-        
+
         # default to 1 node of this role if :count isn't set
         if role_details[:count].nil?
           role_details[:count] = 1
         end
-    
+
         if (role_details[:data_dir].nil?)
           role_details[:data_dir] = '/dummy'
         end
-        
+
         # Has the cloud_config_yaml been overridden?
         if (role_details[:cloud_config_yaml])
           role_details[:cloud_config_yaml] = Stack.find_file(config, role_details[:cloud_config_yaml])
@@ -390,24 +390,28 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
           role_details[:post_install_cwd] = '/.'
         end
 
+        if role_details[:post_install_args].nil?
+          role_details[:post_install_args] = ''
+        end
+
         (1..role_details[:count]).each do |p|
           Logger.debug { "Populating the config[:role_details][:azs] array with AZ" }
           role_details[:azs] = Array.new if role_details[:azs].nil?
-          
+
           # is there an az set for this node?
           if role_details[:azs][p-1].nil?
-            # inherit the global az 
+            # inherit the global az
             Logger.debug { "Inheriting the AZ for #{role} (#{config['REGION']})" }
             role_details[:azs][p-1] = config['REGION']
           end
-          
+
           # add this AZ to the AZ list, we'll dedupe later
           config[:azs] << role_details[:azs][p-1]
-          
+
           hostname =  Stack.generate_hostname(config, role, p)
           Logger.debug { "Setting node_details for #{hostname}, using element #{p}-1 from #{role_details[:azs]}" }
           config[:node_details][hostname] = { :az => role_details[:azs][p-1], :region => role_details[:azs][p-1], :role => role }
-        end 
+        end
       end
     end
     config[:azs].uniq!
@@ -429,18 +433,18 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
     if config[:all_instances].nil? || refresh
       # we need to get the server list for each AZ mentioned in the config[:roles][:role][:azs], this is populated by Stack.populate_config
       Stack.populate_config(config)
-      
+
       # get the current list of servers from OpenStack & generate a hash, keyed on name
       servers = Hash.new
       config[:azs].each do |az|
         os = Stack.connect(config, az)
-        os.servers.each do |server| 
-          servers[server[:name]] = { 
+        os.servers.each do |server|
+          servers[server[:name]] = {
             :region => az,
-            :id => server[:id], 
+            :id => server[:id],
             :addresses => os.server(server[:id]).addresses
           }
-        end 
+        end
       end
       config[:all_instances] = servers
     end
@@ -460,9 +464,9 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
   def Stack.add_instance(config, hostname, region, id, addresses)
     config[:all_instances][hostname] = { :region => region, :id => id, :addresses => addresses}
   end
- 
+
   def Stack.ssh(config, hostname = nil, user = ENV['USER'], command = nil)
-    # ssh to a host, or all hosts 
+    # ssh to a host, or all hosts
 
     # get all running instances
     servers = Stack.get_our_instances(config)
@@ -523,10 +527,10 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
     # this also populates out unspecified defaults, like az
     Stack.populate_config(config)
 
-    # get the list of nodes we consider 'ours', i.e. with hostnames that match 
+    # get the list of nodes we consider 'ours', i.e. with hostnames that match
     # those generated by this stack
     ours = Stack.get_our_instances(config)
-  
+
     # do any of the list of servers in OpenStack match one of our hostnames?
     ours.each do |node, node_details|
         Logger.info "Deleting #{node}"
@@ -535,9 +539,9 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
         d.delete!
     end
   end
- 
+
   def Stack.get_public_ip(config, hostname)
-    # get a public address from the instance 
+    # get a public address from the instance
     # (could be either the dynamic or one of our floating IPs
     config[:all_instances][hostname][:addresses].each do |address|
       if address.label == 'public'
@@ -547,7 +551,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
   end
 
   def Stack.set_chef_server(config, chef_server)
-    # set the private & public URLs for the chef server, 
+    # set the private & public URLs for the chef server,
     # called either after we create the Chef Server, or skip over it
     Logger.debug "Setting :chef_server_hostname, chef_server_private & chef_server_public details (using #{chef_server})"
 
@@ -575,18 +579,18 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
     # 2) generate the json to describe that to the "stackhelper secgroup-sync" tool
     # 3) run "stackhelper secgroup-sync --some-file our-ips.json"
     ours = Stack.get_our_instances(config)
-  
+
     secgroup_ips = Hash.new
     # walk the list of hosts, dumping the IPs into role buckets
     ours.each do |instance, instance_details|
       secgroup_ips[instance_details[:role]] = Array.new if secgroup_ips[instance_details[:role]].nil?
 
       #secgroup_ips[instance_details[:role]] << instance_details[:addresses].map { |address| address.address }
-      secgroup_ips[instance_details[:role]] << instance_details[:addresses].map do |address| 
-        if (address.label == 'public') 
-          address.address 
-        else 
-          next 
+      secgroup_ips[instance_details[:role]] << instance_details[:addresses].map do |address|
+        if (address.label == 'public')
+          address.address
+        else
+          next
         end
       end
 
@@ -594,7 +598,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
       secgroup_ips[instance_details[:role]].flatten!
 
       # delete any nil's that we collected due to skipping private ips
-      secgroup_ips[instance_details[:role]].delete_if {|x| x.nil? } 
+      secgroup_ips[instance_details[:role]].delete_if {|x| x.nil? }
     end
 
     # dump the json to a temp file
@@ -618,20 +622,20 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
   # if we're passed a role, only deploy this role.
   def Stack.deploy_all(config, role_to_deploy = nil)
     Stack.validate(config)
-    
+
     # this also populates out unspecified defaults, like az
     node_details = Stack.populate_config(config)
     # get info about all instances running in our account & AZs
     servers = Stack.get_all_instances(config)
 
     # this is our main loop iterator, generates each host
-    config[:roles].each do |role,role_details| 
+    config[:roles].each do |role,role_details|
       Logger.debug { "Iterating over roles, this is #{role}, role_details = #{role_details}" }
 
       (1..role_details[:count]).each do |p|
         hostname = Stack.generate_hostname(config, role, p)
         Logger.debug { "Iterating over nodes in #{role}, this is #{hostname}" }
-        
+
         # configure the global :chef_server details if this the chef server
         if role_details[:chef_server]
           Stack.set_chef_server(config, hostname)
@@ -642,7 +646,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
           Logger.info { "#{hostname} already exists, skipping.." }
           next
         end
-      
+
         Logger.debug { "Deploying #{role}, role_to_deploy = #{role_to_deploy}" }
         if ((role_to_deploy.nil?) || (role_to_deploy.to_s == role.to_s))
           if (role_details[:skip_chef_prereg] == true || role_details[:chef_server])
@@ -654,9 +658,9 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
             knife_client_list.sub!(/\s/,'')
             if knife_client_list.length() > 0
               # we should delete the client to make way for this new machine
-              Logger.info `knife client delete --yes #{hostname}`              
+              Logger.info `knife client delete --yes #{hostname}`
             end
-            
+
             # knife node create -d --environment $CHEF_ENVIRONMENT $SERVER_NAME
             # knife node run_list add -d --environment $CHEF_ENVIRONMENT $SERVER_NAME "role[${ROLE}]"
             # this relies on .chef matching the stacks config (TODO: poke the Chef API directly?)
@@ -680,40 +684,47 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
           multipart_cmd = "#{libdir}/write-mime-multipart #{role_details[:bootstrap]} #{role_details[:cloud_config_yaml]}"
           Logger.debug { "multipart_cmd = #{multipart_cmd}" }
           multipart = `#{multipart_cmd}`
+          Logger.debug { "multipart = #{multipart}" }
           # 2) replace the tokens (CHEF_SERVER, CHEF_ENVIRONMENT, SERVER_NAME, ROLE)
+          Logger.debug { "Replacing %HOSTNAME% with #{hostname} in multipart" }
           multipart.gsub!(%q!%HOSTNAME%!, hostname)
 
-          Logger.info  "Chef server is #{config[:chef_server_hostname]}, which is in #{config[:node_details][config[:chef_server_hostname]][:region]}"
-          Logger.info  "#{hostname}'s region is #{config[:node_details][hostname][:region]}"
-          # if this host is in the same region/az, use the private URL, if not, use the public url
-          if (config[:node_details][hostname][:region] == config[:node_details][config[:chef_server_hostname]][:region]) && !config[:chef_server_private].nil?
-            multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_private])
-          elsif !config[:chef_server_public].nil?
-            multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_public])
+          if config[:chef_server_hostname].nil?
+            Logger.info  "config[:chef_server_hostname] is nil, skipping chef server substitution"
           else
-            Logger.warn { "Not setting the chef url for #{hostname} as neither chef_server_private or chef_server_public are valid yet" }
-          end
-          multipart.gsub!(%q!%CHEF_ENVIRONMENT%!, config[:chef_environment])
-          if File.exists?(config[:chef_validation_pem])
-            multipart.gsub!(%q!%CHEF_VALIDATION_PEM%!, File.read(config[:chef_validation_pem]))
-          else
-            Logger.warn "Skipping #{config[:chef_validation_pem]} substitution in user-data"
+            Logger.info  "Chef server is #{config[:chef_server_hostname]}, which is in #{config[:node_details][config[:chef_server_hostname]][:region]}"
+            Logger.info  "#{hostname}'s region is #{config[:node_details][hostname][:region]}"
+            # if this host is in the same region/az, use the private URL, if not, use the public url
+            if (config[:node_details][hostname][:region] == config[:node_details][config[:chef_server_hostname]][:region]) && !config[:chef_server_private].nil?
+              multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_private])
+            elsif !config[:chef_server_public].nil?
+              multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_public])
+            else
+              Logger.warn { "Not setting the chef url for #{hostname} as neither chef_server_private or chef_server_public are valid yet" }
+            end
+            multipart.gsub!(%q!%CHEF_ENVIRONMENT%!, config[:chef_environment])
+            if File.exists?(config[:chef_validation_pem])
+              multipart.gsub!(%q!%CHEF_VALIDATION_PEM%!, File.read(config[:chef_validation_pem]))
+            else
+              Logger.warn "Skipping #{config[:chef_validation_pem]} substitution in user-data"
+            end
           end
+
           multipart.gsub!(%q!%SERVER_NAME%!, hostname)
           multipart.gsub!(%q!%ROLE%!, role.to_s)
           multipart.gsub!(%q!%DATA_DIR%!, role_details[:data_dir])
 
           Logger.info "Creating #{hostname} in #{node_details[hostname][:az]} with role #{role}"
 
-          # this will get put in /meta.js 
+          # this will get put in /meta.js
           metadata = { 'region' => node_details[hostname][:az], 'chef_role' => role }
 
           os = Stack.connect(config, node_details[hostname][:az])
-          newserver = os.create_server(:name => hostname, 
-                                       :imageRef => config[node_details[hostname][:az]]['image_id'], 
+          newserver = os.create_server(:name => hostname,
+                                       :imageRef => config[node_details[hostname][:az]]['image_id'],
                                        :flavorRef => config['flavor_id'],
                                        :security_groups=>[role_details[:security_group]],
-                                       :user_data => Base64.encode64(multipart), 
+                                       :user_data => Base64.encode64(multipart),
                                        :metadata => metadata,
                                        :key_name => config[:key_pair])
 
@@ -742,9 +753,9 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
             Logger.info "Attaching #{floating_ip} to  #{hostname}\n via 'nova add-floating-ip'"
             # nova --os-region-name $REGION add-floating-ip $SERVER_NAME $FLOATING_IP
             floating_ip_add = `nova --os-region-name #{node_details[hostname][:az]} add-floating-ip #{hostname} #{floating_ip}`
-            Logger.info floating_ip_add 
-          end 
-          
+            Logger.info floating_ip_add
+          end
+
           # refresh the secgroups ASAP
           Stack.secgroup_sync(config)
 
@@ -754,7 +765,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
             # convert when we got passed to an absolute path
             post_install_script_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_script])
             post_install_cwd_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_cwd])
-            
+
             # replace any tokens in the argument
             public_ip = Stack.get_public_ip(config, hostname)
             role_details[:post_install_args].sub!(%q!%PUBLIC_IP%!, public_ip)
@@ -766,7 +777,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
         else
           Logger.info "Skipped role #{role}"
         end
-      end 
+      end
     end
   end
 
@@ -775,7 +786,7 @@ cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
     # 1) cwd
     # 2) stackhome
     # 3) gemhome/lib
-    dirs = [ './' ] 
+    dirs = [ './' ]
     dirs.push(config[:stackhome])
     dirs.push(@@gemhome + '/lib')
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stack-kicker
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-22 00:00:00.000000000 Z
+date: 2013-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -133,13 +133,19 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rvmrc
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - README.rdoc
 - Rakefile
 - bin/stack-kicker
-- doc/examples/Stackfile
+- doc/examples/.rvmrc
+- doc/examples/Gemfile
+- doc/examples/apache-via-chef-server/Stackfile
+- doc/examples/apache-via-cloud-init/Stackfile
+- doc/examples/apache-via-cloud-init/cloud-init.sh
+- doc/examples/apache-via-cloud-init/cloud-init.yaml
 - features/stack-kicker.feature
 - features/step_definitions/stack-kicker_steps.rb
 - features/support/env.rb