stack-kicker 0.0.1 → 0.0.2

data/lib/stack.rb

@@ -1,212 +1,787 @@
-require 'pp'
-require 'fog'
+#!/usr/bin/env ruby
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+# Author: Simon McCartney <simon.mccartney@hp.com>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+require 'bundler'
+require 'rubygems'
 
+require 'base64'
+require 'erb'
+require 'openstack'     # https://github.com/ruby-openstack/ruby-openstack
+require 'json'          # http://www.ruby-doc.org/stdlib-1.9.3/libdoc/json/rdoc/JSON.html
+require 'tempfile'
+
+
+#
+# This really needs to be converted into a class....
+# 
 module Stack
 
-  def Stack.load_config(configfile, stack)
-    config_raw = File.read(configfile)
-    eval(config_raw) 
-    config = StackConfig::Stacks[stack]
-    config
+  # Shadow the global constant Logger with Stack::Logger 
+  # (if you want access to the global constant, use ::Logger from inside the Stack module)
+  Logger = Logger.new(STDOUT)
+  Logger.level = ::Logger::INFO
+  Logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+  Logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime} #{severity}: #{msg}\n"
   end
 
-  def Stack.connect(config)
-    connection = Fog::Compute.new({ :provider => config[:provider],
-                                    :aws_access_key_id => config[:aws_access_key_id],
-                                    :aws_secret_access_key => config[:aws_secret_access_key],
-                                    :region => config[:region] })
-    connection
+  # location of gem, where config[:gemhome]/lib contains our default cloud-init templates
+  @@gemhome = File.absolute_path(File.realpath(File.dirname(File.expand_path(__FILE__)) + '/..'))
+
+  # Methadone::CLILogger is a Class, Stack is still a module, so we can't include it
+  # so this is a QADH to propagate the log_level
+  def Stack.log_level(level)
+    Logger.debug { "Setting the Logger.level to #{level}" }
+    Logger.level = level
   end
 
-  def Stack.populate_config(config)
-    # build out the full config for each node, supplying defaults from the 
-    # global config if explicitly supplied
-    config[:node_details] = Hash.new if config[:node_details].nil?
+  def Stack.show_stacks(stackfile = 'Stackfile')
+    # our local config file
+    config_raw = File.read(stackfile)
+    eval(config_raw)
 
-    config[:roles].each do |role, role_details|
-      fqdn = role.to_s + '.' + config[:dns_domain]
-      
-      config[:node_details][fqdn] = { 
-        # set the node details from the role, if not specified in the role, use the config global
-        # (takes advantage of left to right evaluation of ||)
-        :flavor_id          => (role_details[:flavor_id] || config[:flavor_id]),
-        :count              => (role_details[:count] || 1),
-        :publish_private_ip => (role_details[:publish_private_ip] || false),
-        :dns_wildcard       => (role_details[:dns_wildcard] || false)
-      }
+    Logger.info { "Stacks:" }
+    StackConfig::Stacks.each do |name, details|
+      Logger.info { "  #{name}" }
     end
   end
+ 
+  def Stack.show_stack(config)
+    # syntax_check is a light weight check that doesn't talk to OpenStalk
+    Stack.syntax_check(config)
+    # generate an array of hostnames that this stack would create
+    hostnames = Stack.generate_server_names(config)
 
-  def Stack.generate_hostnames(config)
-    stack_hostnames = Array.new
-    config[:roles].each do |role, role_details|
-      fqdn = role.to_s + '.' + config[:dns_domain]
-      stack_hostnames << fqdn
-    end
-    stack_hostnames 
+    hostnames.each { |hostname| Logger.info "  #{hostname}" }
   end
 
-  def Stack.deploy_all(config)
-    # create a connection
-    connection = Stack.connect(config)
+  def Stack.select_stack(stackfile = 'Stackfile', stack_name)
+    # our local config file
+    config_raw = File.read(stackfile)
+    eval(config_raw)
 
-    running_instances = Stack.get_running(config)
-    config[:roles].each do |role, role_details|
-      hostname = role.to_s
-      fqdn = role.to_s + '.' + config[:dns_domain]
+    # if there is only one stack defined in the Stackfile, load it:
+    if StackConfig::Stacks.count == 1 && stack_name.nil? 
+      stack_name = StackConfig::Stacks.keys[0]
+      Logger.debug { "defaulting to #{stack_name} as there is a single stack defined and no stack named" }
+    end
 
-      if !running_instances[fqdn].nil?
-        puts "Skipping #{fqdn} as it already exists"
-        next
+    # returns a config object, injecting the name into the returned config
+    if StackConfig::Stacks[stack_name].nil?
+      Logger.error { "#{stack_name} is invalid, defined stacks are:" }
+      StackConfig::Stacks.each do |name, details|
+        Logger.error { "  #{name}" }
       end
+      exit 2
+    end
+
+    config = StackConfig::Stacks[stack_name]
+    config[:name] = stack_name
+    # set the stackhome to the directory containing the Stackfile
+    config[:stackhome] = File.dirname(File.expand_path(stackfile))
+    Logger.info "stackhome is #{config[:stackhome]}"
+
+    config
+  end
+
+  def Stack.connect(config, region = nil)
+    # region & az concepts are confused in HPCS land
+    region = config['REGION'] if (region.nil? || region.length() < 1)
+
+    Logger.info "Connecting to OpenStack with region = #{region}"
+
+    OpenStack::Connection.create({
+                                  :auth_method=> 'password',
+                                  :username => config['USERNAME'],
+                                  :api_key=> config['PASSWORD'],
+                                  :auth_url => config['AUTH_URL'],
+                                  :authtenant_name => config['TENANT_NAME'],
+                                  :region => region,
+                                  :service_type=>"compute"
+                                  })
+  end
+
+  # expects server to be OpenStack::Compute::Server object
+  def Stack.get_addresses(server)
+
+    # get the addressess associated with an OpenStack::Compute::Server object
+    address_description = String.new
+    server.addresses.each do |address|
+      address_description << "#{address.address}(#{address.label}) "
+    end
+    address_description
+  end
+
+  # check that all the required config items are set
+  def Stack.syntax_check(config)
+    if config['REGION'].nil? || config['USERNAME'].nil? || config['PASSWORD'].nil? || config['AUTH_URL'].nil? || config['TENANT_NAME'].nil? && 
+       config['REGION'].empty? || config['USERNAME'].empty? || config['PASSWORD'].empty? || config['AUTH_URL'].empty? || config['TENANT_NAME'].empty?
+      Logger.error { "REGION, USERNAME, PASSWORD, AUTH_URL & TENANT_NAME must all be set" } 
+      exit
+    end
 
-      # Ubuntu 8.04/Hardy doesn't do full cloud-init, so we have to script setting the hostname
-      user_data = File.read('user-data.sh')
-      user_data.gsub!(/rentpro-unconfigured/, hostname)
-      user_data.gsub!(/rentpro-stage.local/, config[:dns_domain])
-
-      # pp multipart
-      #
-      puts "Bootstraping new instance - #{fqdn}"
-      server = connection.servers.create({ 
-                                        :name => fqdn,
-                                        :hostname => fqdn,
-                                        :availability_zone => config[:availability_zone], 
-                                        :flavor_id => config[:node_details][fqdn],
-                                        :image_id => config[:image_id],
-                                        :key_name => config[:keypair],
-                                        :user_data => user_data,
-                                        :tags => { 'Name' => fqdn },
-                                        })
-
-      print "Waiting for instance to be ready..."
-      server.wait_for { ready? }
-      puts "#{role.to_s} is booted, #{server.public_ip_address}/#{server.private_ip_address}"
-
-      # create/update the public & private DNS for this host
-      Stack.update_dns(role.to_s + '-public.' + config[:dns_domain], server.public_ip_address, config)
-      Stack.update_dns(role.to_s + '-private.' + config[:dns_domain], server.private_ip_address, config)
+    # load defaults for any items not configured
+    Stack.populate_config(config)
+
+    if config[:provisioner] == 'chef'
+      # check that we have semi-sensible Chef setup
+      # at a bare minimum, we need the directory where we're going to download
+      # validation.pem to to exist
+      dot_chef_abs = File.absolute_path(config[:stackhome] + '/' + config[:dot_chef])
+      if !File.directory?(dot_chef_abs)
+        Logger.warn "#{dot_chef_abs} doesn't exist"
+      end
       
-      # create the dns
-      if (role_details[:publish_private_ip] == true && (!role_details[:publish_private_ip].nil?))
-        ip_address = server.private_ip_address
+      # Check we have a #{dot_chef_abs}/.chef/knife.rb
+      knife_rb_abs = dot_chef_abs + '/knife.rb'
+      if File.exists?(knife_rb_abs)
+        Logger.info "Found #{knife_rb_abs}, lets hope it contains something sensible"
       else
-        ip_address = server.public_ip_address
-      end
-      Stack.update_dns(fqdn, ip_address, config)
-      #
-      # is this a wildcard DNS host, then claim the *.domain.net
-      if (role_details[:dns_wildcard] == true && (!role_details[:dns_wildcard].nil?))
-        wildcard = "*." + config[:dns_domain]
-        Stack.update_dns(wildcard, ip_address, config)
+        Logger.warn "#{knife_rb_abs} doesn't exist, please run './stack.rb configure-knife <stack-name>'"
       end
     end
   end
 
-  def Stack.update_dns(fqdn, ip_address, config)
-    # now register it in DNS
-    dns = Fog::DNS.new({ :provider => config[:provider],
-                          :aws_access_key_id => config[:aws_access_key_id],
-                          :aws_secret_access_key => config[:aws_secret_access_key] })
-          
-    # pp dns.get_hosted_zone(config[:dns_id])
+  # validate that all our OpenStack creds, image_id, flavors, keys etc are valid
+  def Stack.validate(config)
+
+    Stack.syntax_check(config)
+
+    # check that the ssh-key is loaded, otherwise most post-install scripts will fail
+    # this lazily assumes that the :key_pair name matches the file the keys were loaded
+    # from
+    if (0 == 1)
+      ssh_keys_loaded = `ssh-add -L`
+      Logger.debug "ssh_keys_loaded: #{ssh_keys_loaded}"
+      Logger.debug "Looking for #{config[:key_pair]}"
+      if ssh_keys_loaded.include?(config[:key_pair])
+        Logger.info "Found #{config[:key_pair]} in the ssh-agent key list"
+      else
+        Logger.error "Couldn't find #{config[:key_pair]} key in the ssh-agent key list! Aborting!"
+        Logger.erroLogger.error "ssh_keys_loaded: #{ssh_keys_loaded}"
+        exit 2
+      end 
+    end
+
+    # populate the config & then walk through the AZs verifying the config
+    Stack.populate_config(config)
+
+    # Check that we have valid details for each AZ
+    config[:azs].each do |az|
+
+      # check that credentials, flavor & image are valid
+      os = connect(config, az)
+
+      Logger.info "Checking that flavor #{config['flavor_id']} exists in #{az}..."
+      flavor = os.get_flavor(config['flavor_id'])
+      Logger.info "#{config['flavor_id']} is #{flavor.name}"
+
+      Logger.info "Checking that image #{config[az]['image_id']} exists in #{az}..."
+      image = os.get_image(config[az]['image_id'])
+      Logger.info "#{config[az]['image_id']} is #{image.name}"
+
+      Logger.info "Checking that keypair #{config[:key_pair]} exists in #{az}...."
+      keypairs = os.keypairs()
+      if (keypairs[config[:key_pair]].nil? && keypairs[config[:key_pair].to_sym].nil?)
+        Logger.warn "#{config[:key_pair]} isn't available, uploading the key"
         
-    bmtw =  dns.zones.get(config[:dns_id])
-      
-    record = bmtw.records.get(fqdn) 
-    if record 
-      record.modify(:value => ip_address) if record 
-    else
-      bmtw.records.create(:value => ip_address, :name => fqdn, :type => 'A')
+        # upload the key
+        key =  os.create_keypair({:name=> config[:key_pair], :public_key=> File.read(config[:key_public])})
+        Logger.warn "#{config[:key_pair]} fingerprint=#{key[:fingerprint]}"
+      else
+        Logger.info "#{config[:key_pair]} fingerprint=#{keypairs[config[:key_pair].to_sym][:fingerprint]}"
+      end
+
+      # TODO: check that security group exists
+      # we should have a security group that matches each role
+      # get all the secgroups
+      security_groups = os.security_groups()
+      # extract the names
+      sg_names = security_groups.map { |secgroup, secgroup_details| secgroup_details[:name] }
+
+      config[:roles].each do |role, role_details|
+        # is does the secgroup exist?
+        if sg_names.include?(role.to_s)
+          Logger.info "security group #{role} exists in #{az}"
+        else
+          Logger.error "security group #{role} is missing in #{az}"
+        end
+      end
     end
   end
 
-  def Stack.show_dns(config)
-    # now register it in DNS
-    dns = Fog::DNS.new({ :provider => config[:provider],
-                          :aws_access_key_id => config[:aws_access_key_id],
-                          :aws_secret_access_key => config[:aws_secret_access_key] })
-          
-    zone = dns.zones.get(config[:dns_id])
-    if zone.records.empty?
-      puts "No DNS records found in #{config[:dns_domain]}"
-    else
-      printf("%40s %20s %5s %5s\n", 'fqdn', 'value', 'type', 'ttl')
-      zone.records.each do |record|
-        printf("%40s %20s %5s %5d\n", record.name, record.value, record.type, record.ttl)
+  def Stack.generate_knife_rb(config)
+    # generate a project/.chef/knife.rb from our config
+    # (assumes the chef server is running for public IP access etc)
+
+    # find the chef server, if we need to
+    if config[:chef_server_hostname].nil? || config[:chef_server_private].nil? || config[:chef_server_public]
+      Logger.debug { "Attempting to discover the chef server details" }
+      ours = Stack.get_our_instances(config)
+      ours.each do |node, node_details|
+        if node_details[:role] == :chef
+          Logger.debug { "Found the Chef server: #{node} #{node_details}" }
+          Stack.set_chef_server(config, node)
+        end
       end
-    end 
-  end
+    end
+
+    # CWD shoud be chef-repo/bootstrap, so the project .chef directory should be
+    dot_chef_abs = File.absolute_path(config[:stackhome] + '/' + config[:dot_chef])
+
+    if !File.directory?(dot_chef_abs)
+      Logger.warn "#{dot_chef_abs} doesn't exist, creating it..."
+      Dir.mkdir(dot_chef_abs)
+    end
 
-  def Stack.get_running(config)
-    # create a connection
-    connection = Stack.connect(config)
+    client_key = dot_chef_abs + '/' + config[:name] + '-' + ENV['USER'] + '.pem'
+    validation_key = dot_chef_abs + '/' + config[:name] + '-' + 'validation.pem'
   
-    # generate all the names that this stack will use
-    stack_hostnames = Stack.generate_hostnames(config)
-
-    # Amazon EC2, use the tags hash to find hostnames
-    running_instances = Hash.new
-    connection.servers.each do |instance|
-      # pp instance
-      if (!instance.tags['Name'].nil? && instance.state != 'terminated' && instance.state != 'shutting-down')
-        hostname = instance.tags['Name']
-        if stack_hostnames.include?(hostname)
-          running_instances[hostname] = instance
+    Logger.debug "stackhome: #{config[:stackhome]}"
+    Logger.debug "Current user client key: #{client_key}"
+    Logger.debug "New Host Validation key: #{validation_key}"
+
+    knife_rb_template = %q{
+log_level                :info
+log_location             STDOUT
+node_name                '<%=ENV['USER']%>'
+# use the HPCS_ENV environment name to pick the correct key
+client_key               '<%=dot_chef_abs%>/' + ENV['HPCS_ENV'] + '-' + ENV['USER'] + '.pem'
+validation_client_name   'chef-validator'
+validation_key           '<%=dot_chef_abs%>/' + ENV['HPCS_ENV'] + '-validation.pem'
+chef_server_url          '<%=config[:chef_server_public]%>'
+cache_type               'BasicFile'
+cache_options( :path =>  '<%=dot_chef_abs%>/checksums' )
+cookbook_path [ '<%=config[:stackhome]%>/cookbooks' ]
+    }
+
+    knife_rb_erb = ERB.new(knife_rb_template)
+    knife_rb = knife_rb_erb.result(binding)
+
+    krb = File.new(dot_chef_abs + '/knife.rb', "w")
+    krb.truncate(0)
+    krb.puts knife_rb
+    krb.close
+  end
+
+  # position is really the node number in a role, i.e. 1..count
+  def Stack.generate_hostname(config, role, position)
+    role_details = config[:roles][role]
+
+    # TODO: don't calculate this everytime, shift out to a hash lookup
+    Logger.debug config
+    Logger.debug config[:site_template]
+    Logger.debug role_details
+    Logger.debug role_details[:azs]
+
+    site = sprintf(config[:site_template], role_details[:azs][position-1].split('.')[0].sub(/-/, ''))
+    
+    # generate the hostname
+    hostname = sprintf(config[:name_template], config[:global_service_name], site, role, position)
+
+    hostname 
+  end
+
+  def Stack.generate_server_names(config)
+    Stack.populate_config(config)
+    config[:hostnames] = config[:node_details].keys
+    config[:hostnames]
+  end
+
+  def Stack.populate_config(config)
+    # config[:role_details] contains built out role details with defaults filled in from stack defaults
+    # config[:node_details] contains node details built out from role_details 
+
+    # set some sensible defaults to the stack-wide defaults if they haven't been set in the Stackfile.
+    if config[:provisioner].nil?
+      Logger.warn { "Defaulting to chef for config[:provisioner] "}
+      config[:provisioner] = 'chef'
+    end
+
+    if config[:dot_chef].nil?
+      Logger.warn { "Defaulting to .chef for config[:dot_chef] "}
+      config[:dot_chef] = '.chef'
+    end
+
+    if config[:chef_environment].nil?
+      Logger.warn { "Defaulting to _default for config[:chef_environment]" }
+      config[:chef_environment] = '_default'
+    end
+
+    if config[:chef_validation_pem].nil?
+      Logger.warn { "Defaulting to .chef/validation.pem for config[:chef_validation_pem]" }
+      config[:chef_validation_pem] = '.chef/validation.pem'
+    end
+
+    if config[:name_template].nil?
+      Logger.warn { "Defaulting to '%s-%s-%s%04d' for config[:name_template]" }
+      config[:name_template] = '%s-%s-%s%04d' 
+    end
+
+    if config[:site_template].nil?
+      Logger.warn { "Defaulting to '%s' for config[:site_template]" }
+      config[:site_template] = '%s'
+    end
+
+    if config[:global_service_name].nil?
+      Logger.error { "Defaulting to 'UNKNOWN' for config[:global_service_name]" }
+      config[:site_template] = 'UNKNOWN'
+    end
+
+    
+    if config[:node_details].nil?
+      Logger.debug { "Initializing config[:node_details] and config[:azs]" }
+      config[:node_details] = Hash.new
+      config[:azs] = Array.new
+
+      config[:roles].each do |role,role_details| 
+        Logger.debug { "Setting defaults for #{role}" }
+        
+        # default to 1 node of this role if :count isn't set
+        if role_details[:count].nil?
+          role_details[:count] = 1
         end
+    
+        if (role_details[:data_dir].nil?)
+          role_details[:data_dir] = '/dummy'
+        end
+        
+        # Has the cloud_config_yaml been overridden?
+        if (role_details[:cloud_config_yaml])
+          role_details[:cloud_config_yaml] = Stack.find_file(config, role_details[:cloud_config_yaml])
+        else
+          role_details[:cloud_config_yaml] = Stack.find_file(config, 'cloud-config.yaml')
+        end
+
+        # Has the default bootstrap script been overridden
+        if (role_details[:bootstrap])
+          if (role_details[:bootstrap].empty?)
+            Logger.debug { "role_details[:bootstrap] is empty, ignoring" }
+          else
+            role_details[:bootstrap] = Stack.find_file(config, role_details[:bootstrap])
+          end
+        else
+          role_details[:bootstrap] = Stack.find_file(config, 'chef-client-bootstrap-excl-validation-pem.sh')
+        end
+
+        # we default to the role name for the security group unless explicitly set
+        if role_details[:security_group].nil?
+          role_details[:security_group] = role.to_s
+        end
+
+        (1..role_details[:count]).each do |p|
+          Logger.debug { "Populating the config[:role_details][:azs] array with AZ" }
+          role_details[:azs] = Array.new if role_details[:azs].nil?
+          
+          # is there an az set for this node?
+          if role_details[:azs][p-1].nil?
+            # inherit the global az 
+            Logger.debug { "Inheriting the AZ for #{role} (#{config['REGION']})" }
+            role_details[:azs][p-1] = config['REGION']
+          end
+          
+          # add this AZ to the AZ list, we'll dedupe later
+          config[:azs] << role_details[:azs][p-1]
+          
+          hostname =  Stack.generate_hostname(config, role, p)
+          Logger.debug { "Setting node_details for #{hostname}, using element #{p}-1 from #{role_details[:azs]}" }
+          config[:node_details][hostname] = { :az => role_details[:azs][p-1], :region => role_details[:azs][p-1], :role => role }
+        end 
       end
     end
-    running_instances 
+    config[:azs].uniq!
+
+    # if set the region specific settings from the global settings if not already specified
+    config[:azs].each do |az|
+      # we store region spefic stuff in hash
+      config[az] = Hash.new if config[az].nil?
+
+      config[az]['image_id'] = config['image_id'] if config[az]['image_id'].nil?
+    end
+
+    config[:node_details]
+  end
+
+  # get all instances running in the current config
+  # return a hash where key is the instance name, value is another hash containing :region, :id, :addresses
+  def Stack.get_all_instances(config, refresh = false)
+    if config[:all_instances].nil? || refresh
+      # we need to get the server list for each AZ mentioned in the config[:roles][:role][:azs], this is populated by Stack.populate_config
+      Stack.populate_config(config)
+      
+      # get the current list of servers from OpenStack & generate a hash, keyed on name
+      servers = Hash.new
+      config[:azs].each do |az|
+        os = Stack.connect(config, az)
+        os.servers.each do |server| 
+          servers[server[:name]] = { 
+            :region => az,
+            :id => server[:id], 
+            :addresses => os.server(server[:id]).addresses
+          }
+        end 
+      end
+      config[:all_instances] = servers
+    end
+    config[:all_instances]
   end
 
   def Stack.show_running(config)
-    running_instances = Stack.get_running(config)
-    running_instances.each do |instance, instance_details|
-      # display some details
-      puts "#{instance} id=#{instance_details.id} flavor_id=#{instance_details.flavor_id} public_ip=#{instance_details.public_ip_address} private_ip=#{instance_details.private_ip_address}"
+    # TODO: optionally show the hosts that are missing
+    ours = Stack.get_our_instances(config)
+
+    ours.each do |node, node_details|
+      printf("%-30s %20s %8d %16s %s\n", node, node_details[:region], node_details[:id], node_details[:role], node_details[:addresses].map { |address| address.address })
     end
   end
-   
-  def Stack.delete_node(config, fqdn)
-    running_instances = Stack.get_running(config)
-    if running_instances[fqdn].nil?
-      puts "ERROR: #{fqdn} isn't running!"
-      exit
+
+  # Add an instance to the :all_instances hash, instead of having to poll the whole lot again
+  def Stack.add_instance(config, hostname, region, id, addresses)
+    config[:all_instances][hostname] = { :region => region, :id => id, :addresses => addresses}
+  end
+ 
+  def Stack.ssh(config, hostname = nil, user = ENV['USER'], command = nil)
+    # ssh to a host, or all hosts 
+
+    # get all running instances
+    servers = Stack.get_our_instances(config)
+
+    if hostname.nil?
+      Logger.debug { "request to SSH to all hosts" }
+      servers.each do |host, details|
+        public_ip = Stack.get_public_ip(config, host)
+        Logger.info { "#{host} #{public_ip}" }
+        cmd_output =  `ssh -oStrictHostKeyChecking=no -l #{user} #{public_ip} "#{command}"`
+        Logger.info { "#{host} #{public_ip} #{cmd_output}" }
+      end
     else
-      connection = Stack.connect(config)
-      pp running_instances[fqdn]
-      running_instances[fqdn].destroy
+      Logger.debug { "request to SSH to #{servers[hostname]}" }
     end
   end
 
-  def Stack.show_details(config)
-    # create a connection
-    connection = Stack.connect(config)
-    
-    pp connection.describe_regions
-    pp connection.describe_availability_zones
 
-    pp connection.servers
+  def Stack.get_our_instances(config)
+    # build an hash of running instances that match our generated hostnames
+    node_details = Stack.populate_config(config)
 
-    Stack.populate_config(config)
-    pp config[:node_details]
+    # get all of our hostnames
+    hostnames = Stack.generate_server_names(config)
+
+    # get all running instances
+    servers = Stack.get_all_instances(config)
+
+    running = Hash.new
+    # do any of the list of servers in OpenStack match one of our hostnames?
+    hostnames.each do |hostname|
+      if (servers.include?(hostname))
+        # return the instance details merged with the node_details (info like role)
+        running[hostname] = servers[hostname].merge(node_details[hostname])
+      end
+    end
+
+    running
   end
 
-  def upload_keys(config)
-    if (key_pair = connection.key_pairs.get(config[:keypair]).nil?)
-      key_pair = connection.key_pairs.create( :name => config[:keypair], :public_key => File.read("rentpro-deploy.pub") )
+  def Stack.delete_node(config, node)
+    # this also populates out unspecified defaults, like az
+    Stack.populate_config(config)
+    # get info about all instances running in our account & AZs
+    Stack.get_all_instances(config)
+
+    if (config[:all_instances][node].nil?)
+      Logger.info "Sorry, #{node} doesn't exist or isn't running"
     else
-      puts "#{config[:keypair]} key_pair already exists"
+      Logger.info "Deleting node #{node} in #{config[:all_instances][node][:region]}..."
+      os = Stack.connect(config, config[:all_instances][node][:region])
+      instance = os.get_server(config[:all_instances][node][:id])
+      instance.delete!
+    end
+  end
+
+  def Stack.delete_all(config)
+    # this also populates out unspecified defaults, like az
+    Stack.populate_config(config)
+
+    # get the list of nodes we consider 'ours', i.e. with hostnames that match 
+    # those generated by this stack
+    ours = Stack.get_our_instances(config)
+  
+    # do any of the list of servers in OpenStack match one of our hostnames?
+    ours.each do |node, node_details|
+        Logger.info "Deleting #{node}"
+        os = Stack.connect(config, config[:all_instances][node][:region])
+        d = os.get_server(config[:all_instances][node][:id])
+        d.delete!
+    end
+  end
+ 
+  def Stack.get_public_ip(config, hostname)
+    # get a public address from the instance 
+    # (could be either the dynamic or one of our floating IPs
+    config[:all_instances][hostname][:addresses].each do |address|
+      if address.label == 'public'
+        return address.address
+      end
+    end
+  end
+
+  def Stack.set_chef_server(config, chef_server)
+    # set the private & public URLs for the chef server, 
+    # called either after we create the Chef Server, or skip over it
+    Logger.debug "Setting :chef_server_hostname, chef_server_private & chef_server_public details (using #{chef_server})"
+
+    config[:chef_server_hostname] = chef_server
+    # get the internal IP of this instance....which we should have stored in config[:all_instances]
+    if config[:all_instances][chef_server] && config[:all_instances][chef_server][:addresses]
+      config[:all_instances][chef_server][:addresses].each do |address|
+        # find the private IP, any old private IP will do...
+        if (address.label == 'private')
+          config[:chef_server_private] = "http://#{address.address}:4000/"
+          Logger.info "Setting the internal Chef URL to #{config[:chef_server_private]}"
+        end
+
+        # only set the public url if it hasn't been set in the config
+        if ((config[:chef_server_public].nil? || config[:chef_server_public].empty?) && address.label == 'public')
+          config[:chef_server_public] = "http://#{address.address}:4000/"
+          Logger.info "Setting the public Chef URL to #{config[:chef_server_public]}"
+        end
+      end
+    end
+  end
+
+  def Stack.secgroup_sync(config)
+    # 1) get all the IP information we have
+    # 2) generate the json to describe that to the "stackhelper secgroup-sync" tool
+    # 3) run "stackhelper secgroup-sync --some-file our-ips.json"
+    ours = Stack.get_our_instances(config)
+  
+    secgroup_ips = Hash.new
+    # walk the list of hosts, dumping the IPs into role buckets
+    ours.each do |instance, instance_details|
+      secgroup_ips[instance_details[:role]] = Array.new if secgroup_ips[instance_details[:role]].nil?
+
+      #secgroup_ips[instance_details[:role]] << instance_details[:addresses].map { |address| address.address }
+      secgroup_ips[instance_details[:role]] << instance_details[:addresses].map do |address| 
+        if (address.label == 'public') 
+          address.address 
+        else 
+          next 
+        end
+      end
+
+      # we potentially have an array of arrays, so flatten them
+      secgroup_ips[instance_details[:role]].flatten!
+
+      # delete any nil's that we collected due to skipping private ips
+      secgroup_ips[instance_details[:role]].delete_if {|x| x.nil? } 
+    end
+
+    # dump the json to a temp file
+    #sg_json = Tempfile.new(['secgroup_ips', '.json'])
+    sg_json = File.new('secgroup_ips.json', "w")
+    sg_json.write(secgroup_ips.to_json)
+    sg_json.close
+
+    # run the secgroup-sync tool, across each AZ/REGION
+    config[:azs].each do |az|
+      Logger.info "Syncing security groups in #{az}"
+      system("stackhelper --os-region-name #{az} secgroup-sync --secgroup-json secgroups.json --additional-group-json #{sg_json.path}")
+    end
+  end
+
+  # if we're passed a role, only deploy this role.
+  def Stack.deploy_all(config, role_to_deploy = nil)
+    Stack.validate(config)
+    
+    # this also populates out unspecified defaults, like az
+    node_details = Stack.populate_config(config)
+    # get info about all instances running in our account & AZs
+    servers = Stack.get_all_instances(config)
+
+    # this is our main loop iterator, generates each host
+    config[:roles].each do |role,role_details| 
+      Logger.debug { "Iterating over roles, this is #{role}, role_details = #{role_details}" }
+
+      (1..role_details[:count]).each do |p|
+        hostname = Stack.generate_hostname(config, role, p)
+        Logger.debug { "Iterating over nodes in #{role}, this is #{hostname}" }
+        
+        # configure the global :chef_server details if this the chef server
+        if role_details[:chef_server]
+          Stack.set_chef_server(config, hostname)
+        end
+
+        # does this node already exist?
+        if (!servers[hostname].nil?)
+          Logger.info { "#{hostname} already exists, skipping.." }
+          next
+        end
+      
+        Logger.debug { "Deploying #{role}, role_to_deploy = #{role_to_deploy}" }
+        if ((role_to_deploy.nil?) || (role_to_deploy.to_s == role.to_s))
+          if (role_details[:skip_chef_prereg] == true || role_details[:chef_server])
+            Logger.debug "Skipping Chef pre-reg for #{hostname}"
+          else
+            # Prepare Chef
+            # 1) delete the client if it exists
+            knife_client_list = `knife client list | grep #{hostname}`
+            knife_client_list.sub!(/\s/,'')
+            if knife_client_list.length() > 0
+              # we should delete the client to make way for this new machine
+              Logger.info `knife client delete --yes #{hostname}`              
+            end
+            
+            # knife node create -d --environment $CHEF_ENVIRONMENT $SERVER_NAME
+            # knife node run_list add -d --environment $CHEF_ENVIRONMENT $SERVER_NAME "role[${ROLE}]"
+            # this relies on .chef matching the stacks config (TODO: poke the Chef API directly?)
+            cmd = "EDITOR=\"perl -p -i -e 's/_default/#{config[:chef_environment]}/'\" knife node create --server-url #{config[:chef_server_public]} #{hostname}"
+            Logger.debug cmd
+            knife_node_create = `#{cmd}`
+            Logger.info "Priming Chef Server: #{knife_node_create}"
+
+            cmd = "knife node run_list add -d --environment #{config[:chef_environment]} #{hostname} \"role[#{role}]\""
+            Logger.info cmd
+            knife_node_run_list = `#{cmd}`
+            Logger.info "Priming Chef Server: #{knife_node_run_list}"
+          end
+
+          # build the user-data content for this host
+          # (we have a local copy of https://github.com/lovelysystems/cloud-init/blob/master/tools/write-mime-multipart)
+          # 1) generate the mimi-multipart file
+          # libdir = where our shipped scripts live
+          # (use config[:stackhome] for "project" config/scripts)
+          libdir = File.realpath(@@gemhome + '/lib')
+          multipart_cmd = "#{libdir}/write-mime-multipart #{role_details[:bootstrap]} #{role_details[:cloud_config_yaml]}"
+          Logger.debug { "multipart_cmd = #{multipart_cmd}" }
+          multipart = `#{multipart_cmd}`
+          # 2) replace the tokens (CHEF_SERVER, CHEF_ENVIRONMENT, SERVER_NAME, ROLE)
+          multipart.gsub!(%q!%HOSTNAME%!, hostname)
+
+          Logger.info  "Chef server is #{config[:chef_server_hostname]}, which is in #{config[:node_details][config[:chef_server_hostname]][:region]}"
+          Logger.info  "#{hostname}'s region is #{config[:node_details][hostname][:region]}"
+          # if this host is in the same region/az, use the private URL, if not, use the public url
+          if (config[:node_details][hostname][:region] == config[:node_details][config[:chef_server_hostname]][:region]) && !config[:chef_server_private].nil?
+            multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_private])
+          elsif !config[:chef_server_public].nil?
+            multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_public])
+          else
+            Logger.warn { "Not setting the chef url for #{hostname} as neither chef_server_private or chef_server_public are valid yet" }
+          end
+          multipart.gsub!(%q!%CHEF_ENVIRONMENT%!, config[:chef_environment])
+          if File.exists?(config[:chef_validation_pem])
+            multipart.gsub!(%q!%CHEF_VALIDATION_PEM%!, File.read(config[:chef_validation_pem]))
+          else
+            Logger.warn "Skipping #{config[:chef_validation_pem]} substitution in user-data"
+          end
+          multipart.gsub!(%q!%SERVER_NAME%!, hostname)
+          multipart.gsub!(%q!%ROLE%!, role.to_s)
+          multipart.gsub!(%q!%DATA_DIR%!, role_details[:data_dir])
+
+          Logger.info "Creating #{hostname} in #{node_details[hostname][:az]} with role #{role}"
+
+          # this will get put in /meta.js 
+          metadata = { 'region' => node_details[hostname][:az], 'chef_role' => role }
+
+          os = Stack.connect(config, node_details[hostname][:az])
+          newserver = os.create_server(:name => hostname, 
+                                       :imageRef => config[node_details[hostname][:az]]['image_id'], 
+                                       :flavorRef => config['flavor_id'],
+                                       :security_groups=>[role_details[:security_group]],
+                                       :user_data => Base64.encode64(multipart), 
+                                       :metadata => metadata,
+                                       :key_name => config[:key_pair])
+
+          # wait for the server to become ACTIVE before proceeding
+          while (newserver.status != 'ACTIVE') do
+            print '.'
+            sleep 1
+            # refresh the status
+            newserver.refresh
+          end
+          puts
+
+          # refresh the config[:all_instances] with the newly built server
+          # TODO: we should be able to just add this server, instead of re-polling everything
+          Stack.get_all_instances(config, true)
+
+          # refresh the chef_server details..we should have IPs now
+          if role_details[:chef_server]
+            Stack.set_chef_server(config, hostname)
+            Stack.generate_knife_rb(config)
+          end
+
+          # attach a floating IP to this if we have one
+          if role_details[:floating_ips] && role_details[:floating_ips][p-1]
+            floating_ip = role_details[:floating_ips][p-1]
+            Logger.info "Attaching #{floating_ip} to  #{hostname}\n"
+            # nova --os-region-name $REGION add-floating-ip $SERVER_NAME $FLOATING_IP
+            floating_ip_add = `nova --os-region-name #{node_details[hostname][:az]} add-floating-ip #{hostname} #{floating_ip}`
+            Logger.info floating_ip_add 
+          end 
+          
+          # refresh the secgroups ASAP
+          Stack.secgroup_sync(config)
+
+          # run any post-install scripts, these are run from the current host, not the nodes
+          if role_details[:post_install_script]
+            # convert when we got passed to an absolute path
+            post_install_script_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_script])
+            post_install_cwd_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_cwd])
+            
+            # replace any tokens in the argument
+            public_ip = Stack.get_public_ip(config, hostname)
+            role_details[:post_install_args].sub!(%q!%PUBLIC_IP%!, public_ip)
+            # we system this, as they are can give live feed back
+            Logger.info "Executing '#{post_install_script_abs} #{role_details[:post_install_args]}' as the post_install_script"
+            system("cd #{post_install_cwd_abs} ; #{post_install_script_abs} #{role_details[:post_install_args]}")
+          end
+        else
+          Logger.info "Skipped role #{role}"
+        end
+      end 
     end
   end
 
-  def shutdown_all(config)
-    # shutdown all instances
-    connection.servers.select do |server| 
-      puts "Running server:"
-      # pp server
-    #  server.ready? && server.destroy
+  def Stack.find_file(config, filename)
+    # find a file, using the standard path precedence
+    # 1) cwd
+    # 2) stackhome
+    # 3) gemhome/lib
+    dirs = [ './' ] 
+    dirs.push(config[:stackhome])
+    dirs.push(@@gemhome + '/lib')
+
+    Logger.debug "find_file, looking for #{filename} in #{dirs}"
+    filename_fqp = ''
+    dirs.each do |dir|
+      fqp = dir + '/' + filename
+      Logger.debug "find_file: checking #{fqp}"
+      if File.file?(fqp)
+      Logger.debug "find_file: found #{fqp}!"
+        filename_fqp =  File.expand_path(fqp)
+      end
     end
+
+    if filename_fqp.empty?
+      Logger.warn "couldn't find #{filename} in #{dirs}"
+    end
+    filename_fqp
   end
 
 end
+