aws-kicker 0.0.3 → 0.0.4

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3@aws-kicker"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.17.3 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -2,5 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in kicker.gemspec
 gemspec
-gem "excon", "~>0.17.0"
-gem "fog", "~>1.7.0"

data/aws-kicker.gemspec

@@ -1,15 +1,15 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'kicker/version'
+require 'aws-kicker/version'
 
 Gem::Specification.new do |gem|
   gem.name          = "aws-kicker"
-  gem.version       = Kicker::VERSION
+  gem.version       = Aws::Kicker::VERSION
   gem.authors       = ["Simon McCartney"]
   gem.email         = ["simon@mccartney.ie"]
   gem.description   = %q{Utility for kicking an application stack into life on Amazon EC2}
-  gem.summary       = %q{Stacks are built from a collection of instances required to build an application stack. 
+  gem.summary       = %q{Stacks are built from a collection of instances required to build an application stack.
 Each instance is described in the Stackfile & provisioned using variety of methods. Supported models include:
 EC2 Create with user-data, cloud-init from user-data (installs masterless puppet), puppet provision the instance.
 
@@ -27,6 +27,7 @@ The guiding principle is that your Stackfile should be shareable & re-useable by
   gem.add_development_dependency('rdoc')
   gem.add_development_dependency('aruba')
   gem.add_development_dependency('rake', '~> 0.9.2')
+  gem.add_development_dependency('gem-release')
   gem.add_dependency('methadone', '~> 1.2.4')
-  gem.add_dependency('fog', '~> 1.7.0')
+  gem.add_dependency('fog', '~> 1.12.1')
 end

data/bin/{kicker → aws-kicker}

@@ -2,7 +2,7 @@
 
 require 'optparse'
 require 'methadone'
-require 'kicker'
+require 'aws-kicker'
 require 'stack'
 require 'pp'
 
@@ -41,7 +41,7 @@ class App
     if options['delete-node']
       info { "delete-node passed #{options['delete-node']}" }
       Stack.delete_node(config, options['delete-node'])
-    end 
+    end
 
     if options['validate']
       info { 'Attemptying to validate the config' }
@@ -74,8 +74,8 @@ class App
   on("--replace-node NODE", "Replace a give node")
   on("--delete-node NODE", "Destroy a node, including any shared CM & DNS")
   on("--validate", "Validate the config, check as much as possible with out creating nodes")
-  
-  version Kicker::VERSION
+
+  version Aws::Kicker::VERSION
 
   use_log_level_option
 

data/lib/aws-kicker.rb

@@ -0,0 +1,7 @@
+require "aws-kicker/version"
+
+module Aws
+  module Kicker
+    # Your code goes here...
+  end
+end

data/lib/aws-kicker/version.rb

@@ -0,0 +1,5 @@
+module Aws
+  module Kicker
+    VERSION = "0.0.4"
+  end
+end

data/lib/stack.rb

@@ -3,10 +3,21 @@ require 'fog'
 
 module Stack
 
+  Logger = Logger.new(STDOUT)
+  Logger.level = ::Logger::INFO
+  Logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+  Logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime} #{severity}: #{msg}\n"
+  end
+
+  @@gemhome = File.absolute_path(File.realpath(File.dirname(File.expand_path(__FILE__)) + '/..'))
+
   def Stack.load_config(configfile, stack)
     config_raw = File.read(configfile)
-    eval(config_raw) 
+    eval(config_raw)
+
     config = StackConfig::Stacks[stack]
+    config[:stackhome] = File.dirname(File.expand_path(configfile))
     config
   end
 
@@ -19,20 +30,25 @@ module Stack
   end
 
   def Stack.populate_config(config)
-    # build out the full config for each node, supplying defaults from the 
+
+    config[:find_file_paths] = Array.new if config[:find_file_paths].nil?
+
+    # build out the full config for each node, supplying defaults from the
     # global config if explicitly supplied
     config[:node_details] = Hash.new if config[:node_details].nil?
 
     config[:roles].each do |role, role_details|
       fqdn = role.to_s + '.' + config[:dns_domain]
-      
-      config[:node_details][fqdn] = { 
+
+      config[:node_details][fqdn] = {
         # set the node details from the role, if not specified in the role, use the config global
         # (takes advantage of left to right evaluation of ||)
         :flavor_id          => (role_details[:flavor_id] || config[:flavor_id]),
         :count              => (role_details[:count] || 1),
         :publish_private_ip => (role_details[:publish_private_ip] || false),
-        :dns_wildcard       => (role_details[:dns_wildcard] || false)
+        :dns_wildcard       => (role_details[:dns_wildcard] || false),
+        :bootstrap          => (role_details[:bootstrap] || 'user-data.sh'),
+        :cloud_config_yaml  => (role_details[:cloud_config_yaml] || 'cloud-init.yaml'),
       }
     end
   end
@@ -43,7 +59,7 @@ module Stack
       fqdn = role.to_s + '.' + config[:dns_domain]
       stack_hostnames << fqdn
     end
-    stack_hostnames 
+    stack_hostnames
   end
 
   def Stack.deploy_all(config)
@@ -64,17 +80,25 @@ module Stack
       end
 
       # Ubuntu 8.04/Hardy doesn't do full cloud-init, so we have to script setting the hostname
-      user_data = File.read('user-data.sh')
+      libdir = File.realpath(@@gemhome + '/lib')
+
+      bootstrap_abs = Stack.find_file(config, config[:node_details][fqdn][:bootstrap])
+      cloud_config_yaml_abs = Stack.find_file(config, config[:node_details][fqdn][:cloud_config_yaml])
+
+      multipart_cmd = "#{libdir}/write-mime-multipart #{bootstrap_abs} #{cloud_config_yaml_abs}"
+      user_data = `#{multipart_cmd}`
+
       user_data.gsub!(/rentpro-unconfigured/, hostname)
       user_data.gsub!(/rentpro-stage.local/, config[:dns_domain])
 
+
       # pp multipart
       #
       puts "Bootstraping new instance - #{fqdn}, in #{config[:availability_zone]}, flavor #{config[:node_details][fqdn][:flavor_id]}, image_id #{config[:image_id]}"
-      server = connection.servers.create({ 
+      server = connection.servers.create({
                                         :name => fqdn,
                                         :hostname => fqdn,
-                                        :availability_zone => config[:availability_zone], 
+                                        :availability_zone => config[:availability_zone],
                                         :flavor_id => config[:node_details][fqdn][:flavor_id],
                                         :image_id => config[:image_id],
                                         :key_name => config[:keypair],
@@ -86,10 +110,11 @@ module Stack
       server.wait_for { ready? }
       puts "#{role.to_s} is booted, #{server.public_ip_address}/#{server.private_ip_address}"
 
+      if false
       # create/update the public & private DNS for this host
       Stack.update_dns(role.to_s + '-public.' + config[:dns_domain], server.public_ip_address, config)
       Stack.update_dns(role.to_s + '-private.' + config[:dns_domain], server.private_ip_address, config)
-      
+
       # create the dns
       if (role_details[:publish_private_ip] == true && (!role_details[:publish_private_ip].nil?))
         ip_address = server.private_ip_address
@@ -103,6 +128,7 @@ module Stack
         wildcard = "*." + config[:dns_domain]
         Stack.update_dns(wildcard, ip_address, config)
       end
+      end
     end
   end
 
@@ -111,14 +137,14 @@ module Stack
     dns = Fog::DNS.new({ :provider => config[:provider],
                           :aws_access_key_id => config[:aws_access_key_id],
                           :aws_secret_access_key => config[:aws_secret_access_key] })
-          
+
     # pp dns.get_hosted_zone(config[:dns_id])
-        
+
     bmtw =  dns.zones.get(config[:dns_id])
-      
-    record = bmtw.records.get(fqdn) 
-    if record 
-      record.modify(:value => ip_address) if record 
+
+    record = bmtw.records.get(fqdn)
+    if record
+      record.modify(:value => ip_address) if record
     else
       bmtw.records.create(:value => ip_address, :name => fqdn, :type => 'A')
     end
@@ -129,7 +155,7 @@ module Stack
     dns = Fog::DNS.new({ :provider => config[:provider],
                           :aws_access_key_id => config[:aws_access_key_id],
                           :aws_secret_access_key => config[:aws_secret_access_key] })
-          
+
     zone = dns.zones.get(config[:dns_id])
     if zone.records.empty?
       puts "No DNS records found in #{config[:dns_domain]}"
@@ -138,13 +164,13 @@ module Stack
       zone.records.each do |record|
         printf("%40s %20s %5s %5d\n", record.name, record.value, record.type, record.ttl)
       end
-    end 
+    end
   end
 
   def Stack.get_running(config)
     # create a connection
     connection = Stack.connect(config)
-  
+
     # generate all the names that this stack will use
     stack_hostnames = Stack.generate_hostnames(config)
 
@@ -159,7 +185,7 @@ module Stack
         end
       end
     end
-    running_instances 
+    running_instances
   end
 
   def Stack.show_running(config)
@@ -169,14 +195,14 @@ module Stack
       puts "#{instance} id=#{instance_details.id} flavor_id=#{instance_details.flavor_id} public_ip=#{instance_details.public_ip_address} private_ip=#{instance_details.private_ip_address}"
     end
   end
-   
+
   def Stack.delete_node(config, fqdn)
     running_instances = Stack.get_running(config)
     if running_instances[fqdn].nil?
       puts "ERROR: #{fqdn} isn't running!"
       exit
     else
-      connection = Stack.connect(config)
+      Stack.connect(config)
       pp running_instances[fqdn]
       running_instances[fqdn].destroy
     end
@@ -185,7 +211,7 @@ module Stack
   def Stack.show_details(config)
     # create a connection
     connection = Stack.connect(config)
-    
+
     pp connection.describe_regions
     pp connection.describe_availability_zones
 
@@ -205,7 +231,7 @@ module Stack
 
   def shutdown_all(config)
     # shutdown all instances
-    connection.servers.select do |server| 
+    connection.servers.select do |server|
       puts "Running server:"
       # pp server
     #  server.ready? && server.destroy
@@ -219,4 +245,40 @@ module Stack
     pp connection
     pp connection.images.get(config[:image_id])
   end
+
+  def Stack.find_file(config, filename)
+    # find a file, using the standard path precedence
+    # 1) cwd
+    # 2) stackhome
+    # 2) stackhome + find_file_paths
+    # 3) gemhome/lib
+
+    if filename.nil? || filename.empty?
+      raise ArgumentError
+    end
+
+    dirs = [ '.' ]  # current directory
+    dirs.push(config[:stackhome])
+    config[:find_file_paths].each { |fp| dirs.push(File.join(config[:stackhome], fp)) }
+    dirs.push(File.join(@@gemhome, 'lib'))
+    dirs.push('')   # find absolute paths
+    dirs.flatten!
+
+    Logger.debug "find_file, looking for #{filename} in #{dirs}"
+    filename_fqp = ''
+    dirs.each do |dir|
+      fqp = File.join(dir, filename)
+      Logger.debug "find_file: checking #{fqp}"
+      if File.file?(fqp)
+      Logger.debug "find_file: found #{fqp}!"
+        filename_fqp =  File.expand_path(fqp)
+      end
+    end
+
+    if filename_fqp.empty?
+      Logger.warn "couldn't find #{filename} in #{dirs}"
+      filename_fqp = nil
+    end
+    filename_fqp
+  end
 end

data/lib/write-mime-multipart

@@ -0,0 +1,118 @@
+#!/usr/bin/python2.6
+# largely taken from python examples
+# http://docs.python.org/library/email-examples.html
+
+import os
+import sys
+import smtplib
+# For guessing MIME type based on file name extension
+import mimetypes
+
+from email import encoders
+from email.message import Message
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from optparse import OptionParser
+import gzip
+
+from base64 import b64encode
+
+COMMASPACE = ', '
+
+starts_with_mappings={
+    '#include' : 'text/x-include-url',
+    '#!' : 'text/x-shellscript',
+    '#cloud-config' : 'text/cloud-config',
+    '#upstart-job'  : 'text/upstart-job',
+    '#part-handler' : 'text/part-handler',
+    '#cloud-boothook' : 'text/cloud-boothook'
+}
+
+def get_type(fname,deftype):
+    f = file(fname,"rb")
+    line = f.readline()
+    f.close()
+    rtype = deftype
+    for str,mtype in starts_with_mappings.items():
+        if line.startswith(str):
+            rtype = mtype
+            break
+    return(rtype)
+
+def main():
+    outer = MIMEMultipart()
+    #outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory)
+    #outer['To'] = COMMASPACE.join(opts.recipients)
+    #outer['From'] = opts.sender
+    #outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
+
+    parser = OptionParser()
+    
+    parser.add_option("-o", "--output", dest="output",
+        help="write output to FILE [default %default]", metavar="FILE", 
+        default="-")
+    parser.add_option("-z", "--gzip", dest="compress", action="store_true",
+        help="compress output", default=False)
+    parser.add_option("-d", "--default", dest="deftype",
+        help="default mime type [default %default]", default="text/plain")
+    parser.add_option("--delim", dest="delim",
+        help="delimiter [default %default]", default=":")
+    parser.add_option("-b", "--base64", dest="base64", action="store_true",
+        help="encode content base64", default=False)
+
+    (options, args) = parser.parse_args()
+
+    if (len(args)) < 1:
+        parser.error("Must give file list see '--help'")
+
+    for arg in args:
+        t = arg.split(options.delim, 1)
+        path=t[0]
+        if len(t) > 1:
+            mtype = t[1]
+        else:
+            mtype = get_type(path,options.deftype)
+
+        maintype, subtype = mtype.split('/', 1)
+        if maintype == 'text':
+            fp = open(path)
+            # Note: we should handle calculating the charset
+            msg = MIMEText(fp.read(), _subtype=subtype)
+            fp.close()
+        else:
+            fp = open(path, 'rb')
+            msg = MIMEBase(maintype, subtype)
+            msg.set_payload(fp.read())
+            fp.close()
+            # Encode the payload using Base64
+            encoders.encode_base64(msg)
+
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment',
+            filename=os.path.basename(path))
+
+        outer.attach(msg)
+
+    if options.output is "-":
+        ofile = sys.stdout
+    else:
+        ofile = file(options.output,"wb")
+    
+    if options.base64:
+        output = b64encode(outer.as_string())
+    else:
+        output = outer.as_string()
+        
+    if options.compress:
+        gfile = gzip.GzipFile(fileobj=ofile, filename = options.output )
+        gfile.write(output)
+        gfile.close()
+    else:
+        ofile.write(output)
+
+    ofile.close()
+
+if __name__ == '__main__':
+    main()
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-kicker
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-21 00:00:00.000000000 Z
+date: 2013-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -59,6 +59,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: methadone
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.12.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,29 +106,31 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.12.1
 description: Utility for kicking an application stack into life on Amazon EC2
 email:
 - simon@mccartney.ie
 executables:
-- kicker
+- aws-kicker
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rvmrc
 - Gemfile
 - LICENSE.txt
 - README.md
 - README.rdoc
 - Rakefile
 - aws-kicker.gemspec
-- bin/kicker
+- bin/aws-kicker
 - features/kicker.feature
 - features/step_definitions/kicker_steps.rb
 - features/support/env.rb
-- lib/kicker.rb
-- lib/kicker/version.rb
+- lib/aws-kicker.rb
+- lib/aws-kicker/version.rb
 - lib/stack.rb
+- lib/write-mime-multipart
 - test/tc_something.rb
 homepage: https://github.com/simonmcc/kicker
 licenses: []
@@ -138,15 +156,14 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: ! 'Stacks are built from a collection of instances required to build an application
-  stack.  Each instance is described in the Stackfile & provisioned using variety
-  of methods. Supported models include: EC2 Create with user-data, cloud-init from
-  user-data (installs masterless puppet), puppet provision the instance.  Other models
-  could include using a puppet master, Chef Solo or Chef Server/Hosted. Amazon EC2
-  interaction is done through fog, so other providers should be easily added.  The
-  guiding principle is that your Stackfile should be shareable & re-useable by others,
-  and support templates, so that other users can use the template and easily adjust
-  items in the stack (such as the instance size used, DNS Domain updated during deploy,
-  EC2 account & location)'
+  stack. Each instance is described in the Stackfile & provisioned using variety of
+  methods. Supported models include: EC2 Create with user-data, cloud-init from user-data
+  (installs masterless puppet), puppet provision the instance.  Other models could
+  include using a puppet master, Chef Solo or Chef Server/Hosted. Amazon EC2 interaction
+  is done through fog, so other providers should be easily added.  The guiding principle
+  is that your Stackfile should be shareable & re-useable by others, and support templates,
+  so that other users can use the template and easily adjust items in the stack (such
+  as the instance size used, DNS Domain updated during deploy, EC2 account & location)'
 test_files:
 - features/kicker.feature
 - features/step_definitions/kicker_steps.rb

data/lib/kicker.rb

@@ -1,5 +0,0 @@
-require "kicker/version"
-
-module Kicker
-  # Your code goes here...
-end

data/lib/kicker/version.rb

@@ -1,3 +0,0 @@
-module Kicker
-  VERSION = "0.0.3"
-end