aws-kicker 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+results.html
+pkg
+html

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kicker.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,62 @@
+PATH
+  remote: .
+  specs:
+    aws-kicker (0.0.2)
+      fog (~> 1.7.0)
+      methadone (~> 1.2.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (0.5.1)
+      childprocess (~> 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    builder (3.1.4)
+    childprocess (0.3.7)
+      ffi (~> 1.0, >= 1.0.6)
+    cucumber (1.2.1)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.11.0)
+      json (>= 1.4.6)
+    diff-lcs (1.1.3)
+    excon (0.16.10)
+    ffi (1.3.1)
+    fog (1.7.0)
+      builder
+      excon (~> 0.14)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.4)
+    gherkin (2.11.6)
+      json (>= 1.7.6)
+    json (1.7.6)
+    methadone (1.2.4)
+      bundler
+    mime-types (1.19)
+    multi_json (1.3.7)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.2.2)
+    nokogiri (1.5.5)
+    rake (0.9.6)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    ruby-hmac (0.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+  aws-kicker!
+  rake (~> 0.9.2)
+  rdoc

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Simon McCartney
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Kicker
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'kicker'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kicker
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,19 @@
+= kicker - DESCRIBE YOUR GEM
+
+Author::  YOUR NAME (YOUR EMAIL)
+Copyright:: Copyright (c) 2013 YOUR NAME
+
+
+DESCRIBE YOUR GEM HERE
+
+== Links
+
+* {Source on Github}[LINK TO GITHUB]
+* RDoc[LINK TO RDOC.INFO]
+
+== Install
+
+== Examples
+
+== Contributing
+

data/Rakefile

@@ -0,0 +1,61 @@
+def dump_load_path
+  puts $LOAD_PATH.join("\n")
+  found = nil
+  $LOAD_PATH.each do |path|
+    if File.exists?(File.join(path,"rspec"))
+      puts "Found rspec in #{path}"
+      if File.exists?(File.join(path,"rspec","core"))
+        puts "Found core"
+        if File.exists?(File.join(path,"rspec","core","rake_task"))
+          puts "Found rake_task"
+          found = path
+        else
+          puts "!! no rake_task"
+        end
+      else
+        puts "!!! no core"
+      end
+    end
+  end
+  if found.nil?
+    puts "Didn't find rspec/core/rake_task anywhere"
+  else
+    puts "Found in #{path}"
+  end
+end
+require 'bundler'
+require 'rake/clean'
+
+require 'rake/testtask'
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/tc_*.rb'
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+  
+  rd.main = "README.rdoc"
+  
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:test,:features]
+

data/aws-kicker.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kicker/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "aws-kicker"
+  gem.version       = Kicker::VERSION
+  gem.authors       = ["Simon McCartney"]
+  gem.email         = ["simon@mccartney.ie"]
+  gem.description   = %q{Utility for kicking an application stack into life on Amazon EC2}
+  gem.summary       = %q{Stacks are built from a collection of instances required to build an application stack. 
+Each instance is described in the Stackfile & provisioned using variety of methods. Supported models include:
+EC2 Create with user-data, cloud-init from user-data (installs masterless puppet), puppet provision the instance.
+
+Other models could include using a puppet master, Chef Solo or Chef Server/Hosted.
+Amazon EC2 interaction is done through fog, so other providers should be easily added.
+
+The guiding principle is that your Stackfile should be shareable & re-useable by others, and support templates, so that other users can use the template and easily adjust items in the stack (such as the instance size used, DNS Domain updated during deploy, EC2 account & location)
+}
+  gem.homepage      = "https://github.com/simonmcc/kicker"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency('rdoc')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('rake', '~> 0.9.2')
+  gem.add_dependency('methadone', '~> 1.2.4')
+  gem.add_dependency('fog', '~> 1.7.0')
+end

data/bin/aws-kicker

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "bundler/setup"
+
+require 'optparse'
+require 'methadone'
+require 'kicker'
+require 'stack'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |stack|
+    config = Stack.load_config(options['configfile'], stack)
+    if config.nil?
+      exit_now!("#{stack} doesn't exist in #{options['configfile']}")
+    end
+
+    if options['show-stack']
+      puts stack
+      hostnames = Stack.generate_hostnames(config)
+      hostnames.each { |hostname| puts "    #{hostname}" }
+    end
+
+    if options['show-running']
+      Stack.show_running(config)
+    end
+
+    if options['show-dns']
+      Stack.show_dns(config)
+    end
+
+    if options['show-details']
+      Stack.show_details(config)
+    end
+
+    if options['deploy-all']
+      Stack.deploy_all(config)
+    end
+
+    if options['delete-node']
+      info { "delete-node passed #{options['delete-node']}" }
+      Stack.delete_node(config, options['delete-node'])
+    end 
+  end
+
+  # supplemental methods here
+
+  # Declare command-line interface here
+
+  description "Deploy your stack to the cloud"
+  #
+  # Accept flags via:
+  # on("--flag VAL","Some flag")
+  # on("--flag VAL","Some flag")
+  # options[flag] will contain VAL
+  #
+  # Specify switches via:
+  # on("--[no-]switch","Some switch")
+
+  arg :stack
+  options['configfile'] = 'Stackfile'
+  on("--configfile configfile", "Specify an alternative to ./Stackfile")
+  on("--show-stack", "Show the nodes defined in a stack")
+  on("--show-details", "Show the nodes defined in a stack in detail")
+  on("--show-running", "Show the running nodes")
+  on("--show-dns", "Show the current DNS config")
+  on("--deploy-all", "Deploy the stack, skipping existing nodes")
+  on("--replace-node NODE", "Replace a give node")
+  on("--delete-node NODE", "Destroy a node, including any shared CM & DNS")
+  on("--validate", "Validate the config, check as much as possible with out creating nodes")
+  
+  version Kicker::VERSION
+
+  use_log_level_option
+
+  go!
+end

data/features/kicker.feature

@@ -0,0 +1,13 @@
+Feature: My bootstrapped app kinda works
+  In order to get going on coding my awesome app
+  I want to have aruba and cucumber setup
+  So I don't have to do it myself
+
+  Scenario: App just runs
+    When I get help for "kicker"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+    And the banner should document that this app takes no arguments

data/features/step_definitions/kicker_steps.rb

@@ -0,0 +1 @@
+# Put your step definitions here

data/features/support/env.rb

@@ -0,0 +1,16 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/kicker/version.rb

@@ -0,0 +1,3 @@
+module Kicker
+  VERSION = "0.0.2"
+end

data/lib/kicker.rb

@@ -0,0 +1,5 @@
+require "kicker/version"
+
+module Kicker
+  # Your code goes here...
+end

data/lib/stack.rb

@@ -0,0 +1,215 @@
+require 'pp'
+require 'fog'
+
+module Stack
+
+  def Stack.load_config(configfile, stack)
+    config_raw = File.read(configfile)
+    eval(config_raw) 
+    config = StackConfig::Stacks[stack]
+    config
+  end
+
+  def Stack.connect(config)
+    connection = Fog::Compute.new({ :provider => config[:provider],
+                                    :aws_access_key_id => config[:aws_access_key_id],
+                                    :aws_secret_access_key => config[:aws_secret_access_key],
+                                    :region => config[:region] })
+    connection
+  end
+
+  def Stack.populate_config(config)
+    # build out the full config for each node, supplying defaults from the 
+    # global config if explicitly supplied
+    config[:node_details] = Hash.new if config[:node_details].nil?
+
+    config[:roles].each do |role, role_details|
+      fqdn = role.to_s + '.' + config[:dns_domain]
+      
+      config[:node_details][fqdn] = { 
+        # set the node details from the role, if not specified in the role, use the config global
+        # (takes advantage of left to right evaluation of ||)
+        :flavor_id          => (role_details[:flavor_id] || config[:flavor_id]),
+        :count              => (role_details[:count] || 1),
+        :publish_private_ip => (role_details[:publish_private_ip] || false),
+        :dns_wildcard       => (role_details[:dns_wildcard] || false)
+      }
+    end
+  end
+
+  def Stack.generate_hostnames(config)
+    stack_hostnames = Array.new
+    config[:roles].each do |role, role_details|
+      fqdn = role.to_s + '.' + config[:dns_domain]
+      stack_hostnames << fqdn
+    end
+    stack_hostnames 
+  end
+
+  def Stack.deploy_all(config)
+    # generate the hostnames & details from the config & apply defaults where required
+    Stack.populate_config(config)
+
+    # create a connection
+    connection = Stack.connect(config)
+
+    running_instances = Stack.get_running(config)
+    config[:roles].each do |role, role_details|
+      hostname = role.to_s
+      fqdn = role.to_s + '.' + config[:dns_domain]
+
+      if !running_instances[fqdn].nil?
+        puts "Skipping #{fqdn} as it already exists"
+        next
+      end
+
+      # Ubuntu 8.04/Hardy doesn't do full cloud-init, so we have to script setting the hostname
+      user_data = File.read('user-data.sh')
+      user_data.gsub!(/rentpro-unconfigured/, hostname)
+      user_data.gsub!(/rentpro-stage.local/, config[:dns_domain])
+
+      # pp multipart
+      #
+      puts "Bootstraping new instance - #{fqdn}, in #{config[:availability_zone]}"
+      server = connection.servers.create({ 
+                                        :name => fqdn,
+                                        :hostname => fqdn,
+                                        :availability_zone => config[:availability_zone], 
+                                        :flavor_id => config[:node_details][fqdn][:flavor_id],
+                                        :image_id => config[:image_id],
+                                        :key_name => config[:keypair],
+                                        :user_data => user_data,
+                                        :tags => { 'Name' => fqdn },
+                                        })
+
+      print "Waiting for instance to be ready..."
+      server.wait_for { ready? }
+      puts "#{role.to_s} is booted, #{server.public_ip_address}/#{server.private_ip_address}"
+
+      # create/update the public & private DNS for this host
+      Stack.update_dns(role.to_s + '-public.' + config[:dns_domain], server.public_ip_address, config)
+      Stack.update_dns(role.to_s + '-private.' + config[:dns_domain], server.private_ip_address, config)
+      
+      # create the dns
+      if (role_details[:publish_private_ip] == true && (!role_details[:publish_private_ip].nil?))
+        ip_address = server.private_ip_address
+      else
+        ip_address = server.public_ip_address
+      end
+      Stack.update_dns(fqdn, ip_address, config)
+      #
+      # is this a wildcard DNS host, then claim the *.domain.net
+      if (role_details[:dns_wildcard] == true && (!role_details[:dns_wildcard].nil?))
+        wildcard = "*." + config[:dns_domain]
+        Stack.update_dns(wildcard, ip_address, config)
+      end
+    end
+  end
+
+  def Stack.update_dns(fqdn, ip_address, config)
+    # now register it in DNS
+    dns = Fog::DNS.new({ :provider => config[:provider],
+                          :aws_access_key_id => config[:aws_access_key_id],
+                          :aws_secret_access_key => config[:aws_secret_access_key] })
+          
+    # pp dns.get_hosted_zone(config[:dns_id])
+        
+    bmtw =  dns.zones.get(config[:dns_id])
+      
+    record = bmtw.records.get(fqdn) 
+    if record 
+      record.modify(:value => ip_address) if record 
+    else
+      bmtw.records.create(:value => ip_address, :name => fqdn, :type => 'A')
+    end
+  end
+
+  def Stack.show_dns(config)
+    # now register it in DNS
+    dns = Fog::DNS.new({ :provider => config[:provider],
+                          :aws_access_key_id => config[:aws_access_key_id],
+                          :aws_secret_access_key => config[:aws_secret_access_key] })
+          
+    zone = dns.zones.get(config[:dns_id])
+    if zone.records.empty?
+      puts "No DNS records found in #{config[:dns_domain]}"
+    else
+      printf("%40s %20s %5s %5s\n", 'fqdn', 'value', 'type', 'ttl')
+      zone.records.each do |record|
+        printf("%40s %20s %5s %5d\n", record.name, record.value, record.type, record.ttl)
+      end
+    end 
+  end
+
+  def Stack.get_running(config)
+    # create a connection
+    connection = Stack.connect(config)
+  
+    # generate all the names that this stack will use
+    stack_hostnames = Stack.generate_hostnames(config)
+
+    # Amazon EC2, use the tags hash to find hostnames
+    running_instances = Hash.new
+    connection.servers.each do |instance|
+      # pp instance
+      if (!instance.tags['Name'].nil? && instance.state != 'terminated' && instance.state != 'shutting-down')
+        hostname = instance.tags['Name']
+        if stack_hostnames.include?(hostname)
+          running_instances[hostname] = instance
+        end
+      end
+    end
+    running_instances 
+  end
+
+  def Stack.show_running(config)
+    running_instances = Stack.get_running(config)
+    running_instances.each do |instance, instance_details|
+      # display some details
+      puts "#{instance} id=#{instance_details.id} flavor_id=#{instance_details.flavor_id} public_ip=#{instance_details.public_ip_address} private_ip=#{instance_details.private_ip_address}"
+    end
+  end
+   
+  def Stack.delete_node(config, fqdn)
+    running_instances = Stack.get_running(config)
+    if running_instances[fqdn].nil?
+      puts "ERROR: #{fqdn} isn't running!"
+      exit
+    else
+      connection = Stack.connect(config)
+      pp running_instances[fqdn]
+      running_instances[fqdn].destroy
+    end
+  end
+
+  def Stack.show_details(config)
+    # create a connection
+    connection = Stack.connect(config)
+    
+    pp connection.describe_regions
+    pp connection.describe_availability_zones
+
+    pp connection.servers
+
+    Stack.populate_config(config)
+    pp config[:node_details]
+  end
+
+  def upload_keys(config)
+    if (key_pair = connection.key_pairs.get(config[:keypair]).nil?)
+      key_pair = connection.key_pairs.create( :name => config[:keypair], :public_key => File.read("rentpro-deploy.pub") )
+    else
+      puts "#{config[:keypair]} key_pair already exists"
+    end
+  end
+
+  def shutdown_all(config)
+    # shutdown all instances
+    connection.servers.select do |server| 
+      puts "Running server:"
+      # pp server
+    #  server.ready? && server.destroy
+    end
+  end
+
+end

data/test/tc_something.rb

@@ -0,0 +1,7 @@
+require 'test/unit'
+
+class TestSomething < Test::Unit::TestCase
+  def test_truth
+    assert true
+  end
+end

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: aws-kicker
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Simon McCartney
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: methadone
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+description: Utility for kicking an application stack into life on Amazon EC2
+email:
+- simon@mccartney.ie
+executables:
+- aws-kicker
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- README.rdoc
+- Rakefile
+- aws-kicker.gemspec
+- bin/aws-kicker
+- features/kicker.feature
+- features/step_definitions/kicker_steps.rb
+- features/support/env.rb
+- lib/kicker.rb
+- lib/kicker/version.rb
+- lib/stack.rb
+- test/tc_something.rb
+homepage: https://github.com/simonmcc/kicker
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ! 'Stacks are built from a collection of instances required to build an application
+  stack.  Each instance is described in the Stackfile & provisioned using variety
+  of methods. Supported models include: EC2 Create with user-data, cloud-init from
+  user-data (installs masterless puppet), puppet provision the instance.  Other models
+  could include using a puppet master, Chef Solo or Chef Server/Hosted. Amazon EC2
+  interaction is done through fog, so other providers should be easily added.  The
+  guiding principle is that your Stackfile should be shareable & re-useable by others,
+  and support templates, so that other users can use the template and easily adjust
+  items in the stack (such as the instance size used, DNS Domain updated during deploy,
+  EC2 account & location)'
+test_files:
+- features/kicker.feature
+- features/step_definitions/kicker_steps.rb
+- features/support/env.rb
+- test/tc_something.rb