aws-carb 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a8091f519cc7a1feb9f06b8e7970b160e9fa18d8
+  data.tar.gz: 3ec84346a2226990efd24aa108e5a8f346c42977
+SHA512:
+  metadata.gz: 4042bb7bdb921791fa89ff6c6c76da063b95a95dbf89c91df3c66ce4b3bb7f95cdcd58c69c6f475dfdc560d30f5439d42bc14a41873d84127998d51fcac10ab5
+  data.tar.gz: 189636adab38df5f0412792e82e2173fbf861e8a76b49b2634719b7faad8c252c52887400c720b21efead5976afc2b5d220613df363d30368c752e8bd01a203e

data/.gitignore

@@ -0,0 +1,5 @@
+.*.sw?
+*.lock
+*.gem
+.bundle
+config.yaml

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,31 @@
+# aws-carb: amazon web services - cloudinit and route53 bootstrap
+
+## Install
+
+via rubygems:
+
+```
+gem install aws-carb
+```
+
+using bundler:
+
+```
+bundler install
+```
+
+## Configuration
+
+At minimum, Carb needs to know your aws-ec2 credentials. The simplest way to allow this is to edit ~/.carb/config/config.yaml. See config/config.yaml.example for an example config.
+
+
+## Example usage
+
+carb can be used to create ec2 instances
+
+```
+aws create
+```
+
+
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/aws-carb.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aws-carb/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "aws-carb"
+  spec.version       = AWSCarb::VERSION
+  spec.authors       = ["Rob Wilson"]
+  spec.email         = ["roobert@gmail.com"]
+  spec.summary       = "aws - cloudinit and route53 bootstrap"
+  spec.description   = "a tool for provisioning ec2 instances with a templated cloudinit configuration, with the optional ability to update route53 with dns records to point at your new instance"
+  spec.homepage      = "http://github.com/roobert/aws-carb"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake",    "~> 10.0.0"
+
+  spec.add_dependency "activesupport", ">= 4.0.0"
+  spec.add_dependency "andand",        ">= 1.3.0"
+  spec.add_dependency "awesome_print", ">= 1.2.0"
+  spec.add_dependency "aws-sdk",       ">= 1.33.0"
+  spec.add_dependency "colorize",      ">= 0.6.0"
+  spec.add_dependency "erubis",        ">= 2.7.0"
+  spec.add_dependency "shell-spinner", ">= 1.0.0"
+  spec.add_dependency "subcommand",    ">= 1.0.0"
+end

data/bin/carb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '../lib'))
+
+require 'aws-carb'
+require 'aws-carb/helpers'
+require 'aws-carb/cli_argument_parser'
+require 'aws-carb/config'
+require 'aws-carb/user_data'
+require 'aws-carb/services/route53'
+require 'aws-carb/services/ec2'
+require 'aws-carb/monkey_patches'
+
+AWSCarb.run

data/configs/config.yaml.example

@@ -0,0 +1,24 @@
+common:
+  domain: <domain>
+
+ec2:
+  access_key_id: <key_id>
+  secret_access_key: <secret_key>
+
+template_options:
+  repository_server: <repo_server>
+  debian_distro: <distro>
+  puppet_master: <puppetmaster>
+  s3_access_key_id: <key_id>
+  s3_secret_access_key: <secret_key>
+  s3_region: us-east-1
+  ssh_keys:
+    - <key>
+    - <key>
+    - <key>
+
+route53:
+  access_key_id: <key_id>
+  secret_access_key: <secret_key>
+  zone: <zone_id>
+  ttl: 600

data/lib/aws-carb/cli_argument_parser.rb

@@ -0,0 +1,359 @@
+#!/usr/bin/env ruby
+
+include Subcommands
+
+module AWSCarb
+  module CliArgumentParser
+    def self.parse
+      cli_arguments                               = OpenStruct.new
+      cli_arguments.global                        = OpenStruct.new
+      cli_arguments.subcommand                    = OpenStruct.new
+      cli_arguments.subcommand.user_data_template = OpenStruct.new
+      cli_arguments.subcommand.config_overrides   = OpenStruct.new
+      cli_arguments.subcommand.ec2                = OpenStruct.new
+      cli_arguments.subcommand.route53            = OpenStruct.new
+
+      indent = ' ' * 10 
+
+      banner = <<-HEREDOC.strip_heredoc
+      #{File.basename($0)}(1)
+
+      #{'NAME'.colorize({:color => :white, :mode => :bold})}
+
+                              carb
+
+      #{'DESCRIPTION'.colorize({:color => :white, :mode => :bold})}
+
+            amazon web services - cloudinit and route53 bootstrap
+
+      HEREDOC
+
+
+      # these are the only defaults we need to bother setting since they get used before we load the config file..
+      cli_arguments.global.verbose = false
+      cli_arguments.global.config_file = File.join(Dir.home, ".carb/config", "config.yaml")
+
+      global_options do |option|
+
+        option.summary_width  = 50
+        option.summary_indent = '    '
+
+        synopsis = <<-HEREDOC.strip_heredoc
+        #{"SYNOPSIS".colorize({:color => :white, :mode => :bold})}
+        
+            #{File.basename($0)} [global options] [subcommand [options]]
+
+        HEREDOC
+
+        option.banner = banner + synopsis
+        option.separator "#{'GLOBAL OPTIONS'.colorize({:color => :white, :mode => :bold})}"
+        option.separator ""
+
+        option.on("-c", "--config=FILE", "") do |file|
+          cli_arguments.global.config_file = file
+        end
+
+        option.separator ""
+
+        option.on("-v", "--verbose", "") do |boolean|
+          cli_arguments.global.verbose = boolean
+
+          # FIXME: use a stupidly named global(!!!!!) variable to avoid clashing with rubys $VERBOSE
+          $GLOBAL_VERBOSE = boolean
+        end
+
+        option.separator ""
+        option.separator "    -h, --help"
+      end
+
+      command :create do |option|
+
+        option.summary_width  = 50
+        option.summary_indent = '  '
+
+        synopsis = <<-HEREDOC.strip_heredoc
+        #{'SYNOPSIS'.colorize({:color => :white, :mode => :bold})}
+
+            #{File.basename($0)} create [options]
+
+        HEREDOC
+
+        option.banner      = banner + synopsis
+        option.description = "create an ec2 instance"
+
+        option.separator ""
+        option.separator "    user_data tempate options:"
+        option.separator ""
+
+        option.on "--user-data-template=FILE", "\n\n#{indent}user data template" do |user_data_template|
+          cli_arguments.subcommand.user_data_template.file = user_data_template
+        end
+
+        option.separator ""
+
+        option.on "--show-parsed-template=BOOLEAN", "\n\n#{indent}display parsed template file" do |show_parsed_template|
+          cli_arguments.subcommand.user_data_template.show_parsed_template = show_parsed_template
+        end
+
+        option.separator ""
+
+        option.separator ""
+        option.separator "    config file overrides:"
+        option.separator ""
+
+        option.on "--common-variables=HASH", String, "\n\n#{indent}common variables" do |common_variables|
+          begin
+            data = eval(common_variables)
+            raise unless data.class == Hash
+            cli_arguments.subcommand.config_overrides.common_variables = data.deep_symbolize_keys
+          rescue => e
+            puts "# could not parse argument for --common-variables, is it a valid hash?"
+            die e
+          end
+        end
+
+        option.separator ""
+
+        option.on "--general-variables=HASH", String, "\n\n#{indent}general variables" do |general_variables|
+          begin
+            data = eval(general_variables)
+            raise unless data.class == Hash
+            cli_arguments.subcommand.config_overrides.general_variables = data.deep_symbolize_keys
+          rescue => e
+            puts "# could not parse argument for --general-variables, is it a valid hash?"
+            die e
+          end
+        end
+
+        option.separator ""
+
+        option.on "--ec2-variables=HASH", String, "\n\n#{indent}ec2 variables" do |ec2_variables|
+          begin
+            data = eval(ec2_variables)
+            raise unless data.class == Hash
+            cli_arguments.subcommand.config_overrides.ec2_variables = data.deep_symbolize_keys
+          rescue => e
+            puts "# could not parse argument for --ec2-variables, is it a valid hash?"
+            die e
+          end
+        end
+
+        option.separator ""
+
+        option.on "--route53-variables=HASH", String, "\n\n#{indent}route53 variables" do |route53_variables|
+          begin
+            data = eval(route53_variables)
+            raise unless data.class == Hash
+            cli_arguments.subcommand.config_overrides.route53_variables = data.deep_symbolize_keys
+          rescue => e
+            puts "# could not parse argument for --route53-variables, is it a valid hash?"
+            die e
+          end
+        end
+
+        option.separator ""
+
+        option.on "--user-data-template-variables=HASH", String, "\n\n#{indent}user data template variables" do |user_data_template_variables|
+          begin
+            data = eval(user_data_template_variables)
+            raise unless data.class == Hash
+            cli_arguments.subcommand.config_overrides.user_data_template_variables = data.deep_symbolize_keys
+          rescue => e
+            puts "# could not parse argument for --user-data-template-variables, is it a valid hash?"
+            die e
+          end
+        end
+
+        option.separator ""
+        option.separator "          long descriptions for these parameters can be found here:\n            http://<TODO>"
+        option.separator ""
+
+        option.separator ""
+        option.separator "    ec2 config convenience arguments:"
+        option.separator ""
+
+        option.on "--ec2-access-key-id=STRING", "\n\n#{indent}access key id".downcase do |access_key_id|
+          cli_arguments.subcommand.ec2.access_key_id = access_key_id
+        end
+
+        option.separator ""
+
+        option.on "--ec2-secret-access-key=STRING", "\n\n#{indent}secret access key".downcase do |secret_access_key|
+          cli_arguments.subcommand.ec2.secret_access_key = secret_access_key
+        end
+
+        option.separator ""
+
+        option.on "--image-id=STRING", "\n\n#{indent}ID of the AMI you want to launch.".downcase do |image_id|
+          cli_arguments.subcommand.ec2.image_id = image_id
+        end
+
+        option.separator ""
+
+        option.on "--instance-type=STRING", "\n\n#{indent}The type of instance to launch, for example \"m1.small\".".downcase do |instance_type|
+          cli_arguments.subcommand.ec2.instance_type = instance_type
+        end
+
+        option.separator ""
+
+        option.on "--key-name=STRING", "\n\n#{indent}The name of the key pair to use.".downcase do |key_name|
+          cli_arguments.subcommand.ec2.key_name = key_name
+        end
+
+        option.separator ""
+
+
+        block_device_help = <<-HEREDOC.strip_heredoc
+
+           :virtual_name - (String) Specifies the virtual device name.
+           :device_name - (String) Specifies the device name (e.g., /dev/sdh).
+           :ebs - (Hash) Specifies parameters used to automatically setup Amazon\n#{indent}             EBS volumes when the instance is launched.
+             :snapshot_id - (String) The ID of the snapshot from which the volume will be created.
+             :volume_size - (Integer) The size of the volume, in gigabytes.
+             :delete_on_termination - (Boolean) Specifies whether the Amazon EBS volume is\n#{indent}                          deleted on instance termination.
+             :volume_type - (String) Valid values include:
+               standard
+               io1
+           :iops - (Integer)
+           :no_device - (String) Specifies the device name to suppress during instance launch.
+        HEREDOC
+
+        block_device_help = block_device_help.lines.map { |line| indent + "  #{line}" }
+
+        block_device_help = "\n\n#{indent}Specifies how block devices are exposed to the instance. Each mapping\n#{indent}is made up of a virtualName and a deviceName.\n" + block_device_help.join.downcase
+
+        option.on "--block-device-mappings=HASH", block_device_help do |key_name|
+          cli_arguments.subcommand.ec2.key_name = key_name
+        end
+
+        option.separator ""
+
+        option.on "--user-data=STRING", "\n\n#{indent}Arbitrary user data. note: this is merged with user_data_template if also specified.".downcase do |user_data|
+          cli_arguments.subcommand.ec2.user_data = user_data
+        end
+
+        option.separator ""
+
+        option.on "--iam-instance-profile=STRING", "\n\n#{indent}the name or ARN of an IAM instance profile.".downcase do |profile|
+          cli_arguments.subcommand.ec2.iam_instance_profile = profile
+        end
+
+        option.separator ""
+
+        option.on "--monitoring-enabled=BOOLEAN", "\n\n#{indent}enable CloudWatch monitoring.".downcase do |boolean|
+          cli_arguments.subcommand.ec2.monitoring_enabled = boolean
+        end
+
+        option.separator ""
+
+        option.on "--availability-zone=STRING", "\n\n#{indent}availability zone.".downcase do |zone|
+          cli_arguments.subcommand.ec2.availability_zone = zone
+        end
+
+        option.separator ""
+
+        option.on "--security-groups=ARRAY", Array, "\n\n#{indent}Security groups. can be a single value or an array of values.\n#{indent}Values should be space deliminated group name strings.".downcase do |groups|
+          cli_arguments.subcommand.ec2.security_groups = groups
+        end
+
+        option.separator ""
+
+        option.on "--security-group-ids=ARRAY", Array, "\n\n#{indent}security_group_ids accepts a single ID or an array of\n#{indent}security group IDs.".downcase do |group_ids|
+          cli_arguments.subcommand.ec2.security_group_ids = group_ids
+        end
+
+        option.separator ""
+
+        option.on "--disable-api-termination=BOOLEAN", "\n\n#{indent}instance termination via the instance API.".downcase do |api_termination|
+          cli_arguments.subcommand.ec2.disable_api_termination = api_termination
+        end
+
+        option.separator ""
+
+        option.on "--instance-initiated-shutdown-behavior=STRING", "\n\n#{indent}instance termination on instance-initiated shutdown".downcase do |shutdown_behavior|
+          cli_arguments.subcommand.ec2.instance_initiated_shutdown_behavior = shutdown_behavior
+        end
+
+        option.separator ""
+
+        option.on "--subnet=STRING", "\n\n#{indent}The VPC Subnet (or subnet id string) to launch the instance in.".downcase do |subnet|
+          cli_arguments.subcommand.ec2.subnet = subnet
+        end
+
+        option.separator ""
+
+        option.on "--private_ip_address=STRING", "\n\n#{indent}If you're using VPC, you can optionally use this option to assign the\n#{indent}instance a specific available IP address from the subnet (e.g., '10.0.0.25').\n#{indent}This option is not valid for instances launched outside a VPC (i.e.\n#{indent}those launched without the :subnet option).".downcase do |ip|
+          cli_arguments.subcommand.ec2.private_ip_address = ip
+        end
+
+        option.separator ""
+
+        option.on "--dedicated-tenancy=BOOLEAN", "\n\n#{indent}Instances with dedicated tenancy will not share physical hardware with\n#{indent}instances outside their VPC. NOTE: Dedicated tenancy incurs an \n#{indent}additional service charge. This option is not valid for\n#{indent}instances launched outside a VPC (e.g.those launched without the :subnet option).".downcase do |tenancy|
+          cli_arguments.subcommand.ec2.dedicated_tenancy = tenancy
+        end
+
+        option.separator ""
+
+        option.on "--ebs-optimized=BOOLEAN", "\n\n#{indent}EBS-Optimized instances enable Amazon EC2 instances to fully utilize the\n#{indent}IOPS provisioned on an EBS volume. EBS-optimized instances deliver dedicated\n#{indent}throughput between Amazon EC2 and Amazon EBS, with options between\n#{indent}500 Mbps and 1000 Mbps depending on the instance type used. When attached\n#{indent}to EBS-Optimized instances, Provisioned IOPS volumes are designed to\n#{indent}deliver within 10% of their provisioned performance 99.9% of the time.\n#{indent}NOTE: EBS Optimized instances incur an additional service charge.\n#{indent}This option is only valid for certain instance types.".downcase do |ebs_optimized|
+          cli_arguments.subcommand.ec2.ebs_optimized = ebs_optimized
+        end
+
+        option.separator ""
+        option.separator "        long descriptions for these parameters can be found here:\n          http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/InstanceCollection.html"
+        option.separator ""
+
+        option.separator ""
+        option.separator "    route53 convenience arguments:"
+        option.separator ""
+
+        option.on "--route53-access-key-id=STRING", "\n\n#{indent}access key id".downcase do |access_key_id|
+          cli_arguments.subcommand.route53.access_key_id = access_key_id
+        end
+
+        option.separator ""
+
+        option.on "--route53-secret-access-key=STRING", "\n\n#{indent}secret access key".downcase do |secret_access_key|
+          cli_arguments.subcommand.route53.secret_access_key = secret_access_key
+        end
+
+        option.separator ""
+
+
+        option.on "--zone=STRING", "\n\n#{indent}route53 zone".downcase do |zone|
+          cli_arguments.subcommand.route53.route = route
+        end
+
+        option.separator ""
+
+        option.on "--ttl=STRING", "\n\n#{indent}ttl".downcase do |ttl|
+          cli_arguments.subcommand.route53.ttl = ttl
+        end
+
+        option.separator ""
+
+      end
+
+      begin
+        #cli_arguments.chosen_subcommand = opt_parse
+        subcommand = opt_parse
+
+        # show help if no arguments passed in
+        if subcommand.nil?
+          add_subcommand_help
+          puts @global
+
+          exit 1
+        end
+
+        if ARGV.length > 0
+          raise ArgumentError, "unknown command line argument(s): #{ARGV.inspect.to_s}"
+        end
+      rescue => e
+        puts e
+        exit 1
+      end
+
+      return cli_arguments
+    end
+  end
+end

data/lib/aws-carb/config.rb

@@ -0,0 +1,166 @@
+#!/usr/bin/env ruby
+
+module AWSCarb
+  class Config
+    include Singleton
+
+    attr_reader :config
+
+    def create(cli_arguments)
+
+      @config = load_file(cli_arguments.global.config_file)
+
+      merge_cli_arguments_with_config(cli_arguments)
+
+      establish_hostname_and_domain
+
+      check_route53_settings
+    end
+
+    def check_route53_settings
+      die 'route53: no zone id specified!' if @config[:route53][:zone].nil?
+      die 'route53: no ttl specified!'     if @config[:route53][:zone].nil?
+    end
+
+    def merge_cli_arguments_with_config(cli_arguments)
+      begin
+
+        config_sections = [:common, :general, :ec2, :route53, :user_data_template_variables, :user_data_template]
+
+        # special condition: common command line arguments are shared between all instances first..
+        if cli_arguments.subcommand.config_overrides.common_variables
+          @config[:common] ||= {}
+          @config[:common].merge! cli_arguments.subcommand.config_overrides.common_variables
+        end
+
+        # all sections share 'common' variables..
+        config_sections.each do |section|
+          @config[section] ||= {}
+          @config[section].merge! @config[:common]
+        end
+
+        # merge the config overrides hashes into config
+        if cli_arguments.subcommand.config_overrides
+          cli_arguments.subcommand.config_overrides.marshal_dump.each do |key, value|
+
+            next if key == :common
+
+            # key differs from command line argument - we lose the _variables suffix
+            config_key = key.to_s.gsub('_variables', '').to_sym
+
+            @config[config_key] ||= {}
+            @config[config_key].merge! cli_arguments.subcommand.config_overrides.send(key)
+          end
+        end
+
+        # merge the convenience arguments..
+        config_sections.each do |section|
+          if cli_arguments.subcommand.send(section.to_s)
+            @config[section].merge! cli_arguments.subcommand.send(section.to_s).marshal_dump
+          end
+        end
+
+        # merge the convenience argument parameters with config
+        @config.deep_symbolize_keys!
+
+      rescue => e
+        puts "# failed to merge cli arguments with config"
+        die e
+      end
+    end
+
+    def load_file(cli_argument_config_file)
+
+      # allow forcing of no config file..
+      return if cli_argument_config_file.empty?
+
+      config_file = cli_argument_config_file
+
+      begin
+        # make keys symbols so we can more easily merge with cli arg structs..
+        @config = YAML.load_file(config_file).deep_symbolize_keys
+      rescue => e
+        puts "# failed to load config file: '#{config_file}'"
+        die e
+      end
+    end
+
+    # when looking for a key, check 'common' section first, then override if a value
+    # in the supplied context is found..
+    def find_with_context(key, context) 
+      return @config[context][key] if @config[context][key]
+      return @config[:common][key] if @config[:common][key]
+      return nil
+    end
+
+    def [](key)
+      @config[key]
+    end
+
+    # try and work out the hostname, presidence is:
+    #
+    # * config file
+    # * user_data_template_variables cli args
+    # 
+    # note: raw user_data is not checked (not to be confused with user_data_template or user_data_template_variables..)
+    #
+    def establish_hostname_and_domain
+      ShellSpinner "# checking whether hostname and domain have been set", false do
+
+        hostname, domain = nil
+
+        @config[:route53][:hostname] = find_with_context(:hostname, :user_data_template_variables) if find_with_context(:hostname, :user_data_template_variables)
+        @config[:route53][:domain]   = find_with_context(:domain,   :user_data_template_variables) if find_with_context(:domain,   :user_data_template_variables)
+        @config[:route53][:hostname] = find_with_context(:hostname, :route53)                      if find_with_context(:hostname, :route53)
+        @config[:route53][:domain]   = find_with_context(:domain, :route53)                        if find_with_context(:domain, :route53)
+
+        help = <<-HEREDOC.strip_heredoc
+          #       
+          #         checked:
+          #          'common', 'user_data_template_variables',
+          #          and 'route53' sections of config
+          #          --common-variables, --route53-variables,
+          #          and --user-data-template-variables
+          #
+          #          route53 dynamic DNS will not be updated!
+        HEREDOC
+
+        domain   = @config[:route53][:domain]
+        hostname = @config[:route53][:hostname]
+
+        if domain.nil? and hostname.nil?
+          debug "# WARNING: hostname and domain not found"
+          debug help
+          debug
+        elsif domain and hostname.nil?
+          debug "# WARNING: hostname not found"
+          debug help
+          debug
+        elsif domain.nil? and hostname
+          debug "# WARNING: domain not found"
+          debug help
+          debug
+        else
+          debug "# found hostname and domain:"
+          debug "hostname: #{hostname}"
+          debug "domain:   #{domain}"
+          debug
+
+          @config[:route53][:new_dns_records] = {
+            :public  => { :alias => "#{hostname}.#{domain}.",         :target => nil },
+            :private => { :alias => "#{hostname}-private.#{domain}.", :target => nil }
+          }
+        end
+      end
+
+      puts
+    end
+
+
+    def display
+      puts "# config:"
+      ap @config
+      puts
+    end
+  end
+end

data/lib/aws-carb/helpers.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+def die(error)
+  if $GLOBAL_VERBOSE and error.respond_to?('backtrace')
+    puts "# stack trace:"
+    puts error.backtrace
+    puts
+  end
+
+  puts "error: #{error}"
+
+  exit 1
+end
+
+def debug(message = nil)
+  puts message if @GLOBAL_VERBOSE
+end