cloud-maker 0.0.0

data/bin/cloud-maker

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+require 'thor'
+require 'colorize'
+require 'cloud-maker'
+# require 'pry'
+
+class CloudMakerCLI < Thor
+  include Thor::Actions
+
+  desc "launch [INSTANCE_CONFIG_YAML]", "Launch a new EC2 instance as described by INSTANCE_CONFIG_YAML"
+  method_option :aws_access_key_id,
+    :desc => "Your AWS access key id",
+    :default => ENV['AWS_ACCESS_KEY_ID'],
+    :required => true
+  method_option :aws_secret_access_key,
+    :desc => "Your AWS secret access key",
+    :default => ENV['AWS_SECRET_ACCESS_KEY'],
+    :required => true
+  method_option :set,
+    :alias => '-s',
+    :type => :hash,
+    :default => {},
+    :desc => "Set parameters in the CloudMaker config"
+  def launch(instance_config_yaml)
+    puts "---------------------------"
+    puts "Launching new EC2 instance"
+    puts "---------------------------\n"
+
+    config = CloudMaker::Config.from_yaml(instance_config_yaml)
+    options.set.each_pair {|key, val| config[key] = val}
+
+    if !config.valid?
+      say "Before an instance can be launched we need a few more values to be specified.".yellow
+      say "Currently missing: #{config.missing_values.map{|key| key.cyan}.join(', ')}"
+      puts
+
+      config.missing_values.each do |key|
+        config[key] = ENV[key] if ENV[key]
+      end
+
+      config.missing_values.each do |key|
+        if (config.options[key]["description"])
+          say config.options[key]["description"]
+        end
+        config[key] = ask "Please choose a value for #{key.cyan}: "
+      end
+    end
+
+    cloud_maker = CloudMaker::Ec2.new(config,
+      :aws_access_key_id => options.aws_access_key_id,
+      :aws_secret_access_key => options.aws_secret_access_key
+    )
+
+    if (!config.includes.empty?)
+      puts
+      say 'Include URLs:'.green
+      config.includes.each do |url|
+        puts url
+      end
+    end
+
+    puts
+    say "CloudMaker configuration:".green
+    print_table config.to_hash.map {|key, val| [key.dup.cyan, val]}
+
+    puts
+    say "Configuration file:".green + " " + config.extra_options[:config_path]
+    puts
+
+    if yes?("Launch a new EC2 instance with the options above? (y/n)")
+      instance = cloud_maker.launch
+      puts
+      say "Successfully launched new EC2 instance: ".green + instance[:aws_instance_id].magenta
+      puts
+      print_table instance.map {|key, val|
+        [key.to_s.cyan, val.to_s]
+      }
+    else
+      say "Launch aborted!".red
+    end
+  end
+
+  desc "scratch", "Play with Thor"
+  def scratch(arg1)
+    say 'hello'
+    say 'hello in red'.red
+  end
+end
+
+CloudMakerCLI.start

data/lib/cloud-maker.rb

@@ -0,0 +1,4 @@
+require 'cloud_maker/config'
+require 'cloud_maker/ec2'
+require 'cloud_maker/s3_archiver'
+require 'cloud_maker/shell_executor'

data/lib/cloud_maker/config.rb

@@ -0,0 +1,302 @@
+require 'yaml'
+require 'stringio'
+
+module CloudMaker
+  class Config
+    # Public: Gets/Sets the CloudMaker specific properties. The options hash
+    # is formatted as:
+    #    {
+    #      'key1' => {
+    #         'environment': boolean,
+    #         'required': boolean,
+    #         'value': value
+    #      },
+    #      'key2' => { ... },
+    #      ...
+    #    }
+    attr_accessor :options
+    # Public: Gets/Sets the Hash of Cloud Init properties. See
+    # https://help.ubuntu.com/community/CloudInit for valid options
+    attr_accessor :cloud_config
+    # Public: Gets/Sets an Array of URLs to be included, this corresponds to the
+    # list of URLs in a Cloud Init includes file.
+    attr_accessor :includes
+    # Public: Gets/Sets extra information about the config to be stored for
+    # archival purposes.
+    attr_accessor :extra_options
+
+    # Internal: A mime header for the Cloud Init config section of the user data
+    CLOUD_CONFIG_HEADER = %Q|Content-Type: text/cloud-config; charset="us-ascii"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Disposition: attachment; filename="cloud-config.yaml"\n\n|
+    # Internal: A mime header for the includes section of the user data
+    INCLUDES_HEADER = %Q|Content-Type: text/x-include-url; charset="us-ascii"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Disposition: attachment; filename="includes.txt"\n\n|
+    # Internal: A mime header for the Cloud Boothook section of the user data
+    BOOTHOOK_HEADER = %Q|Content-Type: text/cloud-boothook; charset="us-ascii"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Disposition: attachment; filename="boothook.sh"\n\n|
+    # Internal: A multipart mime header for describing the entire user data
+    # content. It includes a placeholder for the boundary text '___boundary___'
+    # that needs to be replaced in the actual mime document.
+    MULTIPART_HEADER = %Q|Content-Type: multipart/mixed; boundary="___boundary___"\nMIME-Version: 1.0\n\n|
+
+    # Internal: If you don't specify a property associated with a key in the
+    # cloud_maker config file we will use these properties to fill in the blanks
+    DEFAULT_KEY_PROPERTIES = {
+      "environment" => true,
+      "required" => false,
+      "value" => nil
+    }
+
+    # Public: Initializes a new CloudMaker object.
+    #
+    # cloud_config - A Hash describing all properties of the CloudMaker config.
+    #   'cloud_maker' - The configuration properties for CloudMaker. These can
+    #                   be specified either as
+    #                      key: value
+    #                   or as
+    #                      key: {
+    #                        environment: boolean,
+    #                        required: boolean,
+    #                        value: value
+    #                      }
+    #                   If specified as key: value DEFAULT_KEY_PROPERTIES will
+    #                   be used. If the detailed version is used all properties
+    #                   are optional and DEFAULT_KEY_PROPERTIES will be used to
+    #                   fill in the blanks.
+    #
+    #   'include'     - An array of URLs or a String containing 1 URL per line
+    #                   with optional # prefixed lines as comments.
+    #   ...           - All valid properties of a Cloud Init config
+    #                   are also valid here. See:
+    #                   https://help.ubuntu.com/community/CloudInit
+    # extra_options - Extra information about the instantiation. These will not
+    #                 be used to launch the instance but will be stored in the
+    #                 archive describing the instance.
+    #
+    # Returns a CloudMaker object
+    def initialize(cloud_config, extra_options)
+      self.extra_options = extra_options
+      cloud_config = cloud_config.dup
+      self.options = extract_cloudmaker_config!(cloud_config)
+      self.includes = extract_includes!(cloud_config)
+      self.cloud_config = cloud_config
+    end
+
+    # Public: Check if the CloudMaker config is in a valid state.
+    #
+    # Returns true if and only if all required properties have non-nil values
+    #   and false otherwise.
+    def valid?
+      self.options.all? {|key, option| !(option["required"] && option["value"].nil?)}
+    end
+
+    # Public: Finds a list of keys in the CloudMaker config that are required to
+    # have a value but do not yet have one.
+    #
+    # Returns an Array of required keys that are missing values.
+    def missing_values
+      self.options.select {|key, option| option["required"] && option["value"].nil?}.map(&:first).map(&:dup)
+    end
+
+    # Returns a Hash of all of the CloudMaker specific properties in the configuration.
+    def to_hash
+      self.options.map {|key, properties| [key, properties["value"]]}
+    end
+
+    # Public: Generates a multipart userdata string suitable for use with Cloud Init on EC2
+    #
+    # Returns a String containing the mime encoded userdata
+    def to_user_data
+      # build a multipart document
+      parts = []
+
+      parts.push(BOOTHOOK_HEADER + boothook_data)
+      parts.push(CLOUD_CONFIG_HEADER + cloud_config_data)
+      parts.push(INCLUDES_HEADER + includes_data)
+
+      #not that it's likely but lets make sure that we don't choose a boundary that exists in the document.
+      boundary = ''
+      while parts.any? {|part| part.index(boundary)}
+        boundary = "===============#{rand(8999999999999999999) + 1000000000000000000}=="
+      end
+
+      header = MULTIPART_HEADER.sub(/___boundary___/, boundary)
+
+      return [header, *parts].join("\n--#{boundary}\n") + "\n--#{boundary}--"
+    end
+
+    # Public: Generates a cloud-init configuration
+    #
+    # Returns a String containing the cloud init configuration in YAML format
+    def cloud_config_data
+      return "#cloud-config\n#{self.cloud_config.to_yaml}"
+    end
+
+    # Public: Generates a shell script to set environment variables for all
+    # CloudMaker config options, will get executed at machine boot.
+    #
+    # Returns a String containing the shell script
+    def boothook_data
+      env_run_cmds = []
+
+      self.options.each_pair do |key, properties|
+        if properties["environment"] && !properties["value"].nil?
+          env_run_cmds.push(set_environment_variable_cmd(key, properties["value"]))
+        end
+      end
+
+      return "#cloud-boothook\n#!/bin/sh\n#{env_run_cmds.join("\n")}\n"
+    end
+
+
+    # Public: Generates a cloud-init includes list
+    #
+    # Returns a String containing the cloud init includes list
+    def includes_data
+      ["#include", *self.includes.map(&:to_s)].join("\n")
+    end
+
+
+    # Public: Access values in the cloudmaker options object
+    #
+    # key - The key of the property you're accessing
+    #
+    # Returns the value property for options[key]
+    def [](key)
+      self.options[key] ? self.options[key]["value"] : nil
+    end
+
+    # Public: Sets the value property for key in the cloudmaker options hash
+    #
+    # key - The key of the property you're accessing
+    # val - The value you wish to assign to the key
+    #
+    # Returns val
+    def []=(key, val)
+      if (self.options[key])
+        self.options[key]["value"] = val
+      else
+        self.options[key] = DEFAULT_KEY_PROPERTIES.merge('value' => val)
+      end
+      val
+    end
+
+    # Returns a String representation of the CloudMaker config
+    def inspect
+      "CloudMakerConfig#{self.options.inspect}"
+    end
+
+    class << self
+
+      # Public: Takes the path of a YAML file and loads a new Config object
+      # from it.
+      #
+      # instance_config_yaml - The path of the YAML file
+      #
+      # Returns a new Config
+      # Raises: Exception if the file doesn't exist.
+      # Raises: SyntaxError if the YAML file is invalid.
+      def from_yaml(instance_config_yaml)
+        begin
+          full_path = File.expand_path(instance_config_yaml)
+          cloud_yaml = File.open(full_path, "r") #Right_AWS will base64 encode this for us
+        rescue
+          raise "ERROR: The path to the CloudMaker config is incorrect"
+        end
+
+        CloudMaker::Config.new(YAML::load(cloud_yaml), :config_path => full_path)
+      end
+    end
+
+
+
+    private
+
+    # Internal: Takes a CloudMaker config and parses out the CloudMaker
+    # specific portions of it. For each key/value it fills in any property
+    # blanks from the DEFAULT_KEY_PROPERTIES. It also deletes the cloud_maker
+    # property from config.
+    #
+    # config - A hash that should contain a 'cloud_maker' key storing CloudMaker
+    #   configuration properties.
+    #
+    # Returns a Hash in the format of
+    #   {'key1' => {
+    #      'environment' => ...,
+    #      'value' => ...,
+    #      'required' ...
+    #   }, 'key2' => ... , ...}
+    def extract_cloudmaker_config!(config)
+      cloud_maker_config = config.delete('cloud-maker') || {}
+      cloud_maker_config.keys.each do |key|
+        #if key is set to anything but a hash then we treat it as the value property
+        if !advanced_config?(cloud_maker_config[key])
+          cloud_maker_config[key] = {
+            "value" => cloud_maker_config[key]
+          }
+        end
+
+        cloud_maker_config[key] = DEFAULT_KEY_PROPERTIES.merge(cloud_maker_config[key])
+      end
+      cloud_maker_config
+    end
+
+    # Internal: Determines if value should be treated as a value or a property
+    # configuration hash. A property configuration hash would specify at least
+    # one of environment, value, or required.
+    #
+    # value - The value to evaluate
+    #
+    # Returns true if value is a property configuration, and false if it's just
+    #   a value
+    def advanced_config?(value)
+      value.kind_of?(Hash) && !(DEFAULT_KEY_PROPERTIES.keys & value.keys).empty?
+    end
+
+    # Internal: Takes a CloudMaker config and parses out the includes list. If
+    # the list is an array it treats each entry as a URL. If it is a string it
+    # treats the string as the contents of a Cloud Init include file.
+    #
+    # config - A hash that should contain an 'include' key storing the include
+    # information.
+    #
+    # Returns an Array of URLs
+    def extract_includes!(config)
+      includes = config.delete('include')
+
+      #if we didn't specify it just use a blank array
+      if includes.nil?
+        includes = []
+      #if we passed something other than an array turn it into a string and split it up into urls
+      elsif !includes.kind_of?(Array)
+        includes = includes.to_s.split("\n")
+        includes.reject! {|line| line.strip[0] == "#" || line.strip.empty?}
+      end
+
+      includes
+    end
+
+    # Internal: Generates the shell command necessary to set an environment
+    # variable. It escapes the value but assumes there are no special characters
+    # in the key. If value is an array or a hash it generates an environment
+    # variable for each value in the array/hash with the key set to key_index.
+    #
+    # key   - The key of the environment variable
+    # value - The value to set the key to
+    #
+    # Returns a string that can be executed to globally set the environment variable.
+    def set_environment_variable_cmd(key, value)
+      if (value.kind_of?(Hash))
+        value.keys.map { |hash_key|
+          set_environment_variable_cmd("#{key}_#{hash_key}", value[hash_key])
+        }.join(';')
+      elsif (value.kind_of?(Array))
+        strings = []
+        value.each_with_index { |arr_val, i|
+          strings.push(set_environment_variable_cmd("#{key}_#{i}", arr_val))
+        }
+        strings.join(';')
+      else
+        escaped_value = value.to_s.gsub(/"/, '\\\\\\\\\"')
+        "echo \"#{key}=\\\"#{escaped_value}\\\"\" >> /etc/environment"
+      end
+    end
+  end
+end

data/lib/cloud_maker/ec2.rb

@@ -0,0 +1,80 @@
+require 'right_aws'
+
+module CloudMaker
+  class Ec2
+    # Public: Gets/Sets the AWS access key.
+    attr_accessor :aws_secret_access_key
+    # Public: Gets/Sets the AWS secret.
+    attr_accessor :aws_access_key_id
+    # Public: Gets/Sets the CloudMaker::Config
+    attr_accessor :config
+
+    # Public: Creates a new Ec2 instance
+    #
+    # cloud_maker_config - A CloudMaker::Config object describing the instance
+    #                      to be managed.
+    # options            - S3 configuration options
+    #                      :aws_access_key_id     - (required) The AWS access key
+    #                      :aws_secret_access_key - (required) The AWS secret
+    #
+    # Returns a new CloudMaker::Ec2 instance
+    # Raises RuntimeError if any of the required options are not specified
+    def initialize(cloud_maker_config, options)
+      required_keys = [:aws_access_key_id, :aws_secret_access_key]
+      unless (required_keys - options.keys).empty?
+        raise RuntimeError.new("Instantiated #{self.class} without required attributes: #{required_keys - options.keys}.")
+      end
+
+      self.config = cloud_maker_config
+      self.aws_access_key_id = options[:aws_access_key_id]
+      self.aws_secret_access_key = options[:aws_secret_access_key]
+    end
+
+    # Public: Launches a new EC2 instance, associates any specified elastic IPS
+    # with it, adds any specified tags, and archives the launch details to S3.
+    #
+    # Returns a RightAws supplied Hash describing the launched instance.
+    def launch
+      ec2 = RightAws::Ec2.new(self.aws_access_key_id, self.aws_secret_access_key)
+
+      user_data = self.config.to_user_data
+
+      instance = ec2.launch_instances(config['ami'],
+        :group_names => config['security_group'],
+        :instance_type => config['instance_type'],
+        :key_name => config['key_pair'],
+        :user_data => user_data
+      ).first
+
+      instance_id = instance[:aws_instance_id]
+
+      puts "Launched instance: #{instance_id.to_s.on_light_blue}"
+
+      ec2.create_tags(instance_id, self.config["tags"]) if self.config["tags"]
+
+      if (self.config["elastic_ip"])
+        #we can't associate IPs while the state is pending
+        while instance[:aws_state] == 'pending'
+          print '.'
+          STDOUT.flush
+          #this is going to hammer EC2 a bit, it might be necessary to add some delay in here
+          instance = ec2.describe_instances([instance_id]).first
+        end
+
+        ec2.associate_address(instance_id, :public_ip => self.config["elastic_ip"])
+
+        instance = ec2.describe_instances([instance_id]).first # So we get the correct IP address
+      end
+
+      archiver = S3Archiver.new(
+        :instance_id => instance_id,
+        :aws_access_key_id => self.aws_access_key_id,
+        :aws_secret_access_key => self.aws_secret_access_key,
+        :bucket_name => self.config["s3_archive_bucket"]
+      )
+      archiver.store_archive(self.config, instance)
+
+      instance
+    end
+  end
+end

data/lib/cloud_maker/s3_archiver.rb

@@ -0,0 +1,72 @@
+require 'right_aws'
+
+module CloudMaker
+  class S3Archiver
+
+    # Public: Gets/Sets the AWS access key.
+    attr_accessor :aws_secret_access_key
+    # Public: Gets/Sets the AWS secret.
+    attr_accessor :aws_access_key_id
+    # Public: Gets/Sets the EC2 instance ID string.
+    attr_accessor :instance_id
+    # Internal: Gets/Sets the RightAws::S3::Bucket used for storing/loading archives.
+    attr_accessor :bucket
+
+    # Public: All archive keys will be prefixed with KEY_PREFIX/
+    KEY_PREFIX = "cloud-maker"
+
+    # Public: Creates a new S3 Archiver instance
+    #
+    # options - S3 configuration options
+    #           :aws_access_key_id     - (required) The AWS access key
+    #           :aws_secret_access_key - (required) The AWS secret
+    #           :bucket_name           - (required) The bucket for the archiver to access
+    #           :instance_id           - (required) The AWS instance ID the archive describes
+    #
+    # Returns a new CloudMaker::S3Archiver instance
+    # Raises RuntimeError if any of the required options are not specified
+    def initialize(options)
+      required_keys = [:aws_access_key_id, :aws_secret_access_key, :instance_id, :bucket_name]
+      unless (required_keys - options.keys).empty?
+        raise RuntimeError.new("Instantiated #{self.class} without required attributes: #{required_keys - options.keys}.")
+      end
+
+      self.instance_id = options[:instance_id]
+      self.aws_access_key_id = options[:aws_access_key_id]
+      self.aws_secret_access_key = options[:aws_secret_access_key]
+
+      s3 = RightAws::S3.new(self.aws_access_key_id, self.aws_secret_access_key)
+      self.bucket = s3.bucket(options[:bucket_name])
+    end
+
+    # Public: Generates an archive with all information relevant to an instance
+    # launch and stores it to S3.
+    #
+    # cloud_maker_config - The CloudMaker::Config the instance was launched with
+    # instance           - A Hash describing the properties of the launched instance
+    #
+    # Returns nothing.
+    def store_archive(cloud_maker_config, instance)
+      userdata = cloud_maker_config.to_user_data
+      self.bucket.put(self.key + "/user_data.cloud_config", userdata)
+      self.bucket.put(self.key + "/instance.yaml", instance.to_yaml)
+      true
+    end
+
+    # Public: Retrieves a previously created archive from S3
+    #
+    # Returns the content of the archive.
+    def load_archive
+      self.bucket.get(self.key)
+    end
+
+    # Public: Returns the key that the archive will be stored under
+    def key
+      if self.instance_id
+        [KEY_PREFIX, self.instance_id].join('/')
+      else
+        raise RuntimeError.new("Attempted to generate a key name without an instance id.")
+      end
+    end
+  end
+end

data/lib/cloud_maker/shell_executor.rb

@@ -0,0 +1,38 @@
+module CloudMaker
+    class ShellExecutor
+    YAML_DOMAIN = "!airbnb.com,2012-07-19"
+    YAML_TYPE = "shell-script"
+
+    attr_accessor :script
+
+    def initialize(script)
+      self.script = script
+    end
+
+    def to_yaml_type
+      "#{YAML_DOMAIN}/#{YAML_TYPE}"
+    end
+
+    def to_yaml(opts = {})
+      YAML.quick_emit( nil, opts ) { |out|
+        out.scalar( "tag:yaml.org,2002:str", execute, :plain )
+      }
+    end
+
+    def execute
+      @result ||= `#{self.script}`.chomp
+    end
+
+    def to_s
+      execute
+    end
+
+    def self.from_string_representation(string_representation)
+      ShellExecutor.new(string_representation)
+    end
+  end
+end
+
+YAML::add_domain_type(CloudMaker::ShellExecutor::YAML_DOMAIN, CloudMaker::ShellExecutor::YAML_TYPE) do |type, val|
+  CloudMaker::ShellExecutor.from_string_representation(val)
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: cloud-maker
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Nathan Baxter
+- Flo Leibert
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: right_aws
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email: nathan.baxter@airbnb.cloud-maker
+executables:
+- cloud-maker
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cloud-maker.rb
+- lib/cloud_maker/config.rb
+- lib/cloud_maker/ec2.rb
+- lib/cloud_maker/s3_archiver.rb
+- lib/cloud_maker/shell_executor.rb
+- bin/cloud-maker
+homepage: https://github.com/airbnb/cloud-maker
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Launch and perform initial configuration of cloud servers.
+test_files: []