inception-server 0.2.1

data/lib/inception/inception_server.rb

@@ -0,0 +1,304 @@
+require "fog"
+
+module Inception
+  class InceptionServer
+
+    DEFAULT_SERVER_NAME = "inception"
+    DEFAULT_FLAVOR = "m1.small"
+    DEFAULT_DISK_SIZE = 16
+    DEFAULT_SECURITY_GROUPS = ["ssh"]
+
+    attr_reader :attributes
+
+    # @provider_client [Inception::Providers::FogProvider] - interact with IaaS
+    # @attributes [ReadWriteSettings]
+    #
+    # Required @attributes:
+    #   {
+    #     "name" => "inception",
+    #     "ip_address" => "54.214.15.178",
+    #     "key_pair" => {
+    #       "name" => "inception",
+    #       "private_key" => "private_key",
+    #       "public_key" => "public_key"
+    #     }
+    #   }
+    #
+    # Including optional @attributes and default values:
+    #   {
+    #     "name" => "inception",
+    #     "ip_address" => "54.214.15.178",
+    #     "security_groups" => ["ssh"],
+    #     "flavor" => "m1.small",
+    #     "key_pair" => {
+    #       "name" => "inception",
+    #       "private_key" => "private_key",
+    #       "public_key" => "public_key"
+    #     }
+    #   }
+    def initialize(provider_client, attributes, ssh_dir)
+      @provider_client = provider_client
+      @ssh_dir = ssh_dir
+      @attributes = attributes.is_a?(Hash) ? ReadWriteSettings.new(attributes) : attributes
+      raise "@attributes must be ReadWriteSettings (or Hash)" unless @attributes.is_a?(ReadWriteSettings)
+    end
+
+    # Create the underlying server, with key pair & security groups, unless it is already created
+    #
+    # The @attributes hash is updated with a `provisioned` key during/after creation.
+    # When saved as YAML it might look like:
+    #   inception:
+    #     provisioned:
+    #       image_id: ami-123456
+    #       server_id: i-e7f005d2
+    #       security_groups:
+    #         - ssh
+    #         - mosh
+    #       username: ubuntu
+    #       disk_device: /dev/sdi
+    #       host: ec2-54-214-15-178.us-west-2.compute.amazonaws.com
+    #       validated: true
+    #       converged: true
+    def create
+      validate_attributes_for_bootstrap
+      ensure_required_security_groups
+      create_missing_default_security_groups
+      bootstrap_vm
+      attach_persistent_disk
+    end
+
+    # Delete the server, volume and release the IP address
+    def delete_all
+      delete_server
+      delete_volume
+      delete_key_pair
+      release_ip_address
+    end
+
+    def delete_server
+      @fog_server = nil # force reload of fog_server model
+      if fog_server
+        print "Deleting server... "
+        fog_server.destroy
+        wait_for_termination(fog_server) unless Fog.mocking?
+        puts "done."
+      else
+        puts "Server already destroyed"
+      end
+      provisioned.delete("host")
+      provisioned.delete("server_id")
+      provisioned.delete("username")
+    end
+    
+    def delete_volume
+      volume_id = provisioned.exists?("disk_device.volume_id")
+      if volume_id && (volume = fog_compute.volumes.get(volume_id)) && volume.ready?
+        print "Deleting volume... "
+        volume.destroy
+        wait_for_termination(volume, "deleting")
+        puts ""
+      else
+        puts "Volume already destroyed"
+      end
+      provisioned.delete("disk_device")
+    end
+    
+    def delete_key_pair
+      key_pair_name = attributes.exists?("key_pair.name")
+      if key_pair_name && key_pair = fog_compute.key_pairs.get(key_pair_name)
+        puts "Deleting key pair '#{key_pair_name}'"
+        key_pair.destroy
+      else
+        puts "Keypair already destroyed"
+      end
+      attributes.delete("key_pair")
+    end
+    
+
+    def release_ip_address
+      public_ip = provisioned.exists?("ip_address")
+      if public_ip && ip_address = fog_compute.addresses.get(public_ip)
+        puts "Releasing IP address #{public_ip}"
+        ip_address.destroy
+      else
+        puts "IP address already released"
+      end
+      provisioned.delete("ip_address")
+    end
+
+    def security_groups
+      @attributes.security_groups
+    end
+
+    def server_name
+      @attributes["name"] ||= DEFAULT_SERVER_NAME
+      @attributes.name
+    end
+
+    def key_name
+      @attributes.key_pair.name
+    end
+
+    def private_key_path
+      @private_key_path ||= File.join(@ssh_dir, key_name)
+    end
+
+    def public_key
+      @attributes.exists?("key_pair.public_key")
+    end
+
+    # Flavor/instance type of the server to be provisioned
+    # TODO: DEFAULT_FLAVOR should become IaaS/provider specific
+    def flavor
+      @attributes["flavor"] ||= DEFAULT_FLAVOR
+    end
+
+    # Size of attached persistent disk for the inception server
+    def disk_size
+      @attributes["disk_size"] ||= DEFAULT_DISK_SIZE
+    end
+
+    def ip_address
+      provisioned.ip_address
+    end
+
+    def image_id
+      @attributes["image_id"] ||= @provider_client.image_id
+    end
+
+    # The progresive/final attributes of the provisioned Inception server &
+    # persistent disk.
+    def provisioned
+      @attributes["provisioned"] = {} unless @attributes["provisioned"]
+      @attributes.provisioned
+    end
+
+    # Because @attributes["provisioned"] is not the same as @attributes.provisioned
+    # we need a helper to export the complete nested attributes.
+    def export_attributes
+      attrs = attributes.to_nested_hash
+      attrs["provisioned"] = provisioned.to_nested_hash
+      attrs
+    end
+
+    def disk_devices
+      provisioned["disk_device"] ||= default_disk_device
+    end
+
+    def external_disk_device
+      disk_devices["external"]
+    end
+
+    def default_disk_device
+      case @provider_client
+      when Inception::Providers::Clients::AwsProviderClient
+        { "external" => "/dev/sdf", "internal" => "/dev/xvdf" }
+      when Inception::Providers::Clients::OpenStackProviderClient
+        { "external" => "/dev/vdc", "internal" => "/dev/vdc" }
+      else
+        raise "Please implement InceptionServer#default_disk_device for #{@provider_client.class}"
+      end
+    end
+
+    def user_host
+      "#{provisioned.username}@#{provisioned.host}"
+    end
+
+    def fog_server
+      @fog_server ||= begin
+        if server_id = provisioned["server_id"]
+          fog_compute.servers.get(server_id)
+        end
+      end
+    end
+
+    def fog_compute
+      @provider_client.fog_compute
+    end
+
+    protected
+    # set_resource_name(fog_server, "inception")
+    # set_resource_name(volume, "inception-root")
+    # set_resource_name(volume, "inception-store")
+    def set_resource_name(resource, name)
+      @provider_client.set_resource_name(resource, name)
+    end
+
+    def fog_attributes
+      @provider_client.fog_attributes(self)
+    end
+
+    def validate_attributes_for_bootstrap
+      missing_attributes = []
+      missing_attributes << "provisioned.ip_address" unless @attributes.exists?("provisioned.ip_address")
+      missing_attributes << "key_pair.private_key" unless @attributes.exists?("key_pair.private_key")
+      if missing_attributes.size > 0
+        raise "Missing InceptionServer attributes: #{missing_attributes.join(', ')}"
+      end
+    end
+
+    # ssh group must be first (bootstrap method looks for port 22 in first group)
+    def ensure_required_security_groups
+      if @attributes["security_groups"] && @attributes["security_groups"].is_a?(Array)
+        unless @attributes["security_groups"].include?("ssh")
+          @attributes["security_groups"] = ["ssh", *@attributes["security_groups"]]
+        end
+      else
+        @attributes["security_groups"] = ["ssh"]
+      end
+    end
+
+    def create_missing_default_security_groups
+      # provider method only creates group if missing
+      @provider_client.create_security_group("ssh", "ssh", {ssh: 22})
+    end
+
+    def bootstrap_vm
+      unless fog_server
+        print "Booting #{flavor} inception server... "
+        @fog_server = @provider_client.bootstrap(fog_attributes)
+        provisioned["server_id"] = fog_server.id
+        provisioned["host"] = fog_server.dns_name || fog_server.public_ip_address
+        provisioned["username"] = fog_attributes[:username]
+        puts provisioned.server_id
+      end
+      set_resource_name(fog_server, server_name)
+    end
+
+    def attach_persistent_disk
+      unless Fog.mocking?
+        print "Confirming ssh access to server... "
+        Fog.wait_for(60) { fog_server.sshable?(ssh_options) }
+        puts "done"
+      end
+
+      unless volume = @provider_client.find_server_device(fog_server, external_disk_device)
+        print "Provisioning #{disk_size}Gb persistent disk for inception server... "
+        volume = @provider_client.create_and_attach_volume("Inception Disk", disk_size, fog_server, external_disk_device)
+        disk_devices["volume_id"] = volume.id
+        puts disk_devices.volume_id
+      end
+      set_resource_name(volume, server_name)
+    end
+
+    def ssh_options
+      {
+        keys: [private_key_path]
+      }
+    end
+
+    # Poll a fog model until it terminates; print . each second
+    def wait_for_termination(fog_model, state_to_wait_for="terminated")
+      fog_model.wait_for do
+        print "."
+        state == state_to_wait_for
+      end
+    end
+
+    protected
+    # TODO emit events rather than writing directly to STDOUT
+    def say(*args)
+      puts(*args)
+    end
+  end
+end

data/lib/inception/inception_server_cookbook.rb

@@ -0,0 +1,90 @@
+module Inception
+  # Perform converge chef cookbooks upon inception server
+  class InceptionServerCookbook
+    include FileUtils
+
+    attr_reader :server, :settings, :project_dir
+
+    class InvalidTarget < StandardError; end
+
+    def initialize(inception_server, settings, project_dir)
+      @server = inception_server
+      @settings = settings
+      @project_dir = project_dir
+    end
+
+    def prepare
+      FileUtils.chdir(project_dir) do
+        prepare_project_dir
+        knife_solo :prepare unless ignore_chef_preparations?
+      end
+    end
+
+    # To be invoked within the settings_dir
+    def converge
+      FileUtils.chdir(project_dir) do
+        knife_solo :cook
+      end
+    end
+
+    def ignore_chef_preparations?
+      @settings.exists?("cookbook.prepared")
+    end
+
+    def user_host; server.user_host; end
+    def key_path; server.private_key_path; end
+
+    def knife_solo(command)
+      attributes = cookbook_attributes_for_inception.to_json
+      sh %Q{knife solo #{command} #{user_host} -i #{key_path} -j '#{attributes}' -r 'bosh_inception'}
+    end
+
+    protected
+    def prepare_project_dir
+      prepare_cookbook
+      prepare_knife_config
+      prepare_berksfile
+    end
+
+    def prepare_cookbook
+      mkdir_p("cookbooks")
+      rm_rf("cookbooks/bosh_inception")
+      cp_r(inception_cookbook_path, "cookbooks/")
+    end
+
+    def prepare_knife_config
+      mkdir_p("nodes") # needed for knife solo
+    end
+
+    def prepare_berksfile
+      unless File.exists?("Berksfile")
+        cp_r(File.join(gem_root_path, "Berksfile"), "Berksfile")
+      end
+    end
+
+    def cookbook_attributes_for_inception
+      {
+        "disk" => {
+          "mounted" => true,
+          "device" => settings.inception.provisioned.disk_device.internal
+        },
+        "git" => {
+          "name" => settings.git.name,
+          "email" => settings.git.email
+        },
+        "user" => {
+          "username" => settings.inception.provisioned.username
+        },
+        "fog" => settings.provider.credentials
+      }
+    end
+
+    def gem_root_path
+      File.expand_path("../../..", __FILE__)
+    end
+
+    def inception_cookbook_path
+      File.join(gem_root_path, "cookbooks/bosh_inception")
+    end
+  end
+end

data/lib/inception/next_deploy_actions.rb

@@ -0,0 +1,20 @@
+module Inception
+  
+  class NextDeployActions
+    def initialize(attributes, cli_options)
+      @attributes = attributes.is_a?(Hash) ? ReadWriteSettings.new(attributes) : attributes
+      raise "@attributes must be ReadWriteSettings (or Hash)" unless @attributes.is_a?(ReadWriteSettings)
+      raise "@cli_options must be Hash" unless cli_options.is_a?(Hash)
+      apply_cli_options(cli_options)
+    end
+
+    def skip_chef_converge?
+      @attributes["no_converge"] || @attributes["no-converge"] || @attributes["skip_chef_converge"]
+    end
+
+    protected
+    def apply_cli_options(cli_options)
+      @attributes.merge(cli_options)
+    end
+  end
+end

data/lib/inception/providers/README.md

@@ -0,0 +1,5 @@
+# Providers
+
+Currently these files are kept in sync with bosh-cloudfoundry project.
+
+TODO - extract into shared library.

data/lib/inception/providers/clients/aws_provider_client.rb

@@ -0,0 +1,144 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Inception; module Providers; module Clients; end; end; end
+
+require "inception/providers/clients/fog_provider_client"
+require "inception/providers/constants/aws_constants"
+
+class Inception::Providers::Clients::AwsProviderClient < Inception::Providers::Clients::FogProviderClient
+  include Inception::Providers::Constants::AwsConstants
+
+  # @return [Integer] megabytes of RAM for requested flavor of server
+  def ram_for_server_flavor(server_flavor_id)
+    if flavor = fog_compute_flavor(server_flavor_id)
+      flavor[:ram]
+    else
+      raise "Unknown AWS flavor '#{server_flavor_id}'"
+    end
+  end
+
+  # @return [Hash] e.g. { :bits => 0, :cores => 2, :disk => 0,
+  #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
+  # or nil if +server_flavor_id+ is not a supported flavor ID
+  def fog_compute_flavor(server_flavor_id)
+    aws_compute_flavors.find { |fl| fl[:id] == server_flavor_id }
+  end
+
+  # @return [Array] of [Hash] for each supported compute flavor
+  # Example [Hash] { :bits => 0, :cores => 2, :disk => 0,
+  #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
+  def aws_compute_flavors
+    Fog::Compute::AWS::FLAVORS
+  end
+
+  def aws_compute_flavor_ids
+    aws_compute_flavors.map { |fl| fl[:id] }
+  end
+
+  # Provision an EC2 or VPC elastic IP addess.
+  # * VPC - provision_public_ip_address(vpc: true)
+  # * EC2 - provision_public_ip_address
+  # @return [String] provisions a new public IP address in target region
+  # TODO nil if none available
+  def provision_public_ip_address(options={})
+    if options.delete(:vpc)
+      options[:domain] = "vpc"
+    else
+      options[:domain] = options.delete(:domain) || "standard"
+    end
+    address = fog_compute.addresses.create(options)
+    address.public_ip
+    # TODO catch error and return nil
+  end
+
+  def associate_ip_address_with_server(ip_address, server)
+    address = fog_compute.addresses.get(ip_address)
+    address.server = server
+  end
+
+  def create_vpc(name, cidr_block)
+    vpc = fog_compute.vpcs.create(name: name, cidr_block: cidr_block)
+    vpc.id
+  end
+
+  # Creates a VPC subnet
+  # @return [String] the subnet_id
+  def create_subnet(vpc_id, cidr_block)
+    subnet = fog_compute.subnets.create(vpc_id: vpc_id, cidr_block: cidr_block)
+    subnet.subnet_id
+  end
+
+  def create_internet_gateway(vpc_id)
+    gateway = fog_compute.internet_gateways.create(vpc_id: vpc_id)
+    gateway.id
+  end
+
+  def find_server_device(server, device)
+    server.volumes.all.find {|v| v.device == device}
+  end
+
+  def create_and_attach_volume(name, disk_size, server, device)
+    volume = fog_compute.volumes.create(
+        size: disk_size,
+        name: name,
+        description: '',
+        device: device,
+        availability_zone: server.availability_zone)
+    # TODO: the following works in fog 1.9.0+ (but which has a bug in bootstrap)
+    # https://github.com/fog/fog/issues/1516
+    #
+    # volume.wait_for { volume.status == 'available' }
+    # volume.attach(server.id, "/dev/vdc")
+    # volume.wait_for { volume.status == 'in-use' }
+    #
+    # Instead, using:
+    volume.server = server
+  end
+
+  # Ubuntu 13.04
+  def image_id
+    region = fog_compute.region
+    # http://cloud-images.ubuntu.com/locator/ec2/
+    image_id = case region.to_s
+    when "ap-northeast-1"
+      "ami-6b26ab6a"
+    when "ap-southeast-1"
+      "ami-2b511e79"
+    when "eu-west-1"
+      "ami-3d160149"
+    when "sa-east-1"
+      "ami-28e43e35"
+    when "us-east-1"
+      "ami-c30360aa"
+    when "us-west-1"
+      "ami-d383af96"
+    when "ap-southeast-2"
+      "ami-84a333be"
+    when "us-west-2"
+      "ami-bf1d8a8f"
+    end
+    image_id || raise("Please add Ubuntu 13.04 64bit (EBS) AMI image id to aws.rb#raring_image_id method for region '#{region}'")
+  end
+
+  # Construct a Fog::Compute object
+  # Uses +attributes+ which normally originates from +settings.provider+
+  def setup_fog_connection
+    configuration = Fog.symbolize_credentials(attributes.credentials)
+    configuration[:provider] = "AWS"
+    configuration[:region] = attributes.region
+    @fog_compute = Fog::Compute.new(configuration)
+  end
+
+  def fog_attributes(inception_server)
+    {
+      image_id: inception_server.image_id,
+      groups: inception_server.security_groups,
+      key_name: inception_server.key_name,
+      private_key_path: inception_server.private_key_path,
+      flavor_id: inception_server.flavor,
+      public_ip_address: inception_server.ip_address,
+      bits: 64,
+      username: "ubuntu",
+    }
+  end
+end