inception 0.1.0

data/lib/bosh/providers/clients/aws_provider_client.rb

@@ -0,0 +1,168 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Providers; module Clients; end; end; end
+
+require "bosh/providers/clients/fog_provider_client"
+require "bosh/providers/constants/aws_constants"
+
+class Bosh::Providers::Clients::AwsProviderClient < Bosh::Providers::Clients::FogProviderClient
+  include Bosh::Providers::Constants::AwsConstants
+
+  # @return [Integer] megabytes of RAM for requested flavor of server
+  def ram_for_server_flavor(server_flavor_id)
+    if flavor = fog_compute_flavor(server_flavor_id)
+      flavor[:ram]
+    else
+      raise "Unknown AWS flavor '#{server_flavor_id}'"
+    end
+  end
+
+  # @return [Hash] e.g. { :bits => 0, :cores => 2, :disk => 0,
+  #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
+  # or nil if +server_flavor_id+ is not a supported flavor ID
+  def fog_compute_flavor(server_flavor_id)
+    aws_compute_flavors.find { |fl| fl[:id] == server_flavor_id }
+  end
+
+  # @return [Array] of [Hash] for each supported compute flavor
+  # Example [Hash] { :bits => 0, :cores => 2, :disk => 0,
+  #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
+  def aws_compute_flavors
+    Fog::Compute::AWS::FLAVORS
+  end
+
+  def aws_compute_flavor_ids
+    aws_compute_flavors.map { |fl| fl[:id] }
+  end
+
+  # Provision an EC2 or VPC elastic IP addess.
+  # * VPC - provision_public_ip_address(vpc: true)
+  # * EC2 - provision_public_ip_address
+  # @return [String] provisions a new public IP address in target region
+  # TODO nil if none available
+  def provision_public_ip_address(options={})
+    if options.delete(:vpc)
+      options[:domain] = "vpc"
+    else
+      options[:domain] = options.delete(:domain) || "standard"
+    end
+    address = fog_compute.addresses.create(options)
+    address.public_ip
+    # TODO catch error and return nil
+  end
+
+  def associate_ip_address_with_server(ip_address, server)
+    address = fog_compute.addresses.get(ip_address)
+    address.server = server
+  end
+
+  def create_vpc(name, cidr_block)
+    vpc = fog_compute.vpcs.create(name: name, cidr_block: cidr_block)
+    vpc.id
+  end
+
+  # Creates a VPC subnet
+  # @return [String] the subnet_id
+  def create_subnet(vpc_id, cidr_block)
+    subnet = fog_compute.subnets.create(vpc_id: vpc_id, cidr_block: cidr_block)
+    subnet.subnet_id
+  end
+
+  def create_internet_gateway(vpc_id)
+    gateway = fog_compute.internet_gateways.create(vpc_id: vpc_id)
+    gateway.id
+  end
+
+  def find_server_device(server, device)
+    server.volumes.all.find {|v| v.device == device}
+  end
+
+  def create_and_attach_volume(name, disk_size, server, device)
+    volume = fog_compute.volumes.create(
+        size: disk_size,
+        name: name,
+        description: '',
+        device: device,
+        availability_zone: server.availability_zone)
+    # TODO: the following works in fog 1.9.0+ (but which has a bug in bootstrap)
+    # https://github.com/fog/fog/issues/1516
+    #
+    # volume.wait_for { volume.status == 'available' }
+    # volume.attach(server.id, "/dev/vdc")
+    # volume.wait_for { volume.status == 'in-use' }
+    #
+    # Instead, using:
+    volume.server = server
+  end
+
+  # Ubuntu 13.04
+  def raring_image_id(region=nil)
+    region = fog_compute.region
+    # http://cloud-images.ubuntu.com/locator/ec2/
+    image_id = case region.to_s
+    when "ap-northeast-1"
+      "ami-6b26ab6a"
+    when "ap-southeast-1"
+      "ami-2b511e79"
+    when "eu-west-1"
+      "ami-3d160149"
+    when "sa-east-1"
+      "ami-28e43e35"
+    when "us-east-1"
+      "ami-c30360aa"
+    when "us-west-1"
+      "ami-d383af96"
+    when "ap-southeast-2"
+      "ami-84a333be"
+    when "us-west-2"
+      "ami-bf1d8a8f"
+    end
+    image_id || raise("Please add Ubuntu 13.04 64bit (EBS) AMI image id to aws.rb#raring_image_id method for region '#{region}'")
+  end
+
+  def bootstrap(new_attributes = {})
+    new_attributes[:image_id] ||= raring_image_id(fog_compute.region)
+    vpc = new_attributes[:subnet_id]
+
+    server = fog_compute.servers.new(new_attributes)
+
+    unless new_attributes[:key_name]
+      raise "please provide :key_name attribute"
+    end
+    unless private_key_path = new_attributes.delete(:private_key_path)
+      raise "please provide :private_key_path attribute"
+    end
+
+    if vpc
+      # TODO setup security group on new server
+    else
+      # make sure port 22 is open in the first security group
+      security_group = fog_compute.security_groups.get(server.groups.first)
+      authorized = security_group.ip_permissions.detect do |ip_permission|
+        ip_permission['ipRanges'].first && ip_permission['ipRanges'].first['cidrIp'] == '0.0.0.0/0' &&
+        ip_permission['fromPort'] == 22 &&
+        ip_permission['ipProtocol'] == 'tcp' &&
+        ip_permission['toPort'] == 22
+      end
+      unless authorized
+        security_group.authorize_port_range(22..22)
+      end
+    end
+
+    server.save
+    unless Fog.mocking?
+      server.wait_for { ready? }
+      server.setup(:keys => [private_key_path])
+    end
+    server
+  end
+
+  # Construct a Fog::Compute object
+  # Uses +attributes+ which normally originates from +settings.provider+
+  def setup_fog_connection
+    configuration = Fog.symbolize_credentials(attributes.credentials)
+    configuration[:provider] = "AWS"
+    configuration[:region] = attributes.region
+    @fog_compute = Fog::Compute.new(configuration)
+  end
+end

data/lib/bosh/providers/clients/fog_provider_client.rb

@@ -0,0 +1,161 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+require "fog"
+module Bosh; module Providers; module Clients; end; end; end
+
+class Bosh::Providers::Clients::FogProviderClient
+  attr_reader :fog_compute
+  attr_reader :attributes
+
+  def initialize(attributes)
+    @attributes = attributes.is_a?(Hash) ? Settingslogic.new(attributes) : attributes
+    raise "@attributes must be Settingslogic (or Hash)" unless @attributes.is_a?(Settingslogic)
+    setup_fog_connection
+  end
+
+  def setup_fog_connection
+    raise "must implement"
+  end
+
+  def create_key_pair(key_pair_name)
+    fog_compute.key_pairs.create(:name => key_pair_name)
+  end
+
+  # set_resource_name(fog_server, "inception")
+  # set_resource_name(volume, "inception-root")
+  # set_resource_name(volume, "inception-store")
+  def set_resource_name(resource, name)
+    fog_compute.tags.create :key => "Name", :value => name, :resource_id => resource.id
+  end
+
+  def delete_key_pair_if_exists(key_pair_name)
+    if fog_key_pair = fog_compute.key_pairs.get(key_pair_name)
+      fog_key_pair.destroy
+    end
+  end
+
+  def delete_servers_with_name(name)
+    fog_compute.servers.select {|s| s.tags["Name"].downcase == name.downcase }.each do |server|
+      puts "Destroying server #{server.id}..."
+      server.destroy
+    end
+  end
+
+  def delete_volumes_with_name(name)
+    fog_compute.volumes.select do |v|
+      volume_name = v.tags["Name"]
+      volume_name && volume_name.downcase == name.downcase
+    end.each do |volume|
+      puts "Destroying volume #{volume.id}..."
+      volume.destroy
+    end
+  end
+
+  # Destroy all IP addresses that aren't bound to a server
+  def cleanup_unused_ip_addresses
+    fog_compute.addresses.each do |a|
+      unless a.server
+        puts "Deleting unused IP address #{a.public_ip}..."
+        a.destroy
+      end
+    end
+  end
+
+  # Creates or reuses an security group and opens ports.
+  #
+  # +security_group_name+ is the name to be created or reused
+  # +ports+ is a hash of name/port for ports to open, for example:
+  # {
+  #   ssh: 22,
+  #   http: 80,
+  #   https: 443
+  # }
+  # protocol defaults to TCP
+  # You can also use a more verbose +ports+ using the format:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_ranges: [ { cidrIp: "196.212.12.34/32" } ] }
+  # }
+  # In this example, 
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def create_security_group(security_group_name, description, ports)
+    security_groups = fog_compute.security_groups
+    unless sg = security_groups.find { |s| s.name == security_group_name }
+      sg = fog_compute.security_groups.create(name: security_group_name, description: description)
+      puts "Created security group #{security_group_name}"
+    else
+      puts "Reusing security group #{security_group_name}"
+    end
+    ip_permissions = ip_permissions(sg)
+    ports_opened = 0
+    ports.each do |name, port_defn|
+      (protocol, port_range, ip_range) = extract_port_definition(port_defn)
+      unless port_open?(ip_permissions, port_range, protocol, ip_range)
+        authorize_port_range(sg, port_range, protocol, ip_range)
+        puts " -> opened #{name} ports #{protocol.upcase} #{port_range.min}..#{port_range.max} from IP range #{ip_range}"
+        ports_opened += 1
+      end
+    end
+    puts " -> no additional ports opened" if ports_opened == 0
+    true
+  end
+
+  def port_open?(ip_permissions, port_range, protocol, ip_range)
+    ip_permissions && ip_permissions.find do |ip| 
+      ip["ipProtocol"] == protocol \
+      && ip["ipRanges"].detect { |range| range["cidrIp"] == ip_range } \
+      && ip["fromPort"] <= port_range.min \
+      && ip["toPort"] >= port_range.max
+    end
+  end
+
+  def authorize_port_range(sg, port_range, protocol, ip_range)
+    sg.authorize_port_range(port_range, {:ip_protocol => protocol, :cidr_ip => ip_range})
+  end
+
+  def ip_permissions(sg)
+    sg.ip_permissions
+  end
+
+  # Any of the following +port_defn+ can be used:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_range: "196.212.12.34/32" }
+  # }
+  # In this example,
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def extract_port_definition(port_defn)
+    protocol = "tcp"
+    ip_range = "0.0.0.0/0"
+    if port_defn.is_a? Integer
+      port_range = (port_defn..port_defn)
+    elsif port_defn.is_a? Range
+      port_range = port_defn
+    elsif port_defn.is_a? Hash
+      protocol = port_defn[:protocol] if port_defn[:protocol]
+      port_range = port_defn[:ports]  if port_defn[:ports]
+      ip_range = port_defn[:ip_range] if port_defn[:ip_range]
+    end
+    [protocol, port_range, ip_range]
+  end
+
+  def provision_or_reuse_public_ip_address(options={})
+    provision_public_ip_address(options) || find_unused_public_ip_address(options)
+  end
+
+  def find_unused_public_ip_address(options={})
+    if address = fog_compute.addresses.find { |s| s.server_id.nil? }
+      address.public_ip
+    end
+  end
+end

data/lib/bosh/providers/clients/openstack_provider_client.rb

@@ -0,0 +1,65 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Providers; module Clients; end; end; end
+
+require "bosh/providers/clients/fog_provider_client"
+require "bosh/providers/constants/openstack_constants"
+
+class Bosh::Providers::Clients::OpenStackProviderClient < Bosh::Providers::Clients::FogProviderClient
+  # @return [String] provisions a new public IP address in target region
+  # TODO nil if none available
+  def provision_public_ip_address(options={})
+    address = fog_compute.addresses.create
+    address.ip
+    # TODO catch error and return nil
+  end
+
+  def associate_ip_address_with_server(ip_address, server)
+    address = fog_compute.addresses.find { |a| a.ip == ip_address }
+    address.server = server
+  end
+
+  # Hook method for FogProviderClient#create_security_group
+  def ip_permissions(sg)
+    sg.rules
+  end
+
+  # Hook method for FogProviderClient#create_security_group
+  def authorize_port_range(sg, port_range, protocol, ip_range)
+    sg.create_security_group_rule(port_range.min, port_range.max, protocol, ip_range)
+  end
+
+  def find_server_device(server, device)
+    va = fog_compute.get_server_volumes(server.id).body['volumeAttachments']
+    va.find { |v| v["device"] == device }
+  end
+
+  def create_and_attach_volume(name, disk_size, server, device)
+    volume = fog_compute.volumes.create(:name => name,
+                                        :description => "",
+                                        :size => disk_size,
+                                        :availability_zone => server.availability_zone)
+    volume.wait_for { volume.status == 'available' }
+    volume.attach(server.id, device)
+    volume.wait_for { volume.status == 'in-use' }
+  end
+
+  def delete_security_group_and_servers(sg_name)
+    raise "not implemented yet"
+  end
+
+  # Construct a Fog::Compute object
+  # Uses +attributes+ which normally originates from +settings.provider+
+  def setup_fog_connection
+    configuration = Fog.symbolize_credentials(attributes.credentials)
+    configuration[:provider] = "OpenStack"
+    unless attributes.region == openstack_constants.no_region_code
+      configuration[:openstack_region] = attributes.region
+    end
+    @fog_compute = Fog::Compute.new(configuration)
+  end
+
+  def openstack_constants
+    Bosh::Providers::Constants::OpenStackConstants
+  end
+end

data/lib/bosh/providers/constants/aws_constants.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Providers; module Constants; end; end; end
+
+module Bosh::Providers::Constants::AwsConstants
+  extend self
+
+  # http://docs.aws.amazon.com/general/latest/gr/rande.html#region
+  def region_labels
+    [
+      { label: "US East (Northern Virginia) Region", code: "us-east-1" },
+      { label: "US West (Oregon) Region", code: "us-west-2" },
+      { label: "US West (Northern California) Region", code: "us-west-1" },
+      { label: "EU (Ireland) Region", code: "eu-west-1" },
+      { label: "Asia Pacific (Singapore) Region", code: "ap-southeast-1" },
+      { label: "Asia Pacific (Sydney) Region", code: "ap-southeast-2" },
+      { label: "Asia Pacific (Tokyo) Region", code: "ap-northeast-1" },
+      { label: "South America (Sao Paulo) Region", code: "sa-east-1" },
+    ]
+  end
+
+  def default_region_code
+    "us-east-1"
+  end
+end

data/lib/bosh/providers/constants/openstack_constants.rb

@@ -0,0 +1,12 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Providers; module Constants; end; end; end
+
+module Bosh::Providers::Constants::OpenStackConstants
+  extend self
+
+  # explicit value representing "no region requested"
+  def no_region_code
+    "no-region-requested"
+  end
+end

data/lib/inception.rb

@@ -0,0 +1,9 @@
+require "inception/version"
+
+module Inception
+end
+
+require "bosh/providers"
+require "inception/inception_server"
+require "inception/inception_server_cookbook"
+require "inception/next_deploy_actions"

data/lib/inception/cli.rb

@@ -0,0 +1,136 @@
+require "thor"
+require "highline"
+require "fileutils"
+require "json"
+
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+require "escape"
+require "inception/cli_helpers/display"
+require "inception/cli_helpers/infrastructure"
+require "inception/cli_helpers/interactions"
+require "inception/cli_helpers/provider"
+require "inception/cli_helpers/settings"
+require "inception/cli_helpers/prepare_deploy_settings"
+
+module Inception
+  class Cli < Thor
+    include FileUtils
+    include Inception::CliHelpers::Display
+    include Inception::CliHelpers::Infrastructure
+    include Inception::CliHelpers::Interactions
+    include Inception::CliHelpers::Provider
+    include Inception::CliHelpers::Settings
+    include Inception::CliHelpers::PrepareDeploySettings
+
+    desc "deploy", "Create/upgrade a Bosh inception server"
+    def deploy
+      migrate_old_settings
+      configure_provider
+      prepare_deploy_settings
+      perform_deploy
+      converge_cookbooks
+    end
+
+    desc "delete", "Destroy target Bosh inception server, volumes & release the IP address"
+    method_option :"non-interactive", aliases: ["-n"], type: :boolean, desc: "Don't ask questions, just get crankin'"
+    def delete
+      migrate_old_settings
+      perform_delete(options[:"non-interactive"])
+    end
+
+    desc "ssh [COMMAND]", "Open an ssh session to the inception server [do nothing if local machine is the inception server]"
+    long_desc <<-DESC
+      If a command is supplied, it will be run, otherwise a session will be opened.
+    DESC
+    def ssh(cmd=nil)
+      migrate_old_settings
+      run_ssh_command_or_open_tunnel(cmd)
+    end
+
+    desc "tmux", "Open an ssh (with tmux) session to the inception server [do nothing if local machine is inception server]"
+    long_desc <<-DESC
+      Opens a connection using ssh and attaches to the most recent tmux session;
+      giving you persistance across disconnects.
+    DESC
+    def tmux
+      migrate_old_settings
+      run_ssh_command_or_open_tunnel(["-t", "tmux attach || tmux new-session"])
+    end
+
+    no_tasks do
+      # update settings.git.name/git.email from local ~/.gitconfig if available
+      # provision public IP address for inception server if not allocated one
+      # Note: helper methods are in inception/cli_helpers/prepare_deploy_settings.rb
+      def prepare_deploy_settings
+        header "Preparing deployment settings"
+        update_git_config
+        provision_or_reuse_public_ip_address_for_inception unless settings.exists?("inception.provisioned.ip_address")
+        recreate_key_pair_for_inception unless settings.exists?("inception.key_pair.private_key")
+        recreate_private_key_file_for_inception
+        validate_deploy_settings
+        setup_next_deploy_actions
+      end
+
+      def perform_deploy
+        header "Provision inception server"
+        server = InceptionServer.new(provider_client, settings.inception, settings_ssh_dir)
+        server.create
+      ensure
+        # after any error handling, still save the current InceptionServer state back into settings.inception
+        settings["inception"] = server.export_attributes
+        save_settings!
+      end
+
+      def setup_next_deploy_actions
+        settings["next_deploy_actions"] ||= {}
+        @next_deploy_actions = NextDeployActions.new(settings.next_deploy_actions, options)
+      end
+
+      # Perform converge chef cookbooks upon inception server
+      # Does not update settings
+      def converge_cookbooks
+        if @next_deploy_actions.skip_chef_converge?
+          header "Prepare inception server", skip: "Requested to be skipped on this deploy."
+        else
+          header "Prepare inception server"
+          server = InceptionServer.new(provider_client, settings.inception, settings_ssh_dir)
+          cookbook = InceptionServerCookbook.new(server, settings, settings_dir)
+          cookbook.prepare
+          settings.set("cookbook.prepared", true)
+          save_settings!
+          cookbook.converge
+        end
+      end
+
+      def perform_delete(non_interactive)
+        server = InceptionServer.new(provider_client, settings.inception, settings_ssh_dir)
+        if non_interactive
+          header "Deleting inception server, volumes and releasing IP address"
+          server.delete_all
+        else
+          raise "Interactive delete not implemented yet"
+        end
+      ensure
+        # after any error handling, still save the current InceptionServer state back into settings.inception
+        settings["inception"] = server.export_attributes
+        save_settings!
+      end
+
+      def run_ssh_command_or_open_tunnel(cmd)
+        recreate_private_key_file_for_inception
+        unless settings.exists?("inception.provisioned.host")
+          exit "inception server has not finished launching; run to complete: inception deploy"
+        end
+
+        server = InceptionServer.new(provider_client, settings.inception, settings_ssh_dir)
+        username = settings.inception.provisioned.username
+        host = settings.inception.provisioned.host
+        result = system Escape.shell_command(["ssh", "-i", server.private_key_path, "#{username}@#{host}", cmd].flatten.compact)
+        exit result
+      end
+    end
+  end
+end