inception-server 0.2.1

data/lib/inception/providers/clients/fog_provider_client.rb

@@ -0,0 +1,185 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+require "fog"
+module Inception; module Providers; module Clients; end; end; end
+
+class Inception::Providers::Clients::FogProviderClient
+  attr_reader :fog_compute
+  attr_reader :attributes
+
+  def initialize(attributes)
+    @attributes = attributes.is_a?(Hash) ? ReadWriteSettings.new(attributes) : attributes
+    raise "@attributes must be ReadWriteSettings (or Hash)" unless @attributes.is_a?(ReadWriteSettings)
+    setup_fog_connection
+  end
+
+  def flavor_id(flavor_name)
+    if flavor = fog_compute.flavors.find { |fl| fl.name == flavor_name }
+      flavor.id
+    end
+  end
+
+  def setup_fog_connection
+    raise "must implement"
+  end
+
+  def create_key_pair(key_pair_name)
+    fog_compute.key_pairs.create(:name => key_pair_name)
+  end
+
+  # set_resource_name(fog_server, "inception")
+  # set_resource_name(volume, "inception-root")
+  # set_resource_name(volume, "inception-store")
+  def set_resource_name(resource, name)
+    fog_compute.tags.create :key => "Name", :value => name, :resource_id => resource.id
+  end
+
+  def delete_key_pair_if_exists(key_pair_name)
+    if fog_key_pair = fog_compute.key_pairs.get(key_pair_name)
+      fog_key_pair.destroy
+    end
+  end
+
+  def delete_servers_with_name(name)
+    fog_compute.servers.select {|s| s.tags["Name"] && (s.tags["Name"].downcase == name.downcase) }.each do |server|
+      puts "Destroying server #{server.id}... "
+      server.destroy
+    end
+  end
+
+  def delete_volumes_with_name(name)
+    fog_compute.volumes.select do |v|
+      volume_name = v.tags["Name"]
+      volume_name && volume_name.downcase == name.downcase
+    end.each do |volume|
+      puts "Destroying volume #{volume.id}... "
+      volume.destroy
+    end
+  end
+
+  # Destroy all IP addresses that aren't bound to a server
+  def cleanup_unused_ip_addresses
+    fog_compute.addresses.each do |a|
+      unless a.server
+        puts "Deleting unused IP address #{a.public_ip}... "
+        a.destroy
+      end
+    end
+  end
+
+  # Creates or reuses an security group and opens ports.
+  #
+  # +security_group_name+ is the name to be created or reused
+  # +ports+ is a hash of name/port for ports to open, for example:
+  # {
+  #   ssh: 22,
+  #   http: 80,
+  #   https: 443
+  # }
+  # protocol defaults to TCP
+  # You can also use a more verbose +ports+ using the format:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_ranges: [ { cidrIp: "196.212.12.34/32" } ] }
+  # }
+  # In this example, 
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def create_security_group(security_group_name, description, ports)
+    security_groups = fog_compute.security_groups
+    unless sg = security_groups.find { |s| s.name == security_group_name }
+      sg = fog_compute.security_groups.create(name: security_group_name, description: description)
+      puts "Created security group #{security_group_name}"
+      ip_permissions = ip_permissions(sg)
+      ports_opened = 0
+      ports.each do |name, port_defn|
+        (protocol, port_range, ip_range) = extract_port_definition(port_defn)
+        unless port_open?(ip_permissions, port_range, protocol, ip_range)
+          authorize_port_range(sg, port_range, protocol, ip_range)
+          puts " -> opened #{name} ports #{protocol.upcase} #{port_range.min}..#{port_range.max} from IP range #{ip_range}"
+          ports_opened += 1
+        end
+      end
+    else
+      puts "Reusing security group #{security_group_name}"
+    end
+    true
+  end
+
+  def port_open?(ip_permissions, port_range, protocol, ip_range)
+    ip_permissions && ip_permissions.find do |ip| 
+      ip["ipProtocol"] == protocol \
+      && ip["ipRanges"].detect { |range| range["cidrIp"] == ip_range } \
+      && ip["fromPort"] <= port_range.min \
+      && ip["toPort"] >= port_range.max
+    end
+  end
+
+  def authorize_port_range(sg, port_range, protocol, ip_range)
+    sg.authorize_port_range(port_range, {:ip_protocol => protocol, :cidr_ip => ip_range})
+  end
+
+  def ip_permissions(sg)
+    sg.ip_permissions
+  end
+
+  # Any of the following +port_defn+ can be used:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_range: "196.212.12.34/32" }
+  # }
+  # In this example,
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def extract_port_definition(port_defn)
+    protocol = "tcp"
+    ip_range = "0.0.0.0/0"
+    if port_defn.is_a? Integer
+      port_range = (port_defn..port_defn)
+    elsif port_defn.is_a? Range
+      port_range = port_defn
+    elsif port_defn.is_a? Hash
+      protocol = port_defn[:protocol] if port_defn[:protocol]
+      port_range = port_defn[:ports]  if port_defn[:ports]
+      ip_range = port_defn[:ip_range] if port_defn[:ip_range]
+    end
+    [protocol, port_range, ip_range]
+  end
+
+  def provision_or_reuse_public_ip_address(options={})
+    provision_public_ip_address(options) || find_unused_public_ip_address(options)
+  end
+
+  def find_unused_public_ip_address(options={})
+    if address = fog_compute.addresses.find { |s| s.server_id.nil? }
+      address.public_ip
+    end
+  end
+
+  def bootstrap(new_attributes)
+    server = fog_compute.servers.new(new_attributes)
+
+    unless new_attributes[:key_name]
+      raise "please provide :key_name attribute"
+    end
+    unless private_key_path = new_attributes.delete(:private_key_path)
+      raise "please provide :private_key_path attribute"
+    end
+
+    server.save
+    unless Fog.mocking?
+      server.wait_for { ready? }
+      server.setup(keys: [private_key_path])
+    end
+    server
+  end
+
+end

data/lib/inception/providers/clients/openstack_provider_client.rb

@@ -0,0 +1,84 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Inception; module Providers; module Clients; end; end; end
+
+require "inception/providers/clients/fog_provider_client"
+require "inception/providers/constants/openstack_constants"
+
+class Inception::Providers::Clients::OpenStackProviderClient < Inception::Providers::Clients::FogProviderClient
+  # @return [String] provisions a new public IP address in target region
+  # TODO nil if none available
+  def provision_public_ip_address(options={})
+    address = fog_compute.addresses.create
+    address.ip
+    # TODO catch error and return nil
+  end
+
+  def associate_ip_address_with_server(ip_address, server)
+    address = fog_compute.addresses.find { |a| a.ip == ip_address }
+    address.server = server
+  end
+
+  # Hook method for FogProviderClient#create_security_group
+  def ip_permissions(sg)
+    sg.rules
+  end
+
+  # Hook method for FogProviderClient#create_security_group
+  def authorize_port_range(sg, port_range, protocol, ip_range)
+    sg.create_security_group_rule(port_range.min, port_range.max, protocol, ip_range)
+  end
+
+  def find_server_device(server, device)
+    va = fog_compute.get_server_volumes(server.id).body['volumeAttachments']
+    va.find { |v| v["device"] == device }
+  end
+
+  def create_and_attach_volume(name, disk_size, server, device)
+    volume = fog_compute.volumes.create(:name => name,
+                                        :description => "",
+                                        :size => disk_size,
+                                        :availability_zone => server.availability_zone)
+    volume.wait_for { volume.status == 'available' }
+    volume.attach(server.id, device)
+    volume.wait_for { volume.status == 'in-use' }
+  end
+
+  def image_id
+    raise "Not yet implemented"
+  end
+
+  # Construct a Fog::Compute object
+  # Uses +attributes+ which normally originates from +settings.provider+
+  def setup_fog_connection
+    configuration = Fog.symbolize_credentials(attributes.credentials)
+    configuration[:provider] = "OpenStack"
+    @fog_compute = Fog::Compute.new(configuration)
+  end
+
+  def fog_attributes(inception_server)
+    # :name => "Inception VM",
+    # :key_name => key_name,
+    # :private_key_path => inception_vm_private_key_path,
+    # :flavor_ref => inception_flavor.id,
+    # :image_ref => inception_image.id,
+    # :security_groups => [settings["inception"]["security_group"]],
+    # :username => username
+    {
+      name: inception_server.server_name,
+      key_name: inception_server.key_name,
+      private_key_path: inception_server.private_key_path,
+      image_ref: inception_server.image_id,
+      flavor_ref: flavor_id(inception_server.flavor),
+      security_groups: inception_server.security_groups,
+      public_key: inception_server.public_key,
+      public_ip_address: inception_server.ip_address,
+      bits: 64,
+      username: "ubuntu",
+    }
+  end
+
+  def openstack_constants
+    Inception::Providers::Constants::OpenStackConstants
+  end
+end

data/lib/inception/providers/constants/aws_constants.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Inception; module Providers; module Constants; end; end; end
+
+module Inception::Providers::Constants::AwsConstants
+  extend self
+
+  # http://docs.aws.amazon.com/general/latest/gr/rande.html#region
+  def region_labels
+    [
+      { label: "US East (Northern Virginia) Region", code: "us-east-1" },
+      { label: "US West (Oregon) Region", code: "us-west-2" },
+      { label: "US West (Northern California) Region", code: "us-west-1" },
+      { label: "EU (Ireland) Region", code: "eu-west-1" },
+      { label: "Asia Pacific (Singapore) Region", code: "ap-southeast-1" },
+      { label: "Asia Pacific (Sydney) Region", code: "ap-southeast-2" },
+      { label: "Asia Pacific (Tokyo) Region", code: "ap-northeast-1" },
+      { label: "South America (Sao Paulo) Region", code: "sa-east-1" },
+    ]
+  end
+
+  def default_region_code
+    "us-east-1"
+  end
+end

data/lib/inception/providers/constants/openstack_constants.rb

@@ -0,0 +1,12 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Inception; module Providers; module Constants; end; end; end
+
+module Inception::Providers::Constants::OpenStackConstants
+  extend self
+
+  # explicit value representing "no region requested"
+  def no_region_code
+    "no-region-requested"
+  end
+end

data/lib/inception/providers.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Inception; end
+
+module Inception::Providers
+  extend self
+
+  # returns a BOSH provider (CPI) specific object
+  # with helpers related to that provider
+  # returns nil if +provider_name+ is unknown
+  def provider_client(attributes)
+    attributes = attributes.is_a?(Hash) ? ReadWriteSettings.new(attributes) : attributes
+    case attributes.name.to_sym
+    when :aws
+      @aws_provider_client ||= begin
+        require "inception/providers/clients/aws_provider_client"
+        Inception::Providers::Clients::AwsProviderClient.new(attributes)
+      end
+    when :openstack
+      @openstack_provider_client ||= begin
+        require "inception/providers/clients/openstack_provider_client"
+        Inception::Providers::Clients::OpenStackProviderClient.new(attributes)
+      end
+    else
+      nil
+    end
+  end
+end

data/lib/inception/version.rb

@@ -0,0 +1,3 @@
+module Inception
+  VERSION = "0.2.1"
+end

data/lib/inception.rb

@@ -0,0 +1,9 @@
+require "inception/version"
+
+module Inception
+end
+
+require "inception/providers"
+require "inception/inception_server"
+require "inception/inception_server_cookbook"
+require "inception/next_deploy_actions"

data/spec/assets/gitconfig

@@ -0,0 +1,5 @@
+[user]
+	name = Dr Nic Williams
+	email = drnicwilliams@gmail.com
+[github]
+	user = drnic

data/spec/assets/settings/aws-before-server.yml

@@ -0,0 +1,14 @@
+---
+provider:
+  name: aws
+  region: us-west-2
+  credentials:
+    aws_access_key_id: MOCK_AWS_ACCESS_KEY_ID
+    aws_secret_access_key: MOCK_AWS_SECRET_ACCESS_KEY
+inception:
+  ip_address: 54.214.15.178
+  key_pair:
+    name: inception
+    private_key: private_key
+  extensions:
+    - mosh

data/spec/assets/settings/aws-created-server.yml

@@ -0,0 +1,31 @@
+---
+provider:
+  name: aws
+  region: us-west-2
+  credentials:
+    aws_access_key_id: MOCK_AWS_ACCESS_KEY_ID
+    aws_secret_access_key: MOCK_AWS_SECRET_ACCESS_KEY
+git:
+  name: Dr Nic Williams
+  email: drnicwilliams@gmail.com
+inception:
+  key_pair:
+    name: inception
+    private_key: private_key
+  extensions:
+    - mosh
+  size: m1.small
+  disk_size: 16
+  provisioned:
+    ip_address: 54.214.15.178
+    image_id: ami-123456
+    server_id: i-e7f005d2
+    security_groups:
+      - ssh
+      - mosh
+    username: vcap
+    disk_device:
+      volume_id: vol-123456
+      external: /dev/sdf
+      internal: /dev/xvdf
+    host: ec2-54-214-15-178.us-west-2.compute.amazonaws.com

data/spec/integration/aws/aws_basic_spec.rb

@@ -0,0 +1,38 @@
+require File.expand_path("../../../support/aws/aws_helpers", __FILE__)
+
+describe "AWS deployment without Chef run" do
+  include FileUtils
+  include AwsHelpers
+
+  if AwsHelpers.aws_credentials?
+    before do
+      prepare_aws("basic", aws_region, "next_deploy_actions.no_converge" => true)
+    end
+    after(:all) do
+      destroy_test_constructs unless keep_after_test?
+    end
+
+    def aws_region
+      ENV['AWS_REGION'] || "us-west-2"
+    end
+
+    it "creates an EC2 inception/microbosh with the associated resources" do
+      create_manifest
+
+      manifest_file = home_file(".inception_server", "settings.yml")
+      File.should be_exists(manifest_file)
+
+      cmd.deploy
+
+      inception_servers = fog.servers.select { |s| s.tags["Name"] == test_server_name && s.ready? }
+      inception_servers.size.should == 1
+
+      server = inception_servers.first
+      server.volumes.size.should == 2
+      named_volume = server.volumes.select { |s| s.tags["Name"] == test_server_name }
+      named_volume.should_not be_nil
+    end
+  else
+    it "no AWS integration specs run; missing $AWS_ACCESS_KEY_ID etc"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test)
+
+$:.unshift(File.expand_path("../../lib", __FILE__))
+
+require "rspec/core"
+require "inception/providers"
+require "inception"
+require "inception/cli"
+
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+# load all files in spec/support/* (but not lower down)
+Dir[File.dirname(__FILE__) + '/support/*'].each do |path|
+  require path unless File.directory?(path)
+end
+
+def spec_asset(filename)
+  File.expand_path("../assets/#{filename}", __FILE__)
+end
+
+def setup_home_dir
+  home_dir = File.expand_path("../../tmp/home", __FILE__)
+  FileUtils.rm_rf(home_dir)
+  FileUtils.mkdir_p(home_dir)
+  ENV['HOME'] = home_dir
+  FileUtils.cp_r(spec_asset("gitconfig"), home_file(".gitconfig"))
+end
+
+# returns the file path to a file
+# in the fake $HOME folder
+def home_file(*path)
+  File.join(ENV['HOME'], *path)
+end
+
+RSpec.configure do |c|
+  c.before do
+    setup_home_dir
+    Fog::Mock.reset
+  end
+
+  c.color_enabled = true
+end

data/spec/support/aws/aws_helpers.rb

@@ -0,0 +1,73 @@
+require "active_support/core_ext/hash/keys"
+module AwsHelpers
+  extend self
+  include SettingsHelper
+
+  def keep_after_test?
+    ENV['KEEP_AFTER_TEST']
+  end
+
+  def region
+    @region ||= "us-west-2"
+  end
+
+  def fog
+    @fog ||= Fog::Compute.new(fog_credentials.merge(:region => region))
+  end
+
+  def aws_credentials?
+    access_key = ENV['AWS_ACCESS_KEY_ID']
+    secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+    access_key && secret_key
+  end
+
+  def fog_credentials
+    @fog_credentials ||= begin
+      access_key = ENV['AWS_ACCESS_KEY_ID']
+      secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+      unless access_key && secret_key
+        raise "Please provided $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY"
+      end
+      credentials = {
+        :provider                 => 'AWS',
+        :aws_access_key_id        => access_key,
+        :aws_secret_access_key    => secret_key
+      }
+    end
+  end
+
+  def prepare_aws(spec_name, aws_region, options={})
+    setup_home_dir
+    @cmd = nil
+    @fog = nil
+    create_manifest(options)
+    destroy_test_constructs
+  end
+
+  def unique_number
+    ENV['UNIQUE_NUMBER'] || Random.rand(100000)
+  end
+
+  def test_server_name
+    "test-inception"
+  end
+
+  def create_manifest(options = {})
+    credentials = options.delete(:credentials) || fog_credentials
+    setting "provider.name", "aws"
+    setting "provider.credentials", credentials.stringify_keys
+    setting "provider.region", region
+    setting "inception.name", test_server_name
+    options.each { |key, value| setting(key, value) }
+    cmd.save_settings!
+  end
+
+  def destroy_test_constructs
+    puts "Destroying everything created by previous test... "
+    # destroy servers using inception-vm SG
+    provider.delete_servers_with_name(test_server_name)
+    provider.delete_volumes_with_name(test_server_name)
+    provider.delete_key_pair_if_exists(test_server_name)
+    provider.cleanup_unused_ip_addresses
+  end
+end

data/spec/support/settings_helper.rb

@@ -0,0 +1,20 @@
+# assumes @cmd is Inception::Cli instance
+module SettingsHelper
+  def cmd
+    @cmd ||= Inception::Cli.new
+  end
+
+  def provider
+    cmd.provider_client
+  end
+
+  # Set a nested setting with "key1.key2.key3" notation
+  def setting(nested_key, value)
+    settings.set(nested_key, value)
+  end
+
+  # used by +SettingsSetter+ to access the settings
+  def settings
+    cmd.settings
+  end
+end

data/spec/support/stdout_capture.rb

@@ -0,0 +1,17 @@
+module StdoutCapture
+  # Captures stdout within the block
+  # Usage:
+  #
+  #   out = capture_stdout do
+  #     puts "this will not be shown"
+  #   end
+  #   out.should == "this will not be shown"
+  def capture_stdout(&block)
+    out = StringIO.new
+    $stdout = out
+    yield
+    return out
+  ensure
+    $stdout = STDOUT
+  end
+end

data/spec/unit/cli_delete_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path("../../support/aws/aws_helpers", __FILE__)
+
+require "fog"
+
+describe "AWS deployment deletion" do
+  include FileUtils
+  include StdoutCapture
+  include SettingsHelper
+  include AwsHelpers
+
+  before do
+    setup_home_dir
+    Fog.mock!
+    @cmd = Inception::Cli.new
+    setting "provider.name", "aws"
+    setting "provider.credentials.aws_access_key_id", "aws_access_key_id"
+    setting "provider.credentials.aws_secret_access_key", "aws_secret_access_key"
+    setting "provider.region", "us-west-2"
+    setting "next_deploy_actions.no_converge", true
+    capture_stdout { @cmd.deploy }
+  end
+
+  def perform_delete
+    config = { shell: Thor::Base.shell.new }
+    capture_stdout { @cmd.class.send(:dispatch, :delete, [], {:"non-interactive" => true}, config) }
+  end
+
+  it "clears out settings.yml" do
+    perform_delete
+    settings = ReadWriteSettings.new(File.expand_path("~/.inception_server/settings.yml"))
+    settings.exists?("provider").should_not be_nil
+    settings.exists?("git").should_not be_nil
+    settings.exists?("inception.provisioned.disk_device").should be_nil
+    settings.exists?("inception.provisioned.host").should be_nil
+    settings.exists?("inception.provisioned.ip_address").should be_nil
+    settings.exists?("inception.key_pair").should be_nil
+    settings.exists?("inception.provisioned.disk_device").should be_nil
+  end
+end