inception 0.1.0

data/lib/inception/cli_helpers/display.rb

@@ -0,0 +1,26 @@
+module Inception::CliHelpers
+  module Display
+    # Display header for a new section of the bootstrapper
+    def header(title, options={})
+      say "" # golden whitespace
+      if skipping = options[:skipping]
+        say "Skipping #{title}", [:yellow, :bold]
+        say skipping
+      else
+        say title, [:green, :bold]
+      end
+      say "" # more golden whitespace
+    end
+
+    def error(message)
+      say message, :red
+      exit 1
+    end
+
+    def confirm(message)
+      say "Confirming: #{message}", green
+      say "" # bonus golden whitespace
+    end
+
+  end
+end

data/lib/inception/cli_helpers/infrastructure.rb

@@ -0,0 +1,157 @@
+module Inception::CliHelpers
+  module Infrastructure
+    # Prompts user to choose an Iaas provider
+    # Sets settings.provider.name
+    def configure_provider
+      unless valid_infrastructure?
+        choose_fog_provider unless settings.exists?("provider.name")
+        choose_provider unless settings.exists?("provider.name")
+        setup_provider_credentials
+      end
+      confirm_infrastructure
+    end
+
+    # Displays a prompt for known IaaS that are configured
+    # within .fog config file if found.
+    #
+    # If no ~/.fog file found or user chooses "Alternate credentials"
+    # then no changes are made to settings.
+    #
+    # For example:
+    #
+    # 1. AWS (default)
+    # 2. AWS (bosh)
+    # 3. Alternate credentials
+    # Choose infrastructure:  1
+    #
+    # If .fog config only contains one provider, do not prompt.
+    #
+    # fog config file looks like:
+    # :default:
+    #   :aws_access_key_id:     PERSONAL_ACCESS_KEY
+    #   :aws_secret_access_key: PERSONAL_SECRET
+    # :bosh:
+    #   :aws_access_key_id:     SPECIAL_IAM_ACCESS_KEY
+    #   :aws_secret_access_key: SPECIAL_IAM_SECRET_KEY
+    #
+    # Convert this into:
+    # { "AWS (default)" => {:aws_access_key_id => ...}, "AWS (bosh)" => {...} }
+    #
+    # Then display options to user to choose.
+    #
+    # Currently detects following fog providers:
+    # * AWS
+    # * OpenStack
+    #
+    # If "Alternate credentials" is selected, then user is prompted for fog
+    # credentials:
+    # * provider?
+    # * access keys?
+    # * API URI or region?
+    #
+    # Sets (unless 'Alternate credentials' is chosen)
+    # * settings.provider.name
+    # * settings.provider.credentials
+    #
+    # For AWS, the latter has keys:
+    #   {:aws_access_key_id, :aws_secret_access_key}
+    #
+    # For OpenStack, the latter has keys:
+    #   {:openstack_username, :openstack_api_key, :openstack_tenant
+    #      :openstack_auth_url, :openstack_region }
+    def choose_fog_provider
+      fog_providers = {}
+      # Prepare menu options:
+      # each provider/profile name gets a menu choice option
+      fog_config.inject({}) do |iaas_options, fog_profile|
+        profile_name, profile = fog_profile
+        if profile[:aws_access_key_id]
+          # TODO does fog have inbuilt detection algorithm?
+          fog_providers["AWS (#{profile_name})"] = {
+            "name" => "aws",
+            "provider" => "AWS",
+            "aws_access_key_id" => profile[:aws_access_key_id],
+            "aws_secret_access_key" => profile[:aws_secret_access_key]
+          }
+        end
+        if profile[:openstack_username]
+          # TODO does fog have inbuilt detection algorithm?
+          fog_providers["OpenStack (#{profile_name})"] = {
+            "name" => "openstack",
+            "provider" => "OpenStack",
+            "openstack_username" => profile[:openstack_username],
+            "openstack_api_key" => profile[:openstack_api_key],
+            "openstack_tenant" => profile[:openstack_tenant],
+            "openstack_auth_url" => profile[:openstack_auth_url],
+            "openstack_region" => profile[:openstack_region]
+          }
+        end
+      end
+      # Display menu
+      # Include "Alternate credentials" as the last option
+      if fog_providers.keys.size > 0
+        hl.choose do |menu|
+          menu.prompt = "Choose infrastructure:  "
+          fog_providers.each do |label, credentials|
+            menu.choice(label) do
+              settings.set("provider.name", credentials.delete("name"))
+              settings.set("provider.credentials", credentials)
+              save_settings!
+            end
+          end
+          menu.choice("Alternate credentials")
+        end
+      end
+    end
+
+    # Prompts user to pick from the supported regions
+    def choose_provider
+      hl.choose do |menu|
+        menu.prompt = "Choose infrastructure:  "
+        menu.choice("AWS") do
+          settings.provider["name"] = "aws"
+        end
+        menu.choice("OpenStack") do
+          settings.provider["name"] = "openstack"
+        end
+      end
+    end
+
+    def setup_provider_credentials
+      say "Using provider #{settings.provider.name}:"
+      say ""
+      settings.set_default("provider", {}) # to ensure settings.provider exists
+      provider_cli = Bosh::Providers.provider_cli(settings.provider.name, settings.provider)
+      provider_cli.perform
+      settings["provider"] = provider_cli.export_attributes
+      settings.create_accessors!
+    end
+
+    def valid_infrastructure?
+      settings.exists?("provider.name") &&
+        settings.exists?("provider.region") &&
+        settings.exists?("provider.credentials") &&
+        provider_client
+    end
+
+    def confirm_infrastructure
+      confirm "Using #{settings.provider.name}/#{settings.provider.region}"
+    end
+
+    def fog_config
+      @fog_config ||= begin
+        if File.exists?(File.expand_path(fog_config_path))
+          say "Found infrastructure API credentials at #{fog_config_path} (override with $FOG)"
+          YAML.load_file(File.expand_path(fog_config_path))
+        else
+          say "No existing #{fog_config_path} fog configuration file", :yellow
+          {}
+        end
+      end
+    end
+
+    def fog_config_path
+      ENV['FOG'] || "~/.fog"
+    end
+  end
+end

data/lib/inception/cli_helpers/interactions.rb

@@ -0,0 +1,15 @@
+module Inception::CliHelpers
+  module Interactions
+    def cyan; "\033[36m" end
+    def clear; "\033[0m" end
+    def bold; "\033[1m" end
+    def red; "\033[31m" end
+    def green; "\033[32m" end
+    def yellow; "\033[33m" end
+
+    # Helper to access HighLine for ask & menu prompts
+    def hl
+      @hl ||= HighLine.new
+    end
+  end
+end

data/lib/inception/cli_helpers/prepare_deploy_settings.rb

@@ -0,0 +1,89 @@
+module Inception::CliHelpers
+  module PrepareDeploySettings
+    def update_git_config
+      gitconfig = File.expand_path("~/.gitconfig")
+      if File.exists?(gitconfig)
+        say "Using your git user.name (#{`git config -f #{gitconfig} user.name`.strip})"
+        settings.set("git.name", `git config -f #{gitconfig} user.name`.strip)
+        settings.set("git.email", `git config -f #{gitconfig} user.email`.strip)
+        save_settings!
+      end
+    end
+
+    # Attempt to provision a new public IP; if none available,
+    # then look for a pre-provisioned public IP that's not assigned
+    # to a server; else error. The user needs to go get more
+    # public IP addresses in this region.
+    def provision_or_reuse_public_ip_address_for_inception
+      say "Acquiring a public IP address... "
+      if public_ip = provider_client.provision_or_reuse_public_ip_address
+        say public_ip, :green
+        settings.set("inception.provisioned.ip_address", public_ip)
+        save_settings!
+      else
+        say "none available.", :red
+        error "Please rustle up at least one public IP address and try again."
+      end
+    end
+
+    def default_server_name
+      "inception"
+    end
+
+    def default_key_pair_name
+      default_server_name
+    end
+
+    def recreate_key_pair_for_inception
+      key_pair_name = settings.set_default("inception.key_pair.name", default_key_pair_name)
+      provider_client.delete_key_pair_if_exists(key_pair_name)
+      key_pair = provider_client.create_key_pair(key_pair_name)
+      settings.set("inception.key_pair.private_key", key_pair.private_key)
+      settings.set("inception.key_pair.fingerprint", key_pair.fingerprint)
+      save_settings!
+    end
+
+    def private_key_path_for_inception
+      @private_key_path_for_inception ||= File.join(settings_dir, "ssh", settings.inception.key_pair.name)
+    end
+
+    # The keys for the inception server originate from the provider and are cached in
+    # the manifest. The private key is stored locally; the public key is placed
+    # on the inception server.
+    def recreate_private_key_file_for_inception
+      mkdir_p(File.dirname(private_key_path_for_inception))
+      File.chmod(0700, File.dirname(private_key_path_for_inception))
+      File.open(private_key_path_for_inception, "w") { |file| file << settings.inception.key_pair.private_key }
+      File.chmod(0600, private_key_path_for_inception)
+    end
+
+
+    # Required settings:
+    # * git.name
+    # * git.email
+    def validate_deploy_settings
+      begin
+        settings.git.name
+        settings.git.email
+      rescue Settingslogic::MissingSetting => e
+        error "Please setup local git user.name & user.email config; or specify git.name & git.email in settings.yml"
+      end
+
+      begin
+        settings.provider.name
+        settings.provider.region
+        settings.provider.credentials
+      rescue Settingslogic::MissingSetting => e
+        error "Wooh there, we need provider.name, provider.region, provider.credentials in settings.yml to proceed."
+      end
+
+      begin
+        settings.inception.provisioned.ip_address
+        settings.inception.key_pair.name
+        settings.inception.key_pair.private_key
+      rescue Settingslogic::MissingSetting => e
+        error "Wooh there, we need inception.provisioned.ip_address, inception.key_pair.name, & inception.key_pair.private_key in settings.yml to proceed."
+      end
+    end
+  end
+end

data/lib/inception/cli_helpers/provider.rb

@@ -0,0 +1,14 @@
+module Inception::CliHelpers
+  module Provider
+    def provider_client
+      @provider_client ||= begin
+        Bosh::Providers.provider_client(settings.provider)
+      end
+    end
+
+    # If the +provider_client+ uses fog, then this will return its +fog_compute+ client object
+    def fog_compute
+      provider_client.respond_to?(:fog_compute) ? provider_client.fog_compute : nil
+    end
+  end
+end

data/lib/inception/cli_helpers/settings.rb

@@ -0,0 +1,47 @@
+require "settingslogic"
+
+module Inception::CliHelpers
+  module Settings
+    include FileUtils
+
+    # The base directory for holding the manifest settings file
+    # and private keys
+    #
+    # Defaults to ~/.bosh_inception; and can be overridden with either:
+    # * $SETTINGS - to a folder (supported method)
+    def settings_dir
+      @settings_dir ||= File.expand_path(ENV["SETTINGS"] || "~/.bosh_inception")
+    end
+
+    def settings_ssh_dir
+      File.join(settings_dir, "ssh")
+    end
+
+    def settings_path
+      @settings_path ||= File.join(settings_dir, "settings.yml")
+    end
+
+    def settings
+      @settings ||= begin
+        unless File.exists?(settings_path)
+          mkdir_p(settings_ssh_dir)
+          File.open(settings_path, "w") { |file| file << "--- {}" }
+        end
+        chmod(0600, settings_path)
+        chmod(0700, settings_ssh_dir)
+        Settingslogic.new(settings_path)
+      end
+    end
+
+    # Saves current nested Settingslogic into pure Hash-based YAML file
+    # Recreates accessors on Settingslogic object (since something has changed)
+    def save_settings!
+      File.open(settings_path, "w") { |f| f << settings.to_nested_hash.to_yaml }
+      settings.create_accessors!
+    end
+
+    def migrate_old_settings
+    end
+
+  end
+end

data/lib/inception/inception_server.rb

@@ -0,0 +1,305 @@
+require "fog"
+
+module Inception
+  class InceptionServer
+
+    DEFAULT_SERVER_NAME = "inception"
+    DEFAULT_FLAVOR = "m1.small"
+    DEFAULT_DISK_SIZE = 16
+    DEFAULT_SECURITY_GROUPS = ["ssh"]
+
+    attr_reader :attributes
+
+    # @provider_client [Bosh::Providers::FogProvider] - interact with IaaS
+    # @attributes [Settingslogic]
+    #
+    # Required @attributes:
+    #   {
+    #     "name" => "inception",
+    #     "ip_address" => "54.214.15.178",
+    #     "key_pair" => {
+    #       "name" => "inception",
+    #       "private_key" => "private_key",
+    #       "public_key" => "public_key"
+    #     }
+    #   }
+    #
+    # Including optional @attributes and default values:
+    #   {
+    #     "name" => "inception",
+    #     "ip_address" => "54.214.15.178",
+    #     "security_groups" => ["ssh"],
+    #     "flavor" => "m1.small",
+    #     "key_pair" => {
+    #       "name" => "inception",
+    #       "private_key" => "private_key",
+    #       "public_key" => "public_key"
+    #     }
+    #   }
+    def initialize(provider_client, attributes, ssh_dir)
+      @provider_client = provider_client
+      @ssh_dir = ssh_dir
+      @attributes = attributes.is_a?(Hash) ? Settingslogic.new(attributes) : attributes
+      raise "@attributes must be Settingslogic (or Hash)" unless @attributes.is_a?(Settingslogic)
+    end
+
+    # Create the underlying server, with key pair & security groups, unless it is already created
+    #
+    # The @attributes hash is updated with a `provisioned` key during/after creation.
+    # When saved as YAML it might look like:
+    #   inception:
+    #     provisioned:
+    #       image_id: ami-123456
+    #       server_id: i-e7f005d2
+    #       security_groups:
+    #         - ssh
+    #         - mosh
+    #       username: ubuntu
+    #       disk_device: /dev/sdi
+    #       host: ec2-54-214-15-178.us-west-2.compute.amazonaws.com
+    #       validated: true
+    #       converged: true
+    def create
+      validate_attributes_for_bootstrap
+      ensure_required_security_groups
+      create_missing_default_security_groups
+      bootstrap_vm
+      attach_persistent_disk
+    end
+
+    # Delete the server, volume and release the IP address
+    def delete_all
+      delete_server
+      delete_volume
+      delete_key_pair
+      release_ip_address
+    end
+
+    def delete_server
+      @fog_server = nil # force reload of fog_server model
+      if fog_server
+        print "Deleting server..."
+        fog_server.destroy
+        wait_for_termination(fog_server) unless Fog.mocking?
+        puts "done."
+      else
+        puts "Server already destroyed"
+      end
+      provisioned.delete("host")
+      provisioned.delete("server_id")
+      provisioned.delete("username")
+    end
+    
+    def delete_volume
+      volume_id = provisioned.exists?("disk_device.volume_id")
+      if volume_id && (volume = fog_compute.volumes.get(volume_id)) && volume.ready?
+        print "Deleting volume..."
+        volume.destroy
+        wait_for_termination(volume, "deleting")
+        puts ""
+      else
+        puts "Volume already destroyed"
+      end
+      provisioned.delete("disk_device")
+    end
+    
+    def delete_key_pair
+      key_pair_name = attributes.exists?("key_pair.name")
+      if key_pair_name && key_pair = fog_compute.key_pairs.get(key_pair_name)
+        puts "Deleting key pair '#{key_pair_name}'"
+        key_pair.destroy
+      else
+        puts "Keypair already destroyed"
+      end
+      attributes.delete("key_pair")
+    end
+    
+
+    def release_ip_address
+      public_ip = provisioned.exists?("ip_address")
+      if public_ip && ip_address = fog_compute.addresses.get(public_ip)
+        puts "Releasing IP address #{public_ip}"
+        ip_address.destroy
+      else
+        puts "IP address already released"
+      end
+      provisioned.delete("ip_address")
+    end
+
+    def security_groups
+      @attributes.security_groups
+    end
+
+    def server_name
+      @attributes["name"] ||= DEFAULT_SERVER_NAME
+      @attributes.name
+    end
+
+    def key_name
+      @attributes.key_pair.name
+    end
+
+    def private_key_path
+      @private_key_path ||= File.join(@ssh_dir, key_name)
+    end
+
+    # Flavor/instance type of the server to be provisioned
+    # TODO: DEFAULT_FLAVOR should become IaaS/provider specific
+    def flavor
+      @attributes["flavor"] ||= DEFAULT_FLAVOR
+    end
+
+    # Size of attached persistent disk for the inception server
+    def disk_size
+      @attributes["disk_size"] ||= DEFAULT_DISK_SIZE
+    end
+
+    def ip_address
+      provisioned.ip_address
+    end
+
+    def image_id
+      @attributes["image_id"] ||= @provider_client.raring_image_id
+    end
+
+    # The progresive/final attributes of the provisioned Inception server &
+    # persistent disk.
+    def provisioned
+      @attributes["provisioned"] = {} unless @attributes["provisioned"]
+      @attributes.provisioned
+    end
+
+    # Because @attributes["provisioned"] is not the same as @attributes.provisioned
+    # we need a helper to export the complete nested attributes.
+    def export_attributes
+      attrs = attributes.to_nested_hash
+      attrs["provisioned"] = provisioned.to_nested_hash
+      attrs
+    end
+
+    def disk_devices
+      provisioned["disk_device"] ||= default_disk_device
+    end
+
+    def external_disk_device
+      disk_devices["external"]
+    end
+
+    def default_disk_device
+      case @provider_client
+      when Bosh::Providers::Clients::AwsProviderClient
+        { "external" => "/dev/sdf", "internal" => "/dev/xvdf" }
+      when Bosh::Providers::Clients::OpenStackProviderClient
+        { "external" => "/dev/vdc", "internal" => "/dev/vdc" }
+      else
+        raise "Please implement InceptionServer#default_disk_device for #{@provider_client.class}"
+      end
+    end
+
+    def user_host
+      "#{provisioned.username}@#{provisioned.host}"
+    end
+
+    def fog_server
+      @fog_server ||= begin
+        if server_id = provisioned["server_id"]
+          fog_compute.servers.get(server_id)
+        end
+      end
+    end
+
+    def fog_compute
+      @provider_client.fog_compute
+    end
+
+    protected
+    # set_resource_name(fog_server, "inception")
+    # set_resource_name(volume, "inception-root")
+    # set_resource_name(volume, "inception-store")
+    def set_resource_name(resource, name)
+      @provider_client.set_resource_name(resource, name)
+    end
+
+    def fog_attributes
+      {
+        :image_id => image_id,
+        :groups => security_groups,
+        :key_name => key_name,
+        :private_key_path => private_key_path,
+        :flavor_id => flavor,
+        :public_ip_address => ip_address,
+        :bits => 64,
+        :username => "ubuntu",
+      }
+    end
+
+    def validate_attributes_for_bootstrap
+      missing_attributes = []
+      missing_attributes << "provisioned.ip_address" unless @attributes.exists?("provisioned.ip_address")
+      missing_attributes << "key_pair.private_key" unless @attributes.exists?("key_pair.private_key")
+      if missing_attributes.size > 0
+        raise "Missing InceptionServer attributes: #{missing_attributes.join(', ')}"
+      end
+    end
+
+    # ssh group must be first (bootstrap method looks for port 22 in first group)
+    def ensure_required_security_groups
+      if @attributes["security_groups"] && @attributes["security_groups"].is_a?(Array)
+        unless @attributes["security_groups"].include?("ssh")
+          @attributes["security_groups"] = ["ssh", *@attributes["security_groups"]]
+        end
+      else
+        @attributes["security_groups"] = ["ssh"]
+      end
+    end
+
+    def create_missing_default_security_groups
+      # provider method only creates group if missing
+      @provider_client.create_security_group("ssh", "ssh", {ssh: 22})
+    end
+
+    def bootstrap_vm
+      unless fog_server
+        say "Booting #{flavor} inception server..."
+        @fog_server = @provider_client.bootstrap(fog_attributes)
+        provisioned["server_id"] = fog_server.id
+        provisioned["host"] = fog_server.dns_name || fog_server.public_ip_address
+        provisioned["username"] = fog_attributes[:username]
+      end
+      set_resource_name(fog_server, server_name)
+    end
+
+    def attach_persistent_disk
+      unless Fog.mocking?
+        Fog.wait_for(60) { fog_server.sshable?(ssh_options) }
+      end
+
+      unless volume = @provider_client.find_server_device(fog_server, external_disk_device)
+        say "Provisioning #{disk_size}Gb persistent disk for inception server..."
+        volume = @provider_client.create_and_attach_volume("Inception Disk", disk_size, fog_server, external_disk_device)
+        disk_devices["volume_id"] = volume.id
+      end
+      set_resource_name(volume, server_name)
+    end
+
+    def ssh_options
+      {
+        keys: [private_key_path]
+      }
+    end
+
+    # Poll a fog model until it terminates; print . each second
+    def wait_for_termination(fog_model, state_to_wait_for="terminated")
+      fog_model.wait_for do
+        print "."
+        state == state_to_wait_for
+      end
+    end
+
+    protected
+    # TODO emit events rather than writing directly to STDOUT
+    def say(*args)
+      puts(*args)
+    end
+  end
+end