dew 0.1.0

data/lib/dew/models/deploy/run.rb

@@ -0,0 +1,19 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'deploy', 'puge'))
+
+module Deploy
+
+  class Run
+
+    attr_reader :deploy_type, :environment, :opts
+
+    def initialize deploy_type, environment, opts
+      @deploy_type = deploy_type
+      @environment = environment
+      @opts = opts
+    end
+
+    def deploy
+      Deploy.const_get(deploy_type.capitalize).new(@environment.servers, @opts).deploy
+    end
+  end
+end

data/lib/dew/models/environment.rb

@@ -0,0 +1,199 @@
+require 'dew/validations'
+
+class Environment
+  include Validations
+
+  attr_reader :name, :servers, :database
+
+  def initialize name, servers=[], database=nil
+    @name = name
+    @servers = servers
+    @database = database
+    valid?
+  end
+
+  def valid?
+    Validation::validates_format_of @name, /^[a-zA-Z0-9-]+$/
+  end
+
+  def self.get name
+    servers = Server.find('Environment', name)
+    database = Database.get(name)
+    servers.length > 0 || database ? new(name, servers, database) : nil
+  end
+
+  def self.create(name, profile)
+    raise "Keypair '#{profile.keypair}' is not available in AWS #{Cloud.region}." unless Cloud.keypair_exists?(profile.keypair)
+    raise "AMI for '#{Cloud.region}' is not setup in the '#{profile.profile_name}' profile." unless profile.ami
+
+    environment = new(name)
+    Inform.info "Creating new environment %{name}", :name => name
+
+    # Creating the database is the longest running task, so do that first.
+    password = Password.random if profile.has_rds?
+    environment.add_database(profile.rds_size, password) if profile.has_rds?
+
+    (1..profile.count).each do
+      environment.add_server(profile.ami, profile.size, profile.keypair, profile.security_groups)
+    end
+
+    environment.add_elb(profile.elb_listener_ports) if profile.has_elb?
+
+    environment.wait_until_ready
+
+    environment.configure_servers_for_database password if profile.has_rds?
+    Inform.info "Environment %{name} ready!", :name => name
+    environment
+  end
+
+  def self.owners
+    Cloud.valid_servers.collect(&:tags).collect { |h| {:name => h['Environment'], :owner => h['Creator']} if h['Environment'] }.uniq.compact
+  end
+
+  def self.index
+    #environments = rows.inject({}) { |h, (name, owner)| h[name] = Environment.get(name); h }
+    #items = rows.collect { |name, owner| [name, owner, environments[name].servers.size, environments[name].database] }
+    #table(%w(name owner instance_count database), *rows).to_s.indent
+    Inform.info View.new('Environments', owners, %w(name owner)).index
+  end
+
+  def show
+    result = ""
+
+    result << "ENVIRONMENT:\n"
+
+    result << "NAME: #{name}\n"
+    result << "REGION: #{Cloud.region}\n"
+    result << "ACCOUNT: #{Cloud.account.aws_user_id}\n"
+
+    unless servers.empty?
+      #keys = %w(id flavor_id public_ip_address state created_at key_name tags)
+      keys = %w(id flavor_id public_ip_address state created_at groups key_name availability_zone creator)
+      servers_view = View.new('SERVERS', servers, keys)
+      result << servers_view.index
+    end
+
+    if elb
+      elbs_view = View.new('ELB', [elb], %w(created_at dns_name instances availability_zones))
+      result << elbs_view.show(0)
+    end
+
+    if database
+      databases_view = View.new('DATABASE', [database], %w(flavor_id state created_at availability_zone db_security_groups))
+      result << databases_view.show(0)
+    end
+
+    Inform.info result
+  end
+
+  def destroy
+    servers.each do |server|
+      Inform.info("Destroying server %{serverid}", :serverid => server.id) do
+        server.destroy
+      end
+    end
+    if database
+      Inform.info("Destroying database") do
+        database.destroy(nil)
+      end
+    end
+    if has_elb?
+      Inform.info("Destroying load balancer") do
+        Cloud.elb.delete_load_balancer(name)
+      end
+    end
+  end
+
+  def add_server ami, size, keypair, groups
+    Inform.info "Adding server using AMI %{ami} of size %{size}, keypair %{keypair} and security groups %{groups}",
+      :ami => ami, :size => size, :keypair => keypair, :groups => groups do
+      server = Server.create!( ami, size, keypair, groups )
+      server.add_tag('Environment', name)
+      server.add_tag('Creator', ENV['USER'])
+      server_name = "#{name} #{servers.count + 1}"
+      server.add_tag('Name', server_name)
+      Inform.debug("%{name} ID: %{id} AZ: %{az}", :name => server_name, :id =>server.id, :az => server.availability_zone)
+      servers << server
+    end
+  end
+
+  def add_server_to_elb server
+    Inform.debug("Adding %{id} to ELB", :id => server.id)
+    Cloud.elb.register_instances_with_load_balancer([server.id], name)
+  end
+  
+  def remove_server_from_elb server
+    Inform.debug("Removing %{id} from ELB", :id => server.id)
+    Cloud.elb.deregister_instances_from_load_balancer([server.id], name)
+  end
+      
+  def add_elb listener_ports=[]
+    zones = servers.map {|s| s.availability_zone}.uniq
+    Inform.info "Adding Load Balancer for availability zone(s) %{zones}", :zones => zones.join(', ') do
+      Cloud.elb.create_load_balancer(zones, name, listeners_from_ports(listener_ports))
+      Inform.debug("ELB created, adding instances...")
+      Cloud.elb.register_instances_with_load_balancer(servers.map {|s| s.id}, name)
+    end
+  end
+
+  def add_database size, password
+    Inform.info "Adding Database %{name} of size %{size} with master password %{password}",
+      :name => name, :size => size, :password => password do
+      @database = Database.create!(name, size, password)
+    end
+  end
+
+  def wait_until_ready
+    Inform.info "Waiting for servers to become ready" do
+      servers.each do |server|
+        ip_permissions = server.groups.map { |group_name| Cloud.security_groups[group_name] }.compact.collect(&:ip_permissions)
+        if ip_permissions.flatten.empty?
+          Inform.warning "Server %{id} has no ip_permissions in its security groups %{security_group_names}", :id => server.id, :security_group_names => server.groups.inspect
+        else
+          Inform.debug "Trying to connect to %{id} using ip_permissions %{ip_permissions}", :id => server.id, :ip_permissions => ip_permissions
+          server.wait_until_ready
+        end
+      end
+    end
+    if database
+      Inform.info "Waiting for database to become ready" do
+        database.wait_until_ready if database
+      end
+    end
+  end
+
+  def configure_servers_for_database password
+    Inform.info "Configuring servers for database" do
+      servers.each do |server|
+        Inform.debug "%{id}", :id => server.id
+        server.configure_for_database(database, password)
+      end
+    end
+  end
+
+  def elb
+    @elb ||= Cloud.elb.load_balancers.detect { |elb| elb.id == name }  # XXX - need to refactor
+  end
+
+  def has_elb?
+    elb_descriptions.select { |elb|
+      elb['LoadBalancerName'] == name
+    }.length > 0
+  end
+
+  private
+
+  def listeners_from_ports ports
+    ports.map do |port|
+      {
+              'Protocol' => 'TCP',
+              'LoadBalancerPort' => port,
+              'InstancePort' => port,
+      }
+    end
+  end
+
+  def elb_descriptions name = nil
+    Cloud.elb.describe_load_balancers(name).body['DescribeLoadBalancersResult']['LoadBalancerDescriptions']
+  end
+end

data/lib/dew/models/fog_model.rb

@@ -0,0 +1,23 @@
+class FogModel
+  def initialize fog_object
+    @fog_object = fog_object
+  end
+  
+  def id
+    @fog_object.id
+  end
+  
+  def wait_until_ready
+    @fog_object.wait_for { ready? }
+  end
+  
+  def method_missing method_sym, *args, &block
+    @fog_object.send(method_sym, *args, &block)
+  end
+  
+  protected
+  
+  def fog_object
+    @fog_object
+  end
+end

data/lib/dew/models/profile.rb

@@ -0,0 +1,60 @@
+require 'yaml'
+
+class Profile
+
+  attr_reader :profile_name
+  attr_accessor :ami, :size, :security_groups, :keypair, :count
+  attr_accessor :rds_size, :elb_listener_ports
+
+  AWS_RESOURCES = YAML.load(File.read(File.join(File.dirname(__FILE__), '..', 'aws_resources.yaml')))
+
+  def self.read(profile_name)
+    file = File.read(File.join(ENV['HOME'], '.dew', 'profiles', "#{profile_name}.yaml"))
+    yaml = YAML.load(file)
+    profile = new(profile_name)
+    if yaml['instances']
+      profile.ami = yaml['instances']['amis'][Cloud.region]
+      profile.size = yaml['instances']['size']
+      profile.security_groups = yaml['instances']['security-groups'] || ['default']
+      profile.keypair = yaml['instances']['keypair']
+      profile.count = yaml['instances']['count'].to_i
+    end
+    if yaml['elb']
+      profile.elb_listener_ports = yaml['elb']['listener_ports']
+    end
+    if yaml['rds']
+      profile.rds_size = yaml['rds']['size']
+    end
+    profile
+  end
+  
+  def has_elb?
+    elb_listener_ports != nil
+  end
+  
+  def has_rds?
+    rds_size != nil
+  end
+
+  def initialize(profile_name)
+    @profile_name = profile_name
+    # :ami, :size, :security_groups, :keypair, :count
+    # :rds_size, :elb_listener_ports
+  end
+
+  def to_s
+    db_instance_str = "%{memory} memory, %{processor} processor, %{platform} platform, %{io_performance} I/O performance"
+    #instance_str = "%{memory} memory, %{processor} processor, %{storage} storage, %{platform} platform, %{io_performance} I/O performance"
+    instance_str = "%{memory} GB memory, %{processor} ECUs processor, %{storage} GB storage, %{platform}-bit platform, %{io_performance} I/O performance"
+    flavor = Cloud.compute.flavors.detect { |f| f.id == size }
+    instance_hash = { :memory => flavor.ram.to_s, :processor => flavor.cores.to_s, :storage => flavor.disk.to_s, :platform => flavor.bits.to_s, :io_performance => '??' }
+    table { |t|
+      t << [ "#{count} instance#{'s' if count > 1}", "#{size.inspect} (#{instance_str % instance_hash })"]
+      t << ['disk image', ami.inspect]
+      t << ['load balancer', "listener ports: #{elb_listener_ports.inspect}"] if has_elb?
+      t << ['database', "#{rds_size.inspect} (#{db_instance_str % AWS_RESOURCES['db_instance_types'][rds_size]})"] if has_rds?
+      t << ['security groups', security_groups.inspect]
+      t << ['keypair', keypair.inspect]
+    }.to_s
+  end
+end

data/lib/dew/models/server.rb

@@ -0,0 +1,134 @@
+require 'gofer'
+
+class Server < FogModel
+  TEN_SECONDS = 10
+  SIXTY_SECONDS = 60
+  TWO_MINUTES = 120
+  RUNNING_SERVER_STATES = %w{pending running}
+
+  def self.create! ami, size, keypair, groups
+    new(Cloud.compute.servers.create(:image_id => ami, :flavor_id => size, :key_name => keypair, :groups => groups))
+  end
+
+  def self.find tag_name, tag_value
+    Cloud.compute.servers.all("tag:#{tag_name}" => tag_value).select{|s| RUNNING_SERVER_STATES.include?(s.state)}.map {|s| new s }
+  end
+
+  def creator
+    @creator ||= fog_object.tags["Creator"]
+  end
+
+  def add_tag key, val
+    try_for(SIXTY_SECONDS) {
+      Cloud.compute.tags.create(:resource_id => id, :key => key, :value => val)
+    }
+  end
+
+  def configure_for_database database, password
+    ssh.write(database.db_environment_file(password), '/tmp/envfile')
+    ssh.run('sudo mv /tmp/envfile /etc/environment')
+  end
+
+  def username
+    'ubuntu'
+  end
+
+  def ssh
+    Gofer::Host.new(public_ip_address, username, :key_data => [File.read(Cloud.keyfile_path(key_name))], :paranoid => false, :quiet => true)
+  end
+
+  def wait_until_ready ssh_timeout=TWO_MINUTES
+    super()
+    Inform.debug("%{id} online at %{ip}, waiting for SSH connection...", :id => id, :ip => public_ip_address)
+    wait_for_ssh ssh_timeout
+    Inform.debug("Connected to %{id} via SSH successfully", :id => id)
+  end
+
+  def create_ami ami_name
+    image_id = Cloud.compute.create_image(id, ami_name, "Created by #{ENV['USER']} on #{Time.now.strftime("%Y-%m-%d")}").body['imageId']
+
+    Inform.debug("Created image at %{id}, waiting for AWS to recognize it...", :id => image_id)
+    # Sometimes takes a while for AWS to realise there's a new image...
+    image = Timeout::timeout(SIXTY_SECONDS) do
+      image = nil
+      while image == nil
+        image = Cloud.compute.images.get(image_id)
+      end
+      image
+    end
+    Inform.debug("Image recognized, waiting for it to become available...")
+
+    image.wait_for { state == 'available' }
+    Inform.debug("Image available, sharing with other accounts...")
+
+    Account.user_ids.each do |user_id|
+      Inform.debug("Sharing %{id} with %{user_id}", :id => image_id, :user_id => user_id)
+      Cloud.compute.modify_image_attributes(image_id, 'launchPermission', 'add', 'UserId' => user_id)
+    end
+    image_id
+  end
+
+  def credentials
+    @credentials ||= if key_name
+      keyfile_path = Cloud.keyfile_path(key_name)
+
+      sanitize_key_file(key_name, keyfile_path)
+
+      "-i #{keyfile_path} -o StrictHostKeyChecking=no #{username}@#{public_ip_address}"
+    else
+      Inform.warning("Server %{id} has no key and therefore can not be accessed.", :id => id)
+      false
+    end
+  end
+
+  private
+
+  def sanitize_key_file name, path
+    begin
+      stat = File.stat(path)
+      raise "Keyfile at #{keyfile_path} not owned by #{ENV['USER']}, can't SSH" if stat.uid != Process.euid
+      if (stat.mode & 077) != 0
+        Inform.info("Changing permissions on key at %{path} to 0600", :path => path) do
+          File.chmod(0600, path)
+        end
+      end
+    rescue Errno::ENOENT
+      raise "Can't find keyfile for #{name} under this account/region (looking in #{path})"
+    end
+  end
+
+  def wait_for_ssh timeout
+    try_for(timeout) {
+      begin
+        Timeout::timeout(15) do
+          ssh.run('uptime')
+        end
+      # SshTimeout magic is here due to a weird bug with Ruby 1.8.7 where the Timeout::Error
+      # will not be caught by wait_for_proc!
+      rescue Timeout::Error => e
+        raise "SSH Timeout Encountered: #{e.to_s}"
+      end
+    }
+  end
+
+  def try_for(timeout, &block)
+    start_time = Time.now
+    time_spent = 0
+    success = false
+    last_exception = nil
+    while !success && time_spent < timeout
+      begin
+        block.call
+        success = true
+      rescue => e
+        time_spent = Time.now - start_time
+        Inform.debug("Exception: %{m} (%{c}), in block %{block} after %{time_spent}s (retrying until %{timeout}s)...", :c => e.class.to_s, :m => e.message, :block => block, :time_spent => time_spent.to_i, :timeout => timeout)
+        sleep 1
+        last_exception = e
+      end
+    end
+    #raise unless success
+    raise last_exception unless success
+  end
+
+end

data/lib/dew/password.rb

@@ -0,0 +1,7 @@
+class Password
+
+  def self.random length=12
+    (0...length).map{ ('a'..'z').to_a[rand(26)] }.join
+  end
+
+end