dew 0.1.0

data/lib/dew/commands/deploy/templates/apache.conf.erb

@@ -0,0 +1,28 @@
+<VirtualHost *:80>
+    ServerAdmin admin@playup.com
+
+    PassEnv PUGE_DB_HOST PUGE_DB_NAME PUGE_DB_USERNAME PUGE_DB_PASSWORD
+
+    PassengerFriendlyErrorPages off
+    PassengerMaxRequests 50
+
+    RailsEnv <%= rails_env %>
+
+    DocumentRoot <%= working_directory %>/public
+    <Directory <%= working_directory %>/public>
+        allow from all
+        Options -MultiViews
+        AllowOverride None
+        Order allow,deny
+    </Directory>
+
+    ErrorLog /var/log/apache2/<%= application_name %>-error.log
+
+    # Possible values include: debug, info, notice, warn, error, crit,
+    # alert, emerg.
+    LogLevel warn
+
+    CustomLog /var/log/apache2/<%= application_name %>-access.log combined
+
+    PassengerPreStart http://localhost/status
+</VirtualHost>

data/lib/dew/commands/deploy/templates/known_hosts

@@ -0,0 +1,2 @@
+|1|cFg9wYM1j4pHeyeYQaENVE2Jb/g=|hA1okVZROh9DyjcslQNVy2FAH7Q= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+|1|WQr8sEZnbYuTqpjNc5JTjJUZLuc=|PDsO3bSoH/tWy/fpJ4+GUluZC+I= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

data/lib/dew/commands/deploy/templates/rvmrc

@@ -0,0 +1,2 @@
+# rvm_trust_rvmrcs_flag=1
+# unused

data/lib/dew/commands/environments.rb

@@ -0,0 +1,110 @@
+require 'dew/controllers/environments_controller'
+
+
+class EnvironmentsCommand < Clamp::Command
+
+  #STATUS_KEYS=%w(arch cpu_count cpu_freq disk ec2_cost ip_address load_average mem_available mem_used network processes reboot_required release time_utc updates_available uptime users)
+  STATUS_KEYS=%w(ip_address arch cpu_count cpu_freq disk ec2_cost load_average mem_available mem_used network processes release time_utc uptime)
+  STATUS_CMD="'[ -d ~/.byobu ] || mkdir ~/.byobu; touch ~/.byobu/status; for cmd in #{STATUS_KEYS.join(' ')}; do echo `/usr/lib/byobu/$cmd 2>/dev/null`; done'"
+
+  def controller
+    @controller ||= EnvironmentsController.new
+  end
+
+  default_subcommand "index", "Show environments" do
+
+    def execute
+      controller.index
+    end
+
+  end
+
+  subcommand "create", "Create a new environment" do
+
+    option ['-f', '--force'], :flag, "Don't ask for confirmation before creating", :default => false
+    parameter "PROFILE", "Profile describing resources to be created", :attribute_name => 'profile_name'
+    parameter "ENVIRONMENT_NAME", "Name of the environment"
+
+    def execute
+      controller.create(environment_name, profile_name, :force => force?)
+    end
+
+  end
+
+  subcommand "show", "Show environment" do
+
+    parameter "ENVIRONMENT_NAME", "Name of the environment"
+
+    def execute
+      controller.show(environment_name)
+    end
+
+  end
+
+  subcommand "status", "Get the status of an instance (NB. requires byobu to be installed)" do
+    parameter "[ENVIRONMENT_NAME]", "Name of the environment"
+    option ['-i', '--instance'], 'INSTANCE_NUMBER', "Which instance to SSH to", :default => 1 do |s|
+      Integer(s)
+    end
+
+    def execute
+      environment_owners = Environment.owners
+      environment_owners = [environment_owners.detect { |o| o[:name] == environment_name }] if environment_name
+      rows = []
+      environment_owners.collect { |o|
+        environment_name = o[:name]
+        environment_owner = o[:owner]
+        instance_count = Environment.get(environment_name).servers.size
+        (1..instance_count).collect { |instance_no|
+          server = get_server(environment_name, instance_no)
+          if server.credentials
+            command = "ssh #{server.credentials} #{STATUS_CMD}"
+            Inform.debug("Running %{command}", :command => command)
+            rows << [environment_name, environment_owner]+`#{command}`.split("\n")
+          end
+        }
+      }
+      Inform.info "\n#{rows.empty? ? "None" : table(%w(env owner)+STATUS_KEYS, *rows)}"
+    end
+  end
+
+  subcommand "ssh", "SSH to an environment" do
+    parameter "ENVIRONMENT_NAME", "Name of the environment"
+    option ['-i', '--instance'], 'INSTANCE_NUMBER', "Which instance to SSH to", :default => 1 do |s|
+      Integer(s)
+    end
+    option ['-p', '--print'], :flag, "Print the SSH credentials instead of actually performing the SSH operation", :default => false
+
+    def execute
+      server = get_server(environment_name, instance)
+      if server.credentials
+        if print?
+          puts server.credentials
+        else
+          command = "ssh #{server.credentials}"
+          Inform.debug("Running %{command}", :command => command)
+          system command
+        end
+      end
+    end
+  end
+
+  subcommand "destroy", "Destroy an existing environment" do
+
+    option ['-f', '--force'], :flag, "Don't ask for confirmation before destroying", :default => false
+    parameter "ENVIRONMENT_NAME", "Name of the environment to be destroyed"
+
+    def execute
+      controller.destroy(environment_name, :force => force?)
+    end
+
+  end
+
+  def get_server(environment_name, instance_no)
+    env = Environment.get(environment_name)
+    server = env.servers[instance_no - 1]
+    raise "Environment only has #{env.servers.length} instances, can't SSH to instance ##{instance}" unless server
+    server
+  end
+
+end

data/lib/dew/commands/tidy.rb

@@ -0,0 +1,35 @@
+class TidyCommand < Clamp::Command
+
+  option ['--[no-]clean-environments'], :flag, "Clean up environments", :default => true
+  option ['--[no-]clean-amis'], :flag, "Clean up AMIs", :default => true
+  option ['--noop'], :flag, "Print out what we'd do instead of doing it", :default => false
+  
+  def tidy_environments
+    Inform.info("Tidying up Environments...")
+    names = Cloud.valid_servers.collect(&:tags).collect { |h| h['Environment'] if h['Environment'] }
+    names << Cloud.rds.servers.all.select {|a| a.state == 'available'}.map(&:id)
+    names = names.flatten.uniq
+    names = names.grep /^cuke-/
+    names.each do |name|
+      Inform.info("Destroying environment #{name}")
+      Environment.get(name).destroy unless noop?
+    end
+  end
+ 
+  def tidy_amis
+    Inform.info("Tidying up AMIS...")
+    amis = Cloud.compute.images.all
+    amis = amis.select {|a| a.name =~ /^cuke-/}
+    amis.each do |ami|
+      Inform.info("Destroying ami #{ami.name}") do
+        ami.deregister unless noop?
+      end
+    end
+  end
+    
+  def execute
+    Inform.warning("--noop passed, no changes will be made!") if noop?
+    tidy_environments if clean_environments?
+    tidy_amis if clean_amis?
+  end
+end

data/lib/dew/controllers.rb

@@ -0,0 +1,3 @@
+require 'dew/controllers/amis_controller'
+require 'dew/controllers/deploy_controller'
+require 'dew/controllers/environments_controller'

data/lib/dew/controllers/amis_controller.rb

@@ -0,0 +1,82 @@
+require 'dew/controllers/environments_controller'
+
+class AMIsController
+
+  AMI_PROFILE = 'ami-prototype'
+
+  def create ami_name, puppet_node_name
+    Inform.info("Creating new AMI %{ami_name} using puppet node %{puppet}", :ami_name => ami_name, :puppet => puppet_node_name)
+    environment_name = ami_name + '-prototype-' + $$.to_s
+
+    environment = Environment.create(environment_name, Profile.read(AMI_PROFILE))
+    @prototype = environment.servers.first
+    Inform.debug("Using server %{id} at %{ip} as our prototype.", :id => @prototype.id, :ip => @prototype.public_ip_address)
+
+    Inform.debug("Installing puppet...")
+    install_puppet_on_prototype
+    Inform.debug("Copying puppet configuration... ")
+    copy_puppet_to_prototype
+    Inform.debug("Running puppet node %{node}... ", :node => puppet_node_name)
+    run_puppet_node_on_prototype puppet_node_name
+
+    ami_id = Inform.info "Creating new ami with name %{ami_name}", :ami_name => ami_name do
+      @prototype.create_ami ami_name
+    end
+    Inform.info("New AMI id is %{ami_id}", :ami_id => ami_id)
+    environment.destroy
+  end
+
+  def index
+    # Inform.info("AMIs:\n#{Cloud.compute.images.all('owner_id' => Cloud.account.aws_user_id)}")
+    # /home/chris/.rvm/gems/ruby-1.9.2-p180@AWS/gems/excon-0.6.3/lib/excon/connection.rb:179:in `request': InvalidParameterValue => The filter 'owner_id' is invalid (Fog::Service::Error)
+    my_amis = Cloud.compute.images.all.select { |x| x.owner_id == Cloud.account.aws_user_id }
+    keys = %w(name id state architecture kernel_id description)
+    Inform.info(View.new('My AMIs', my_amis, keys).index)
+  end
+
+  def show ami_name
+    my_amis = Cloud.compute.images.all('name' => ami_name)
+    raise "AMI named #{ami_name} not found!" if my_amis.empty?
+    keys = %w(id architecture block_device_mapping description location owner_id state type is_public kernel_id platform product_codes ramdisk_id root_device_type root_device_name tags name)
+    Inform.info(View.new('My AMIs', my_amis, keys).show(0))
+  end
+
+  def destroy ami_name, opts={}
+    ami = Cloud.compute.images.all('name' => ami_name).first
+    raise "AMI named #{ami_name} not found!" unless ami
+    if opts[:force] || agree("<%= color('Are you sure?', YELLOW, BOLD) %> ")
+      Inform.info("Destroying AMI named %{n}", :n => ami_name) do
+        ami.deregister
+      end
+    else
+      Inform.info "Aborting AMI destruction"
+    end
+  end
+
+  private
+
+  def ssh
+    @ssh ||= @prototype.ssh
+  end
+
+  def install_puppet_on_prototype
+    Inform.info("Installing puppet") do
+      ssh.run('sudo apt-get update', :quiet_stderr => true)
+      ssh.run('sudo apt-get -q -y install puppet', :quiet_stderr => true) # chatty
+    end
+  end
+
+  def copy_puppet_to_prototype
+    Inform.info("Uploading puppet configuration") do
+      ssh.upload(File.join(ENV['HOME'], '.dew', 'puppet'), '/tmp/puppet')
+      ssh.run("sudo rm -rf /etc/puppet")
+      ssh.run("sudo mv /tmp/puppet /etc/puppet")
+    end
+  end
+
+  def run_puppet_node_on_prototype puppet_node_name
+    Inform.info("Running puppet node %{name} (this may take a while)", :name => puppet_node_name) do
+      ssh.run("sudo puppet /etc/puppet/manifests/nodes/#{puppet_node_name}.pp #{puppet_node_name} > /tmp/puppet_run_log 2>&1")
+    end
+  end
+end

data/lib/dew/controllers/deploy_controller.rb

@@ -0,0 +1,10 @@
+class DeployController
+  
+  def create deploy_type, environment_name, opts
+    if Environment.get(environment_name).servers.empty?
+      raise "Environment #{environment_name.inspect} doesn't exist or appears to have all instances already terminated"
+    end
+
+    Deploy::Run.new(deploy_type, Environment.get(environment_name), opts).deploy
+  end
+end

data/lib/dew/controllers/environments_controller.rb

@@ -0,0 +1,48 @@
+require 'dew/models/profile'
+
+class EnvironmentsController
+
+  def create(name, profile_name, opts={})
+    profile = Profile.read(profile_name)
+    if opts[:force] || (
+      Inform.info("About to create environment %{name} using the following profile:\n%{profile}" , :name => name, :profile => profile.to_s)
+      agree("<%= color('Do you wish to continue?', YELLOW, BOLD) %> ")
+    )
+      environment = Environment.create(name, profile)
+      environment.show
+      environment
+    else
+      Inform.info "Aborting environment creation"
+    end
+  end
+
+  def index
+    Environment.index
+  end
+
+  def show(name)
+    before_get_environment name
+
+    @environment.show
+  end
+
+  def destroy name, opts={}
+    before_get_environment name
+
+    @environment.show
+    Inform.info "Destroying environment %{name} ...", :name => name
+    if opts[:force] || agree("<%= color('Are you sure?', YELLOW, BOLD) %> ")
+      @environment.destroy
+      Inform.info "Environment %{name} destroyed", :name => name
+    else
+      Inform.info "Aborting environment destruction"
+    end
+  end
+
+  # a rough before filter
+  def before_get_environment(name)
+    @environment = Environment.get(name)
+    raise "Environment named #{name} not found!" unless @environment
+  end
+
+end

data/lib/dew/models.rb

@@ -0,0 +1,7 @@
+require 'fog'
+require 'dew/models/account'
+require 'dew/models/fog_model'
+require 'dew/models/deploy'
+require 'dew/models/server'
+require 'dew/models/database'
+require 'dew/models/environment'

data/lib/dew/models/account.rb

@@ -0,0 +1,30 @@
+require 'yaml'
+
+class Account
+
+  def self.read(account_name)
+    Account.new YAML.load File.read File.join [ ENV['HOME'], '.dew', 'accounts', "#{account_name}.yaml" ]
+  end
+
+  def self.user_ids
+    Dir[File.join(ENV['HOME'], '.dew', 'accounts', '*.yaml')].map do |filename|
+      Account.read(File.basename(filename).gsub(/.yaml$/, '')).aws_user_id
+    end
+  end
+
+  def aws_access_key_id
+    @yaml['aws']['access_key_id']
+  end
+
+  def aws_secret_access_key
+    @yaml['aws']['secret_access_key']
+  end
+
+  def aws_user_id
+    @yaml['aws']['user_id'].gsub('-', '')
+  end
+
+  def initialize(yaml)
+    @yaml = yaml
+  end
+end

data/lib/dew/models/database.rb

@@ -0,0 +1,32 @@
+class Database < FogModel
+
+  def self.create! name, size, password
+    new Cloud.rds.servers.create(
+              :engine => 'MySQL',
+              :master_username => 'root',
+              :password => password,
+              :id => name,
+              :allocated_storage => '5',
+              :flavor_id => size
+    )
+  end
+
+  def self.get name
+    db = Cloud.rds.servers.get(name)
+    new db if db
+  end
+  
+  def public_address
+    fog_object.endpoint['Address']
+  end
+  
+  def db_environment_file password
+    <<-EOF
+PUGE_DB_HOST=#{fog_object.endpoint['Address']}
+PUGE_DB_NAME=#{id}
+PUGE_DB_USERNAME=#{master_username}
+PUGE_DB_PASSWORD=#{password}
+    EOF
+  end
+  
+end

data/lib/dew/models/deploy.rb

@@ -0,0 +1,2 @@
+require 'dew/models/deploy/puge'
+require 'dew/models/deploy/run'

data/lib/dew/models/deploy/puge.rb

@@ -0,0 +1,61 @@
+module Deploy
+
+  class Puge
+
+    def initialize servers, opts
+      @servers = servers
+      @opts = opts
+    end
+
+    def deploy
+      execute_in_parallel_and_wait Proc.new { |server|
+        Inform.info("%{server_id}: Cloning PUGE and checking out tag %{tag}", :server_id => server.id, :tag => @opts['tag'])
+        upload_and_run(server, 'clone_puge.sh', @opts['tag'])
+
+        Inform.info("%{server_id}: Running bundle install", :server_id => server.id)
+        upload_and_run(server, 'bundle_install.sh')
+      }
+
+      # This task cannot run in parallel as there's only one RDS
+      #
+      Inform.info("%{server_id}: Setting up Rails database using %{rails_env} Rails environment", :server_id => @servers.first.id, :rails_env => @opts['rails_env'])
+      upload_and_run(@servers.first, 'setup_rails_database.sh', @opts['rails_env'])
+
+      execute_in_parallel_and_wait Proc.new { |server|
+        Inform.info("%{server_id}: Generating PUGE WAR", :server_id => server.id)
+        upload_and_run(server, 'generate_puge_war.sh', @opts['rails_env'])
+
+        Inform.info("%{server_id}: Copying PUGE WAR into Tomcat directory", :server_id => server.id)
+        upload_and_run(server, 'copy_puge_war_into_tomcat.sh')
+
+        Inform.info("%{server_id}: Restarting Tomcat", :server_id => server.id)
+        upload_and_run(server, 'restart_tomcat.sh')
+      }
+    end
+
+    private
+
+    def execute_in_parallel_and_wait proc
+      threads = []
+
+      @servers.each do |server|
+        threads << Thread.new { proc.call(server) }
+      end
+
+      # wait for all threads to finish
+      #
+      threads.each do |thread|
+        thread.join
+      end
+    end
+
+    def upload_script server, script
+      server.ssh.upload(File.join(ENV['HOME'], '.dew', 'deploy', 'puge', script), '.')
+    end
+    
+    def upload_and_run server, script, *args
+      upload_script(server, script)
+      server.ssh.run(['./' + script, args.map {|a| "'#{a}'"}].flatten.join(" "))
+    end
+  end
+end