slugforge 4.0.0

data/lib/slugforge/models/host.rb

@@ -0,0 +1,233 @@
+require 'net/ssh'
+require 'net/scp'
+
+module Slugforge
+  class Host
+    attr_reader :pattern, :server, :slug_name, :status
+
+    def initialize(pattern, server=nil)
+      @pattern = pattern
+      @server  = server
+      @deploy_results  = []
+      @timeline = []
+      @start_time = Time.now
+      @actions = []
+      self
+    end
+
+    def name
+      "name:#{@pattern}"
+    end
+
+    def ip
+      @pattern
+    end
+
+    def ssh_host
+      @pattern
+    end
+
+    def id
+      nil
+    end
+
+    def is_autoscaled?
+      false
+    end
+
+    def to_status
+      {
+        :name       => name,
+        :ip         => ip,
+        :pattern    => @pattern,
+        :slug_name  => @slug_name,
+        :status     => @status.to_s,
+        :output     => @deploy_results,
+        :start_time => @start_time,
+        :timeline   => timeline,
+      }
+    end
+
+    def elapsed_time
+      Time.at(Time.now - @start_time).strftime('%M:%S')
+    end
+
+    def record_event(status)
+      @status = status
+      @timeline << {:status => status, :elapsed_time => elapsed_time}
+    end
+
+    def timeline
+      @timeline.map { |event| "#{event[:status]} @ #{event[:elapsed_time]}" }.join ', '
+    end
+
+    def complete?
+      success? || failed?
+    end
+
+    def success?
+      [:deployed, :terminated].include?(@status) && output.empty?
+    end
+
+    def failed?
+      @status == :failed?
+    end
+
+    def add_action(action)
+      @actions << action
+    end
+
+    def has_action?(action)
+      @actions.include?(action)
+    end
+
+    def remove_action(action)
+      @actions.delete(action)
+    end
+
+    def stage?
+      @actions.include?(:stage)
+    end
+
+    def install?
+      @actions.include?(:install)
+    end
+
+    def effective_action
+      return "installing" if @actions.include?(:install)
+      "staging"
+    end
+
+    def terminated?
+      @status == :terminated?
+    end
+
+    def output
+      @deploy_results.map { |result| result[:output] unless result[:exit_code] == 0 }.compact
+    end
+
+    def deploy(slug_name, logger, opts)
+      begin
+        record_event :started
+        if opts[:pretend]
+          logger.log "not actually #{effective_action} slug (#{name})", {:color => :yellow, :status => :pretend}
+        else
+          logger.say_status :deploy, "#{effective_action} to host #{name} as #{username(opts)}", :green
+          Net::SSH.start(ssh_host, username(opts), ssh_opts(opts)) do |ssh|
+            host_slug = detect_slug(ssh, slug_name, logger) unless opts[:force]
+            host_slug ||= copy_slug(ssh, slug_name, logger, opts)
+            explode_slug(ssh, host_slug, logger, opts) if stage?
+            install_slug(ssh, host_slug, logger, opts) if install?
+          end
+        end
+        record_event :deployed
+      rescue => e
+        record_event :failed
+        message = "#{e.class.to_s}: #{e.to_s}"
+        logger.log "#{message} (#{ip}: #{name})", {:color => :red, :status => :fail}
+        @deploy_results << {:output => message}
+      end
+      logger.say_status :deploy, "#{effective_action} complete for host: #{name}", success? ? :green : :red
+      @deploy_results
+    end
+
+    private
+    def ssh_opts(opts = {})
+      ssh_opts = { :forward_agent => true }
+      if opts[:identity]
+        ssh_opts[:key_data] = File.read(opts[:identity])
+        ssh_opts[:keys_only] = true
+      end
+      ssh_opts
+    end
+
+    def detect_slug(ssh, slug_name, logger)
+      found_path = ['/tmp', '/mnt'].select do |path|
+        file_count(ssh, path, slug_name) > 0
+      end.compact
+      return nil if found_path.empty?
+      slug_name_with_path = "#{found_path.first}/#{slug_name}"
+      logger.log "found existing slug (#{slug_name_with_path}) on host (#{name}); use --force to overwrite slug", {:color => :yellow, :status => :detect, :log_level => :vervose}
+      record_event :detected
+      slug_name_with_path
+    end
+
+    def copy_slug(ssh, slug_name, logger, opts)
+      slug_name_with_path = "/mnt/#{slug_name}"
+      case opts[:copy_type]
+      when :ssh
+        logger.log "interactive mode enabled (be sure to set slug_name)", {:color => :yellow, :status => :copy, :log_level => :verbose}
+        binding.pry
+      when :scp
+        logger.log "copying slug to host via SCP (#{name})", {:color => :green, :status => :copy, :log_level => :verbose}
+        scp_upload ip, username(opts), opts[:filename], "#{slug_name}", logger, :ssh => ssh_opts
+        logger.log "moving slug from ~ to /mnt as root", {:color => :green, :status => :copy, :log_level => :verbose}
+        @deploy_results << ssh_command(ssh, "sudo mv #{slug_name} #{slug_name_with_path}", logger)
+      else # AWS S3 command line by default
+        logger.log "copying slug to host via aws s3 command (#{name})", {:color => :green, :status => :copy, :log_level => :verbose}
+        @deploy_results << ssh_command(ssh, "sudo -- sh -c 'export AWS_ACCESS_KEY_ID=#{opts[:aws_session][:aws_access_key_id]}; export AWS_SECRET_ACCESS_KEY=#{opts[:aws_session][:aws_secret_access_key]}; export AWS_SECURITY_TOKEN=#{opts[:aws_session][:aws_session_token]}; export AWS_DEFAULT_REGION=#{opts[:aws_session][:aws_region]}; aws s3 cp #{opts[:s3_url]} #{slug_name_with_path}'; echo \"SSH_COMMAND_EXIT_CODE=$?\"", logger)
+      end
+      record_event :copied
+      slug_name_with_path
+    end
+
+    def username(opts)
+      opts[:username] || Net::SSH.configuration_for(ip)[:user] || `whoami`.chomp
+    end
+
+    def explode_slug(ssh, slug_name_with_path, logger, opts)
+      logger.log "exploding package as root #{"for user " + opts[:owner] if opts[:owner]}", {:color => :green, :status => :stage, :log_level => :verbose}
+      @deploy_results << ssh_command(ssh, slug_install_command(slug_name_with_path, opts[:deploy_dir], {:unpack => true, :owner => opts[:owner], :env => opts[:env]}), logger)
+      @slug_name = slug_name
+    end
+
+    def install_slug(ssh, slug_name_with_path, logger, opts)
+      logger.log "installing package as root #{"for user " + opts[:owner] if opts[:owner]}", {:color => :green, :status => :install, :log_level => :verbose}
+      @deploy_results << ssh_command(ssh, slug_install_command(slug_name_with_path, opts[:deploy_dir], {:owner => opts[:owner], :env => opts[:env], :force => opts[:force]}), logger)
+      @slug_name = slug_name
+      record_event :installed
+    end
+
+    def file_count(ssh, path, file)
+      ssh.exec!("find #{path} -maxdepth 1 -name '#{file}' -type f -size +0 | wc -l").to_i
+    end
+
+    def scp_upload(host, user, source, dest, logger, opts)
+      logger.log "SCP: #{source} to #{host}:#{dest}"
+      Net::SCP.upload!(host, user, source, dest, opts) do | ch, name, sent, total |
+        logger.log "\r#{name}: #{(sent * 100.0 / total).to_i}% "
+      end
+      logger.log
+    end
+
+    def ssh_command(ssh, command, logger)
+      output = ssh.exec!("#{command}")
+      exit_code_matches = /^SSH_COMMAND_EXIT_CODE=(\d+)$/.match(output)
+      exit_code = exit_code_matches ? exit_code_matches[1].to_i : 0
+      logger_opts = if exit_code == 0
+                      {:color => :green, :log_level => :verbose}
+                    else
+                      {:color => :red}
+                    end.merge({:status => :command})
+      logger.log "#{command}", logger_opts
+      logger.log "Output:\n#{output}", logger_opts
+      {:exit_code => exit_code, :output => output, :command => command, :username => ssh.options[:user]}
+    end
+
+    def slug_install_command(slug_name_with_path, deploy_dir, opts = {})
+      [ opts[:prefix],
+        "TERM=dumb sudo bash -l -c 'date >> /var/log/slug_deploy.log ; ",
+        "chmod +x #{slug_name_with_path} ",
+        "&& #{opts[:env]} #{slug_name_with_path} ",
+        '-y ', #always clobber existing installs
+        "-i #{deploy_dir} ",
+        opts[:owner] ? "-o #{opts[:owner]} " : '',
+        opts[:force] ? '-f ' : '',
+        opts[:unpack] ? '-u ' : '',
+        '-v ', #verbose
+        '| tee -a /var/log/slug_deploy.log ',
+        "; echo \"SSH_COMMAND_EXIT_CODE=$?\"'"
+      ].join('')
+    end
+  end
+end

data/lib/slugforge/models/host/fog_host.rb

@@ -0,0 +1,33 @@
+require 'slugforge/models/host'
+
+module Slugforge
+  class FogHost < Host
+    def name
+      "instance:#{@server.id}, private_name:#{@server.private_dns_name}, public_name:#{@server.dns_name}, ip:#{@server.public_ip_address}"
+    end
+
+    def ip
+      @server.public_ip_address
+    end
+
+    def ssh_host
+      @server.dns_name
+    end
+
+    def id
+      @server.id
+    end
+
+    def is_autoscaled?
+      !@server.tags["aws:autoscaling:groupName"].nil?
+    end
+
+    def to_status
+      super.merge({
+        :instance_id => @server.id,
+        :private_name => @server.private_dns_name,
+        :public_name => @server.dns_name,
+      })
+    end
+  end
+end

data/lib/slugforge/models/host/hostname_host.rb

@@ -0,0 +1,9 @@
+require 'slugforge/models/host'
+
+module Slugforge
+  class HostnameHost < Host
+    def name
+      "hostname:#{@pattern}"
+    end
+  end
+end

data/lib/slugforge/models/host/ip_address_host.rb

@@ -0,0 +1,9 @@
+require 'slugforge/models/host'
+
+module Slugforge
+  class IpAddressHost < Host
+    def name
+      "ip:#{@pattern}"
+    end
+  end
+end

data/lib/slugforge/models/host_group.rb

@@ -0,0 +1,65 @@
+require 'slugforge/models/host_group/ip_address_group'
+require 'slugforge/models/host_group/ec2_instance_group'
+require 'slugforge/models/host_group/hostname_group'
+require 'slugforge/models/host_group/aws_tag_group'
+require 'slugforge/models/host_group/security_group_group'
+
+module Slugforge
+  class HostGroup
+    attr_reader :name, :hosts
+
+    def self.discover(patterns, compute)
+      patterns.map do |pattern|
+        IpAddressGroup.detect(pattern, compute) ||
+        Ec2InstanceGroup.detect(pattern, compute) ||
+        HostnameGroup.detect(pattern, compute) ||
+        AwsTagGroup.detect(pattern, compute) ||
+        SecurityGroupGroup.detect(pattern, compute) ||
+        # If nothing detected, return a "null" group
+        HostGroup.new(pattern, compute)
+      end
+    end
+
+    def initialize(pattern, compute)
+      @name = pattern
+    end
+
+    def install_all
+      return if @hosts.nil?
+      @hosts.each { |host| host.add_action(:install) }
+    end
+
+    def install_percent_of_hosts(value)
+      return if @hosts.nil?
+      count = (@hosts.count * value / 100.0).ceil
+      sorted_hosts[0...count].each { |host| host.add_action(:install) }
+    end
+
+    def install_number_of_hosts(value)
+      return if @hosts.nil?
+      count = [@hosts.count, value].min
+      sorted_hosts[0...count].each { |host| host.add_action(:install) }
+    end
+
+    def sorted_hosts
+      # We sort the hosts by IP to make the order deterministic before we filter
+      # by number or percent. That way when we move from 5% to 10% we end up at
+      # 10% of the hosts, not some value between 10% and 15%.
+      @hosts.sort_by { |host| host.ip }
+    end
+
+    def success?
+      @hosts.all?(&:success?)
+    end
+
+    def hosts_for_action(action)
+      @hosts.select { |host| host.has_action?(action) }
+    end
+
+    def self.detect(pattern, compute)
+      return nil unless pattern =~ self.matcher
+      group = self.new(pattern, compute)
+      group.hosts.empty? ? nil : group
+    end
+  end
+end

data/lib/slugforge/models/host_group/aws_tag_group.rb

@@ -0,0 +1,22 @@
+require 'slugforge/models/host/fog_host'
+
+module Slugforge
+  class HostGroup ; end
+  
+  class AwsTagGroup < HostGroup
+    def self.matcher
+      /^(\w+)=(\w+)$/
+    end
+
+    def initialize(pattern, compute)
+      matches = self.class.matcher.match(pattern)
+      return nil unless matches
+      @hosts = compute.servers.select do |server|
+        server.tags[matches[1]] == matches[2] && !server.public_ip_address.nil?
+      end.map do |server|
+        FogHost.new(pattern, server)
+      end
+      super
+    end
+  end
+end

data/lib/slugforge/models/host_group/ec2_instance_group.rb

@@ -0,0 +1,21 @@
+require 'slugforge/models/host/fog_host'
+
+module Slugforge
+  class HostGroup ; end
+  
+  class Ec2InstanceGroup < HostGroup
+    def self.matcher
+      /^i-[0-9a-f]{8}$/i
+    end
+
+    def initialize(pattern, compute)
+      server = compute.servers.get(pattern)
+      @hosts = if server.nil? || server.public_ip_address.nil?
+                 []
+               else
+                 [ FogHost.new(pattern, server) ]
+               end
+      super
+    end
+  end
+end

data/lib/slugforge/models/host_group/hostname_group.rb

@@ -0,0 +1,16 @@
+require 'slugforge/models/host/hostname_host'
+
+module Slugforge
+  class HostGroup ; end
+  
+  class HostnameGroup < HostGroup
+    def self.matcher
+      /^[^.]+\./
+    end
+
+    def initialize(pattern, compute)
+      @hosts = [ HostnameHost.new(pattern) ]
+      super
+    end
+  end
+end

data/lib/slugforge/models/host_group/ip_address_group.rb

@@ -0,0 +1,16 @@
+require 'slugforge/models/host/ip_address_host'
+
+module Slugforge
+  class HostGroup ; end
+  
+  class IpAddressGroup < HostGroup
+    def self.matcher
+      /^(\d{1,3}\.){3}(\d{1,3})$/
+    end
+    
+    def initialize(pattern, compute)
+      @hosts = [ IpAddressHost.new(pattern) ]
+      super
+    end
+  end
+end

data/lib/slugforge/models/host_group/security_group_group.rb

@@ -0,0 +1,20 @@
+require 'slugforge/models/host/fog_host'
+
+module Slugforge
+  class HostGroup ; end
+  
+  class SecurityGroupGroup < HostGroup
+    def self.matcher
+      /\w+/
+    end
+    
+    def initialize(pattern, compute)
+      @hosts = compute.servers.select do |server|
+        server.groups.include?(pattern) && !server.public_ip_address.nil?
+      end.map do |server|
+        FogHost.new(pattern, server)
+      end
+      super
+    end
+  end
+end