standup 0.2.0

data/.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+.idea
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Ilia Ablamonov, Cloud Castle Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,8 @@
+= Standup
+
+Standup is an application deployment and infrastructure management tool for Rails and Amazon EC2.
+
+== Copyright
+
+Copyright (c) 2010 Ilia Ablamonov, Cloud Castle Inc.
+See LICENSE for details.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = 'standup'
+    gem.summary = %Q{Standup is an application deployment and infrastructure management tool for Rails and Amazon EC2}
+    gem.description = %Q{}
+    gem.email = 'ilia@flamefork.ru'
+    gem.homepage = 'http://github.com/Flamefork/standup'
+    gem.authors = ['Ilia Ablamonov', 'Cloud Castle Inc.']
+    gem.add_dependency 'activesupport', '>= 3.0'
+    gem.add_dependency 'settingslogic', '>= 2.0'
+    gem.add_dependency 'amazon-ec2', '>= 0.9'
+    gem.add_dependency 'aws-s3', '>= 0.5'
+    gem.add_dependency 'net-ssh', '>= 2.0'
+    gem.add_dependency 'highline', '>= 1.5.2'
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/lib/standup/core_ext.rb

@@ -0,0 +1,11 @@
+module Kernel
+  def bright_p message, color = ''
+    puts "\e[1m#{color}#{message}\e[0m"
+  end
+  
+  def bright_ask message, echo = true
+    require 'highline'
+    bright_p message, HighLine::GREEN
+    HighLine.new.ask('') {|q| q.echo = echo}
+  end
+end

data/lib/standup/ec2/base.rb

@@ -0,0 +1,61 @@
+module Standup
+  module EC2
+    class Base
+      def initialize info = false
+        case info
+          when Hash
+            set_info info
+          when true
+            load_info
+          when false
+            # nothing
+        end
+      end
+
+      def self.info_reader *names
+        names.each do |name|
+          class_eval "def #{name}; read_info_field :#{name}; end", __FILE__, __LINE__
+        end
+      end
+
+      def exists?
+        read_info_field :exists
+      end
+
+      def load_info; end
+
+      protected
+
+      def read_info_field name
+        load_info unless instance_variable_defined?(:"@#{name}")
+        instance_variable_set(:"@#{name}", nil) unless instance_variable_defined?(:"@#{name}")
+        @exists = true
+        instance_variable_get(:"@#{name}")
+      rescue AWS::InvalidGroupNotFound
+      rescue AWS::InvalidInstanceIDNotFound
+      rescue AWS::InvalidVolumeIDNotFound
+        @exists = false
+        nil
+      end
+
+      def set_info info
+        info.each do |key, value|
+          instance_variable_set :"@#{key}", value
+        end
+      end
+
+      def list
+        self.class.list
+      end
+
+      def api
+        self.class.api
+      end
+
+      def self.api
+        @@api ||= AWS::EC2::Base.new :access_key_id => Settings.aws.access_key_id,
+                                     :secret_access_key => Settings.aws.secret_access_key
+      end
+    end
+  end
+end

data/lib/standup/ec2/elastic_ip.rb

@@ -0,0 +1,49 @@
+module Standup
+  module EC2
+    class ElasticIP < Base
+      def initialize ip, info = false
+        @ip = ip
+        super info
+      end
+
+      info_reader :ip, :attached_to
+
+      def self.list reload = false
+        if !class_variable_defined?(:@@list) || reload
+          @@list = {}
+          result = api.describe_addresses
+          result.addressesSet.item.each do |item|
+            @@list[item.publicIp] = new item.publicIp, :attached_to => Instance.new(item.instanceId)
+          end if result.addressesSet
+        end
+        @@list
+      end
+
+      def self.create
+        ip = api.allocate_address.publicIp
+        list[ip] = ElasticIP.new ip
+      end
+
+      def destroy
+        api.release_address :public_ip => @ip
+        list.delete @ip
+      end
+
+      def attach_to instance
+        api.associate_address :instance_id => instance.id,
+                              :public_ip => @ip
+        @attached_to = instance
+      end
+
+      def detach
+        api.disassociate_address :public_ip => @ip
+        @attached_to = nil
+      end
+
+      def load_info
+        result = api.describe_addresses :public_ip => @ip
+        @attached_to = Instance.new(result.addressesSet.item[0].instanceId)
+      end
+    end
+  end
+end

data/lib/standup/ec2/instance.rb

@@ -0,0 +1,80 @@
+module Standup
+  module EC2
+    class Instance < Base
+      def initialize id, info = false
+        @id = id
+        super info
+      end
+
+      info_reader :id, :external_ip, :internal_ip, :state, :security_groups, :architecture
+
+      def self.list reload = false
+        if !class_variable_defined?(:@@list) || reload
+          @@list = {}
+          result = api.describe_instances
+          result.reservationSet.item.each do |ritem|
+            ritem.instancesSet.item.each do |item|
+              @@list[item.instanceId] = new item.instanceId, build_info(ritem, item)
+            end if ritem.instancesSet
+          end if result.reservationSet
+        end
+        @@list
+      end
+      
+      def self.group_running reload = false
+        result = {}
+        SecurityGroup.list(reload).each {|name, _| result[name] = []}
+        list(reload).each do |_, instance|
+          instance.security_groups.each do |sg|
+            result[sg.name] << instance
+          end unless [:terminated, :"shutting-down"].include?(instance.state)
+        end
+        result
+      end
+
+      def self.create image_id, instance_type, security_groups
+        response = api.run_instances :image_id => image_id,
+                                     :key_name => Settings.aws.keypair_name,
+                                     :instance_type => instance_type,
+                                     :security_group => security_groups.map(&:name),
+                                     :availability_zone => Settings.aws.availability_zone
+        id = response.instancesSet.item[0].instanceId
+        list[id] = Instance.new id
+      end
+
+      def terminate
+        api.terminate_instances :instance_id => @id
+      end
+
+      def reboot
+        api.reboot_instances :instance_id => @id
+      end
+      
+      def wait_until timeout = 300
+        sleeping = 0
+        while yield(self) && sleeping < timeout
+          sleeping += sleep 5
+          STDOUT.print '.'
+          STDOUT.flush
+          load_info
+        end
+        print "\n"
+      end
+
+      def load_info
+        ritem = api.describe_instances(:instance_id => @id).reservationSet.item[0]
+        set_info self.class.build_info(ritem, ritem.instancesSet.item[0])
+      end
+
+      protected
+
+      def self.build_info ritem, item
+        return :external_ip => item.ipAddress,
+               :internal_ip => item.privateIpAddress,
+               :state => item.instanceState.name.to_sym,
+               :architecture => item.architecture,
+               :security_groups => ritem.groupSet.item.map{|i| SecurityGroup.new(i.groupId)}
+      end
+    end
+  end
+end

data/lib/standup/ec2/security_group.rb

@@ -0,0 +1,95 @@
+module Standup
+  module EC2
+    class SecurityGroup < Base
+      IPRule = Struct.new(:ip, :protocol, :from_port, :to_port)
+
+      def initialize name, info = false
+        @name = name
+        super info
+      end
+
+      info_reader :name, :description, :rules
+
+      def self.list reload = false
+        if !class_variable_defined?(:@@list) || reload
+          @@list = {}
+          result = api.describe_security_groups
+          result.securityGroupInfo.item.each do |gitem|
+            @@list[gitem.groupName] = new gitem.groupName, build_info(gitem)
+          end if result.securityGroupInfo
+        end
+        @@list
+      end
+
+      def self.create name, description = name
+        api.create_security_group :group_name => name,
+                                  :group_description => description
+        list[name] = SecurityGroup.new name,
+                                       :description => description,
+                                       :rules => []
+      end
+
+      def delete
+        api.delete_security_group :group_name => @name
+        list.delete @name
+      end
+
+      def add_rule rule
+        api.authorize_security_group_ingress build_rules_opts(rule)
+        rules << rule
+      end
+
+      def remove_rule rule
+        api.revoke_security_group_ingress build_rules_opts(rule)
+        rules.delete rule
+      end
+
+      def load_info
+        result = api.describe_security_groups :group_name => [@name]
+        set_info self.class.build_info(result.securityGroupInfo.item[0])
+      end
+
+      def hash
+        @name.hash
+      end
+
+      def eql? other
+        @name == other.name
+      end
+
+      private
+
+      def build_rules_opts rule
+        case rule
+          when SecurityGroup
+            return :group_name => @name,
+                   :source_security_group_name => rule.name,
+                   :source_security_group_owner_id => Settings.aws.account_id
+          when IPRule
+            return :group_name => @name,
+                   :ip_protocol => rule.protocol,
+                   :from_port => rule.from_port,
+                   :to_port => rule.to_port,
+                   :cidr_ip => rule.ip
+        end
+      end
+
+      def self.build_info gitem
+        rules = Set.new
+        gitem.ipPermissions.item.each do |pitem|
+          rules << if pitem.groups
+            SecurityGroup.new pitem.groups.item[0].groupName
+          else
+            IPRule.new pitem.ipRanges.item[0].cidrIp,
+                       pitem.ipProtocol,
+                       pitem.fromPort,
+                       pitem.toPort
+          end
+        end if gitem.ipPermissions
+
+        return :description => gitem.groupDescription,
+               :rules => rules
+      end
+    end
+  end
+end

data/lib/standup/ec2/volume.rb

@@ -0,0 +1,57 @@
+module Standup
+  module EC2
+    class Volume < Base
+      def initialize id, info = false
+        @id = id
+        super info
+      end
+
+      info_reader :id, :attached_to
+
+      def self.list reload = false
+        if !class_variable_defined?(:@@list) || reload
+          @@list = {}
+          response = api.describe_volumes
+          response.volumeSet.item.each do |item|
+            instance = item.attachmentSet ? Instance.new(item.attachmentSet.item[0].instanceId) : nil
+            @@list[item.volumeId] = Volume.new item.volumeId,
+                                                :status => item.status.to_sym,
+                                                :attached_to => instance
+          end if response.volumeSet
+        end
+        @@list
+      end
+
+      def self.create size
+        response = api.create_volume :size => size.to_s,
+                                     :availability_zone => Settings.aws.availability_zone
+        list[response.volumeId] = Volume.new response.volumeId
+      end
+
+      def destroy
+        api.delete_volume :volume_id => @id
+        list.delete @id
+      end
+
+      def attach_to instance, device
+        api.attach_volume :volume_id => @id,
+                          :instance_id => instance.id,
+                          :device => device
+        @attached_to = instance
+      end
+
+      def detach
+        api.detach_volume :volume_id => @id,
+                          :force => 'true'
+        @attached_to = nil
+      end
+
+      def load_info
+        response = api.describe_volumes :volume_id => @id
+        item = response.volumeSet.item[0]
+        @status = item.status.to_sym
+        @attached_to = item.attachmentSet ? Instance.new(item.attachmentSet.item[0].instanceId) : nil
+      end
+    end
+  end
+end

data/lib/standup/ec2.rb

@@ -0,0 +1,5 @@
+require 'standup/ec2/base'
+require 'standup/ec2/instance'
+require 'standup/ec2/security_group'
+require 'standup/ec2/elastic_ip'
+require 'standup/ec2/volume'

data/lib/standup/node.rb

@@ -0,0 +1,48 @@
+module Standup
+  class Node
+    def initialize name
+      @name = name
+      
+      @scripts = ActiveSupport::HashWithIndifferentAccess.new
+      Standup.scripts.each do |sname, script|
+        @scripts[sname] = script.new self
+      end
+      @remoting = nil
+    end
+    
+    attr_reader :name, :scripts
+    
+    def run_script script_name
+      scripts[script_name].titled_run
+      close_remoting
+    end
+    
+    def instance
+      @instance ||= EC2::Instance.group_running[id_group].try(:first)
+    end
+    
+    def ssh_string
+      return '' unless instance
+      "ssh -i #{Settings.aws.keypair_file} -q -o StrictHostKeyChecking=no #{params.ec2.ssh_user}@#{instance.external_ip}"
+    end
+    
+    def params
+      Settings.nodes[@name]
+    end
+    
+    def remoting
+      @remoting ||= Remoting.new self
+    end
+    
+    def id_group
+      "standup_node_#{@name}"
+    end
+    
+    protected
+    
+    def close_remoting
+      @remoting.close if @remoting
+      @remoting = nil
+    end
+  end
+end

data/lib/standup/railtie.rb

@@ -0,0 +1,10 @@
+require 'standup'
+require 'rails'
+
+module Standup
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'tasks/standup.rake'
+    end
+  end
+end

data/lib/standup/remoting.rb

@@ -0,0 +1,133 @@
+module Standup
+  class Remoting
+    def initialize node
+      @node = node
+      @host = @node.instance.external_ip
+      @keypair_file = Settings.aws.keypair_file
+      @user = @node.params.ec2.ssh_user
+      @ssh = nil
+      @path = nil
+    end
+    
+    def download *files
+      options = files.pop
+      rsync wrap_to_remote(files), options[:to], options[:sudo]
+    end
+    
+    def upload *files
+      options = files.pop
+      rsync files, wrap_to_remote(options[:to]), options[:sudo]
+    end
+    
+    def remote_update file, body, opts = {}
+      tmpfile = Tempfile.new('file')
+  
+      download file,
+               :to => tmpfile.path,
+               :sudo => opts[:sudo]
+  
+      opts[:delimiter] ||= '# standup remote_update fragment'
+      
+      initial = File.read(tmpfile.path)
+      
+      if initial.empty?
+        bright_p "error during file upload. skipping", Highline::RED
+        return
+      end
+      
+      to_change = "#{opts[:delimiter]}\n#{body}#{opts[:delimiter]}\n"
+      changed = initial.gsub /#{opts[:delimiter]}.*#{opts[:delimiter]}\n?/m, to_change
+      
+      File.open(tmpfile.path, 'w') {|f| f.write changed}
+      
+      upload tmpfile.path,
+             :to => file,
+             :sudo => opts[:sudo]
+    end
+    
+    def in_dir path
+      raise ArgumentError, 'Only absolute paths allowed' unless path[0,1] == '/'
+      old_path = @path
+      @path = path
+      result = yield
+      @path = old_path
+      result
+    end
+    
+    def exec command
+      command  = @path ? "cd #{@path} && #{command}" : command
+      bright_p command
+      ssh.exec! command do |ch, _, data|
+        ch[:result] ||= ""
+        ch[:result] << data
+        print data
+        STDOUT.flush
+      end
+    end
+      
+    def sudo command
+      exec "sudo #{command}"
+    end
+      
+    def in_temp_dir &block
+      tmp_dirname = "/tmp/standup_tmp_#{rand 10000}"
+      exec "mkdir #{tmp_dirname}"
+      result = in_dir tmp_dirname, &block
+      exec "rm -rf #{tmp_dirname}"
+      result
+    end
+      
+    def file_exists? path
+      exec("if [ -e #{path} ]; then echo 'true'; fi") == "true\n"
+    end
+      
+    def install_packages *packages
+      packages = [*packages].flatten.join(' ')
+      sudo "apt-get -qqy install #{packages}"
+    end
+    alias :install_package :install_packages
+      
+    def install_gem name, version = nil
+      if version
+        unless exec("gem list | grep #{name}").try(:[], version)
+          sudo "gem install #{name} -v #{version} --no-ri --no-rdoc"
+        end
+      else
+        sudo "gem install #{name} --no-ri --no-rdoc"
+      end
+    end
+    
+    def close
+      @ssh.close if @ssh
+      @ssh = nil
+    end
+    
+    protected
+
+    def ssh
+      @ssh ||= Net::SSH.start @host, @user,
+                              :keys => @keypair_file,
+                              :paranoid => false,
+                              :timeout => 10
+    end
+    
+    def rsync source, destination, sudo
+      command = [
+        'rsync -azP --delete',
+        "-e 'ssh -i #{@keypair_file} -q -o StrictHostKeyChecking=no'",
+        ("--rsync-path='sudo rsync'" if sudo),
+        [*source].join(' '),
+        destination
+      ].compact.join(' ')
+      
+      3.times do
+        bright_p command
+        break if system command
+      end
+    end
+    
+    def wrap_to_remote files
+      [*files].map{|f| "#{@user}@#{@host}:#{f}"}.join(' ')
+    end
+  end
+end

data/lib/standup/scripts/base.rb

@@ -0,0 +1,52 @@
+require 'active_support/hash_with_indifferent_access'
+
+module Standup
+  module Scripts
+    class Base
+      def initialize node
+        @node = node
+        @remoting = nil
+        @params = if node.params[name].is_a? Hash
+          ActiveSupport::HashWithIndifferentAccess.new self.class.default_params.merge(node.params[name])
+        else
+          node.params[name] || self.class.default_params
+        end
+      end
+    
+      class_attribute :name
+      
+      class_attribute :default_params
+      self.default_params = {}
+      
+      class_attribute :description
+      
+      delegate :instance, :open_port, :open_ports, :remoting, :scripts,
+               :to => :@node
+    
+      delegate :download, :upload, :remote_update, :exec, :sudo, :in_dir, :in_temp_dir, :file_exists?, :install_package, :install_packages, :install_gem,
+               :to => :remoting
+    
+      attr_accessor :node, :params
+      
+      def name
+        self.class.name
+      end
+      
+      def titled_run
+        bright_p "#{@node.name}:#{name}", HighLine::CYAN
+        run
+      end
+      
+      def script_file filename
+        [Standup.local_scripts_path, Standup.gem_scripts_path].each do |dir|
+          next unless dir
+          path = File.expand_path("#{name}/#{filename}", dir)
+          return path if File.exists? path
+        end
+        nil
+      end
+      
+      def run; end
+    end
+  end
+end

data/lib/standup/settings.rb

@@ -0,0 +1,16 @@
+module Standup
+  begin
+    class Settings < Settingslogic
+      source 'config/standup.yml'
+      load!
+  
+      aws['account_id'].gsub!(/\D/, '')
+      # keypair_file default to ~/.ssh/keypair_name.pem
+      aws['keypair_file'] ||= "#{File.expand_path '~'}/.ssh/#{aws.keypair_name}.pem"
+    end
+  rescue 
+    require 'active_support/hash_with_indifferent_access'
+    remove_const :Settings
+    const_set :Settings, ActiveSupport::HashWithIndifferentAccess.new('nodes' => {})
+  end
+end