rudy 0.2.4 → 0.3.0

data/lib/rudy/command/disks.rb

@@ -4,6 +4,87 @@ module Rudy
   module Command
     class Disks < Rudy::Command::Base
       
+      def print_backups
+        criteria = [@zone]
+        criteria += [@environment, @role] unless @all
+        
+        Rudy::MetaData::Backup.list(@sdb, *criteria).each do |backup|
+          puts "%s (%s)" % [backup.name, backup.awsid]
+        end
+      end
+      
+      def destroy_backup(name)
+        puts "Destroying #{name}"
+        begin
+          backup = Rudy::MetaData::Backup.get(@sdb, name)
+        rescue => ex
+          puts "Error deleteing backup: #{ex.message}"
+        end
+        
+        return unless backup
+        
+        begin
+          puts " -> deleting snapshot..."
+          @ec2.snapshots.destroy(backup.awsid)
+        rescue => ex
+          puts "Error deleting snapshot: #{ex.message}."
+          puts "Continuing..."
+        ensure
+          puts " -> deleting metadata..."
+          @sdb.destroy(RUDY_DOMAIN, name)
+        end
+        puts "Done."
+      end
+      
+      def create_backup
+        machine = find_current_machine
+        disks = Rudy::MetaData::Disk.list(@sdb, machine[:aws_availability_zone], @environment, @role, @position)
+        volumes = @ec2.instances.volumes(machine[:aws_instance_id])
+        
+        puts "Machine: #{machine_name}"
+        
+        raise "The machine #{machine_name} does not have any disk metadata" if disks.empty?
+        raise "The machine #{machine_name} does not have any volumes attached." if volumes.empty?
+        
+        puts "#{disks.size} Disk(s) defined with #{volumes.size} Volume(s) running"
+        
+        volumes.each do |volume|
+          print "Volume #{volume[:aws_id]}... "
+          disk = Rudy::MetaData::Disk.from_volume(@sdb, volume[:aws_id])
+          backup = Rudy::MetaData::Backup.new
+          
+          # TODO: Look for the disk based on the machine
+          raise "No disk associated to volume #{volume[:aws_id]}" unless disk 
+          
+          backup.time_stamp
+          backup.volume = volume[:aws_id]
+          
+          # Populate machine infos
+          [:zone, :environment, :role, :position].each do |n|
+            val = instance_variable_get("@#{n}")
+            backup.send("#{n}=", val) if val
+          end
+          
+          # Populate disk infos
+          [:path, :size].each do |n|
+            backup.send("#{n}=", disk.send(n)) if disk.send(n)
+          end
+          
+          snap = @ec2.snapshots.create(volume[:aws_id])
+          
+          if !snap || !snap.is_a?(Hash)
+            puts "There was an unknown problem creating #{backup.name}. Continuing..."
+            next
+          end
+          
+          backup.awsid = snap[:aws_id]
+          
+          Rudy::MetaData::Backup.save(@sdb, backup)
+          
+          puts backup.name
+        end
+      end
+      
       
       def create_disk
         disk = Rudy::MetaData::Disk.new
@@ -26,17 +107,43 @@ module Rudy
         criteria += [@environment, @role] unless @all
         
         Rudy::MetaData::Disk.list(@sdb, *criteria).each do |disk|
-          print_disk disk
+          backups = Rudy::MetaData::Backup.for_disk(@sdb, disk, 2)
+          print_disk(disk, backups)
+          
         end
 
       end
       
+      def unattach_disk(name)
+        puts "Looking for #{name}"
+        disk = Rudy::MetaData::Disk.get(@sdb, name)
+        instances = @ec2.instances.list(machine_group)
+        @user = "root"
+        check_keys
+        raise "That is not a valid disk" unless disk
+        raise "There are no instances running in #{machine_group}" if !instances || instances.empty?
+        raise "The disk has no attached volume " unless disk.awsid
+        
+        machine = instances.values.first
+        
+        puts "Unmounting #{disk.path}..."
+        ssh machine[:dns_name], keypairpath, user, "umount #{disk.path}"
+        sleep 1
+        
+        puts "Detaching #{disk.awsid}"
+        @ec2.volumes.detach(disk.awsid)
+        
+        Rudy::MetaData::Backup.for_disk(@sdb, disk, 2)
+        
+        puts "Done!"
+      end
+      
       def attach_disk(name)
         puts "Looking for #{name}"
         disk = Rudy::MetaData::Disk.get(@sdb, name)
         instances = @ec2.instances.list(machine_group)
         raise "There are no instances running in #{machine_group}" if !instances || instances.empty?
-        instance_id = instances.keys.first
+        instance_id = instances.keys.first # <--- TODO: This is bad!
         machine = instances.values.first
         
         do_dirty_disk_volume_deeds(disk, machine)

data/lib/rudy/command/environment.rb

@@ -14,29 +14,21 @@ module Rudy
   module Command
     class Environment < Rudy::Command::Base
       
-      # Returns a hash of info for the requested machine. If the requested machine
-      # is not running, it will raise an exception. 
-      def find_requested_machine
-        machine_list = @ec2.instances.list(machine_group)
-        machine = machine_list.values.first  # NOTE: Only one machine per group, for now...
-        raise "There's no machine running in #{machine_group}" unless machine
-        raise "The primary machine in #{machine_group} is not in a running state" unless machine[:aws_state] == 'running'
 
-        machine
-      end
-      
       def connect
         check_keys
-        machine = find_requested_machine
+        machine = find_current_machine
         ssh machine[:dns_name], keypairpath, user, false, false, false, @print
       end
       
       # +paths+ an array of paths to copy. The last element is the "to" path. 
       def copy(paths)
         check_keys
-        machine = find_requested_machine
+        machine = find_current_machine
+        
         paths = paths.flatten
         to_path = paths.pop
+        
         scp machine[:dns_name], keypairpath, user, paths, to_path, @remote, false, @print
       end
       

data/lib/rudy/command/images.rb

@@ -53,8 +53,21 @@ module Rudy
         ssh machine[:dns_name], keypairpath, user, "ec2-bundle-vol -r i386 -p #{@image_name} -k /mnt/pk-*pem -c /mnt/cert*pem -u #{@account}"
         ssh machine[:dns_name], keypairpath, user, "ec2-upload-bundle -b #{@bucket_name} -m /tmp/#{@image_name}.manifest.xml -a #{@access_key} -s #{@secret_key}"
         
-        # Why did I not use the Ruby API?
-        sh "ec2-register -K #{@ec2_private_key} -C #{@ec2_cert} #{@bucket_name}/#{@image_name}.manifest.xml"
+        @ec2.images.register("#{@bucket_name}/#{@image_name}.manifest.xml")
+      end
+      
+      def deregister(ami)
+        raise "You must supply an AMI ID (ami-XXXXXXX)" unless ami
+        puts "Deregistering AMI: #{ami}"
+        
+        are_you_sure?
+        
+        if @ec2.images.deregister(ami)
+          puts "Done!"
+        else
+          puts "There was an unknown problem!"
+        end
+        
       end
       
     end

data/lib/rudy/command/stage.rb

@@ -35,7 +35,7 @@ module Rudy
         
         puts "Running #{basename}..."
         scp machine[:dns_name], keypairpath, user, @rscripts[keypairname], "~/"
-        ssh machine[:dns_name], keypairpath, user, "chmod 755 ~/#{basename} && ~/#{basename} test #{@access_key} #{@secret_key} r1ll1r1ll1" 
+        ssh machine[:dns_name], keypairpath, user, "chmod 755 ~/#{basename} && ~/#{basename} #{@access_key} #{@secret_key}" 
 
       end
       

data/lib/rudy/command/volumes.rb

@@ -4,9 +4,46 @@ module Rudy
   module Command
     class Volumes < Rudy::Command::Base
       
+      def destroy_volume(id)
+        raise "No volume ID provided" unless id
+        raise "I will not help you destroy production!" if @environment == "prod"
+        raise "The volume #{id} doesn't exist!" unless @ec2.volumes.exists?(id)
+        disk = Rudy::MetaData::Disk.from_volume(@sdb, id)
+        
+        puts "Destroying #{id}!"
+        @ec2.volumes.destroy id
+        
+        if disk
+          disk.awsid = nil
+        end
+        
+        Rudy::MetaData::Disk.save(@sdb, disk)
+        
+      end
+      
       def print_volumes
-        y @ec2.volumes.list
+        machines = {}
+        volumes = @ec2.volumes.list
+        @ec2.volumes.list.each do |volume|
+          machine = @ec2.instances.get(volume[:aws_instance_id])
+          machines[ volume[:aws_instance_id] ] ||= {
+            :machine => machine,
+            :volumes => []
+          }
+          machines[ volume[:aws_instance_id] ][:volumes] << volume
+        end
+        
+        machines.each_pair do |instance_id, hash|
+          machine = hash[:machine]
+          env = (machine[:aws_groups]) ? machine[:aws_groups] : "Not-attached"
+          puts "Environment: #{env}"
+          hash[:volumes].each do |vol|
+            disk = Rudy::MetaData::Disk.from_volume(@sdb, vol[:aws_id])
+            print_volume(vol, disk)
+          end
+        end
       end
+      
     end
   end
 end

data/lib/rudy/metadata/backup.rb

@@ -0,0 +1,160 @@
+
+require 'date'
+
+module Rudy
+  module MetaData
+    class Backup < Storable
+      include Rudy::MetaData::ObjectBase
+      extend Rudy::MetaData::ObjectBase
+        
+      @@rtype = "back"
+      
+      field :rtype
+      field :awsid
+      
+      field :region
+      field :zone
+      field :environment
+      field :role
+      field :position
+      field :path
+      
+      field :date
+      field :time
+      field :second
+      
+      field :unixtime => Integer
+      
+      field :size
+      field :volume
+      
+      def initialize
+        @zone = DEFAULT_ZONE
+        @region = DEFAULT_REGION
+        @position = "01"
+        @rtype = @@rtype
+      end
+      
+      def rtype
+        @@rtype
+      end
+      
+      def self.rtype
+        @@rtype
+      end
+      
+      
+      def name
+        time = Time.at(@unixtime)
+        Backup.generate_name(@zone, @environment, @role, @position, @path, time)
+      end
+      
+      def valid?
+        @zone && @environment && @role && @position && @path && @date && @time && @sec
+      end
+      
+      def time_stamp
+        #return [@date, @time] if @date && @time
+        now = Time.now.utc
+        datetime = Backup.format_timestamp(now).split(RUDY_DELIM)
+        @unixtime = now.to_i
+        @date, @time, @second = datetime
+      end
+      
+      def nice_time
+        return "" unless @date && @time
+        t = @date.scan(/(\d\d\d\d)(\d\d)(\d\d)/).join('-')
+        t << " " << @time.scan(/(\d\d)(\d\d)/).join(':')
+        t
+      end
+      
+      def to_query(more=[], remove=[])
+        criteria = [:rtype, :zone, :environment, :role, :position, :path, :date, :time, :second, *more]
+        criteria -= [*remove].flatten
+        query = "select * from #{RUDY_DOMAIN} where unixtime > '0' "
+        criteria.each do |n|
+          query << "and #{n} = '#{self.send(n.to_sym)}'"
+        end
+        query << " order by unixtime desc"
+      end
+      
+      def to_s
+        str = ""
+        field_names.each do |key|
+          str << sprintf(" %22s: %s#{$/}", key, self.send(key.to_sym))
+        end
+        str
+       end
+      
+      def disk
+        Disk.generate_name(@zone, @environment, @role, @position, @path)
+      end
+      
+      # 20090224-1813-36
+      def Backup.format_timestamp(dat)
+        mon, day, hour, min, sec = [dat.mon, dat.day, dat.hour, dat.min, dat.sec].collect { |v| v.to_s.rjust(2, "0") }
+        [dat.year, mon, day, RUDY_DELIM, hour, min, RUDY_DELIM, sec].join
+      end
+      
+      # Times are converted to UTC
+      # back-us-east-1b-stage-app-01-rilli-app-20090224-1813-36
+      def Backup.generate_name(zon, env, rol, pos, pat, dat, sep=File::SEPARATOR)
+        raise "The date you provided is not a Time object" unless dat.is_a?(Time)
+        pos = pos.to_s.rjust 2, '0'
+        dirs = pat.split sep if pat
+        dirs.shift while dirs && (dirs[0].nil? || dirs[0].empty?)
+        timestamp = Backup.format_timestamp(dat.utc)
+        [@@rtype, zon, env, rol, pos, dirs, timestamp].flatten.join(RUDY_DELIM)
+      end
+      
+
+      def Backup.for_disk(sdb, disk, max=50)
+        list = Backup.list(sdb, disk.zone, disk.environment, disk.role, disk.position, disk.path) || []
+        list[0..(max-1)]
+      end
+      
+      def Backup.get(sdb, name)
+        object = sdb.get_attributes(RUDY_DOMAIN, name)
+        raise "Object #{name} does not exist!" unless object.has_key?(:attributes) && !object[:attributes].empty?
+        self.from_hash(object[:attributes])
+      end
+      
+      def Backup.save(sdb, obj, replace = :replace)
+        sdb.store(RUDY_DOMAIN, obj.name, obj.to_hash, replace)
+      end
+      
+      def Backup.list(sdb, zon, env=nil, rol=nil, pos=nil, path=nil, date=nil)
+        query = "select * from #{RUDY_DOMAIN} where "
+        query << "rtype = '#{rtype}' "
+        query << " and zone = '#{zon}'" if zon
+        query << " and environment = '#{env}'" if env
+        query << " and role = '#{rol}'" if rol
+        query << " and position = '#{pos}'" if pos
+        query << " and path = '#{path}'" if path
+        query << " and date = '#{date}'" if date
+        query << " and unixtime != '0' order by unixtime desc"
+        list = []
+        sdb.select(query).each do |obj|
+          list << self.from_hash(obj)
+        end
+        list
+      end
+      
+      def Backup.destroy(sdb, name)
+        back = Backup.get(sdb, name) # get raises an exception if the disk doesn't exist
+        sdb.destroy(RUDY_DOMAIN, name)
+        true # wtf: RightAws::SimpleDB doesn't tell us whether it succeeds. We'll assume!
+      end
+      
+
+      
+      def Backup.is_defined?(sdb, backup)
+        query = backup.to_query()
+        puts query
+        !sdb.select(query).empty?
+      end
+      
+    end
+  end
+end
+

data/lib/rudy/metadata/disk.rb

@@ -1,17 +1,17 @@
 
 
 module Rudy
-  class MetaData
-    attr_accessor :sdb
-    
-    def initalize(sdb)
-      @sdb = sdb
-    end
-    
-  end
-  class MetaData
+  
+  module MetaData
     class Disk < Storable
       
+      @@rtype = 'disk'
+      
+        # This is a flag used internally to specify that a volume has been
+        # created for this disk, but not formated. 
+      attr_accessor :raw_volume
+      
+      field :rtype
       field :awsid
       
       field :environment
@@ -22,7 +22,7 @@ module Rudy
       field :zone
       field :region
       field :device
-      field :backups => Array
+      #field :backups => Array
       field :size
       
       def initialize
@@ -30,7 +30,15 @@ module Rudy
         @zone = DEFAULT_ZONE
         @region = DEFAULT_REGION
         @backups = []
-        
+        @rtype = @@rtype.to_s
+        @raw_volume = false
+      end
+      
+      def rtype
+        @@rtype.to_s
+      end
+      
+      def rtype=(val)
       end
       
       def name
@@ -42,11 +50,11 @@ module Rudy
       end
       
       def to_query(more=[], remove=[])
-        criteria = [:zone, :environment, :role, :position, :path, *more]
+        criteria = [:rtype, :zone, :environment, :role, :position, :path, *more]
         criteria -= [*remove].flatten
         query = []
         criteria.each do |n|
-          query << "['#{n}' = '#{self.send(n.to_sym)}']"
+          query << "['#{n}' = '#{self.send(n.to_sym)}'] "
         end
         query.join(" intersection ")
       end
@@ -68,6 +76,7 @@ module Rudy
       
       def Disk.get(sdb, name)
         disk = sdb.get_attributes(RUDY_DOMAIN, name)
+        
         raise "Disk #{name} does not exist!" unless disk && disk.has_key?(:attributes)
         Rudy::MetaData::Disk.from_hash(disk[:attributes])
       end
@@ -89,6 +98,16 @@ module Rudy
         !sdb.query_with_attributes(RUDY_DOMAIN, query).empty?
       end
       
+      def Disk.from_volume(sdb, vol_id)
+        query = "['awsid' = '#{vol_id}']"
+        res = sdb.query_with_attributes(RUDY_DOMAIN, query)
+        if res.empty?
+          nil
+        else
+          disk = Rudy::MetaData::Disk.from_hash(res.values.first)
+        end
+      end
+      
       def Disk.update_volume(sdb, ec2, disk, machine)
         
         disk = Disk.get(sdb, disk) if disk.is_a?(String)
@@ -103,19 +122,29 @@ module Rudy
         unless disk.awsid
           puts "No active EBS volume found for #{disk.name}"
           
-          backup = disk.backups.last
-          if backup
-            puts "We'll use the most recent backup..."
-            volume = ec2.volumes.create(disk.zone, disk.size, backup)
-            puts "Attaching #{disk.awsid} to #{id} (#{disk.device})"
-            ec2.volumes.attach(machine[:aws_instance_id], disk.awsid, disk.device)
+          # TODO: pull actual backups
+          backups = Rudy::MetaData::Backup.for_disk(sdb, disk, 2)
+          
+          if backups.is_a?(Array) && !backups.empty?
+            backup = backups.first
+            if ec2.snapshots.exists?(backup.awsid)
+              puts "We'll use the most recent backup (#{backup.awsid})..."
+              volume = ec2.volumes.create(disk.zone, disk.size, backup.awsid)
+            else
+              puts "The backup refers to a snapshot that doesn't exist."
+              puts backup.name, backup.awsid
+              puts "You need to delete this backup metadata before continuing."
+              exit 1
+            end
           else
             puts "We'll create one from scratch..."
             volume = ec2.volumes.create(disk.zone, disk.size, nil)
-            disk.awsid = volume[:aws_id]
-            puts "Saving disk metadata"
-            Disk.save(sdb, disk)
+            disk.raw_volume = true
           end
+          
+          puts "Saving disk metadata"
+          disk.awsid = volume[:aws_id]
+          Disk.save(sdb, disk)
           puts ""
         end
         
@@ -124,13 +153,15 @@ module Rudy
       
       def Disk.list(sdb, zon, env=nil, rol=nil, pos=nil)
         query = ''
-        query << "['zone' = '#{zon}']" if zon
+        query << "['rtype' = '#{@@rtype}']" if zon
+        query << " intersection ['zone' = '#{zon}']" if zon
         query << " intersection ['environment' = '#{env}']" if env
         query << " intersection ['role' = '#{rol}']" if rol
         query << " intersection ['position' = '#{pos}']" if pos
         
         list = []
         sdb.query_with_attributes(RUDY_DOMAIN, query).each_pair do |name, hash|
+          #puts "DISK: #{hash.to_yaml}"
           list << Rudy::MetaData::Disk.from_hash(hash)
         end
         list

data/lib/rudy/metadata/ec2startup.rb

@@ -0,0 +1,2 @@
+
+

data/lib/rudy/metadata.rb

@@ -0,0 +1,26 @@
+
+require 'aws_sdb'
+
+module Rudy
+  module MetaData
+    attr_accessor :sdb
+    attr_accessor :ec2
+    
+    def initalize(sdb, ec2)
+      @sdb = sdb
+      @ec2 = ec2
+    end
+    
+  end
+  
+  module MetaData
+    
+    module ObjectBase
+      extend self
+      
+
+      
+    end
+    
+  end
+end

data/lib/rudy.rb

@@ -4,9 +4,11 @@ require 'stringio'
 require 'ostruct'
 require 'yaml'
 
+require 'console'
 require 'storable'
 
 require 'rudy/aws'
+require 'rudy/metadata'
 require 'rudy/scm/svn'
 require 'rudy/utils'
 require 'rudy/command/base'
@@ -32,8 +34,8 @@ module Rudy #:nodoc:
   
   module VERSION #:nodoc:
     MAJOR = 0.freeze unless defined? MAJOR
-    MINOR = 2.freeze unless defined? MINOR
-    TINY  = 4.freeze unless defined? TINY
+    MINOR = 3.freeze unless defined? MINOR
+    TINY  = 0.freeze unless defined? TINY
     def self.to_s
       [MAJOR, MINOR, TINY].join('.')
     end
@@ -68,10 +70,34 @@ def capture(stream)
   result
 end
 
-def you_are_sure?
-  print "Are you sure? "
-  STDIN.gets =~ /^y|yes|ya$/i
+
+def are_you_sure?(len=3)
+  if STDIN.tty? # Only ask a question if there's a human 
+    challenge = strand len
+    STDOUT.print "Are you sure? To continue type \"#{challenge}\": "
+    STDOUT.flush
+    if ((STDIN.gets || "").gsub(/["']/, '') =~ /^#{challenge}$/)
+      true
+    else
+      puts "Nothing changed"
+      exit 0
+    end
+  else
+    true
+  end
+end
+    
+# 
+# Generates a string of random alphanumeric characters
+# These are used as IDs throughout the system
+def strand( len=8, safe=true )
+   chars = ("a".."z").to_a + ("0".."9").to_a
+   chars = [("a".."h").to_a, "j", "k", "m", "n", ("p".."z").to_a, ("2".."9").to_a].flatten if safe
+   newpass = ""
+   1.upto(len) { |i| newpass << chars[rand(chars.size-1)] }
+   newpass
 end
+
 def sh(command, chdir=false)
   prevdir = Dir.pwd
   Dir.chdir chdir if chdir
@@ -92,20 +118,25 @@ end
 
 
 def scp(host, keypair, user, paths, to_path, to_local=false, verbose=false, printonly=false)
-  puts "CONNECTING TO #{host}..."
+  
   paths = [paths] unless paths.is_a?(Array)
   from_paths = ""
   if to_local
     paths.each do |path|
       from_paths << "#{user}@#{host}:#{path} "
-    end
+    end  
+    puts "Copying FROM remote TO this machine", $/
+    
   else
     to_path = "#{user}@#{host}:#{to_path}"
     from_paths = paths.join(' ')
+    puts "Copying FROM this machine TO remote", $/
   end
   
+  
   cmd = "scp -i #{keypair} #{from_paths} #{to_path}"
 
+  puts "CONNECTING TO #{host}..."
   puts cmd if verbose
   printonly ? (puts cmd) : system(cmd)
 end