solutious-rudy 0.5.1 → 0.6.0

data/lib/rudy/machines.rb

@@ -0,0 +1,245 @@
+
+
+
+module Rudy
+  class Machine < Storable 
+    include Rudy::MetaData::ObjectBase
+  
+    field :rtype
+    field :awsid
+
+    field :region
+    field :zone
+    field :environment
+    field :role
+    field :position
+    
+    field :created => Time
+    field :started => Time
+    
+    field :dns_public
+    field :dns_private
+    field :state
+    
+    attr_reader :instance
+    
+    def init
+      #@created = 
+      @rtype = 'm'
+      @region = @@global.region
+      @zone = @@global.zone
+      @environment = @@global.environment
+      @role = @@global.role
+      @position = find_next_position || '01'
+      @state = 'no-instance'
+    end
+    
+    def liner_note
+      update #if !dns_public? && @awsid
+      info = !@dns_public.nil? && !@dns_public.empty? ? @dns_public : "#{@awsid}:#{@state}"
+      "%s  %s" % [self.name.bright, info]
+    end
+    
+    def to_s(with_title=false)
+      lines = []
+      lines << liner_note
+      #if self.running?
+      #  k, g = @keyname || 'no-keypair', self.groups.join(', ')
+      #  lines << @@sformat % %w{zone size ami keyname groups} if with_title
+      #  lines << @@sformat % [@zone, @size, @ami, k, g]
+      #end
+      lines.join($/)
+    end
+    
+    def inspect
+      update #if !dns_public? && @awsid
+      lines = []
+      lines << liner_note
+      field_names.each do |key|
+        next unless self.respond_to?(key)
+        val = self.send(key)
+        lines << sprintf(" %22s: %s", key, (val.is_a?(Array) ? val.join(', ') : val))
+      end
+      lines.join($/)
+    end
+      
+    def find_next_position
+      list = @sdb.select(self.to_select(nil, [:position])) || []
+      pos = list.size + 1
+      pos.to_s.rjust(2, '0')
+    end
+    
+    def name
+      Machine.generate_name(@zone, @environment, @role, @position)
+    end
+
+    def Machine.generate_name(zon, env, rol, pos)
+      pos = pos.to_s.rjust 2, '0'
+      ["m", zon, env, rol, pos].join(Rudy::DELIM)
+    end
+    
+    def get_instance
+      @ec2inst.get(@awsid)
+    end
+    
+    def update
+      return false unless @awsid
+      @instance = get_instance
+      if @instance.is_a?(Rudy::AWS::EC2::Instance)
+        @dns_public = @instance.dns_public
+        @dns_private = @instance.dns_private
+        @state = @instance.state
+        save
+      elsif @instance.nil?
+        @dns_public = @dns_private = nil
+        @state = 'unknown'
+      end
+    end
+    
+    def dns_public?
+      !@dns_public.nil? && !@dns_public.empty?
+    end
+    def dns_private?
+      !@dns_private.nil? && !@dns_private.empty?
+    end
+    
+    def start(opts={})
+      raise "#{name} is already running" if running?
+      
+      opts = {
+        :min  => 1,
+        :size => current_machine_size,
+        :ami => current_machine_image,
+        :group => current_group_name,
+        :keypair => root_keypairname, 
+        :zone => @@global.zone.to_s,
+        :address => current_machine_address,
+        :machine_data => Machine.generate_machine_data.to_yaml
+      }.merge(opts)
+      
+      @ec2inst.create(opts) do |inst|
+        @awsid = inst.awsid
+        @created = @starts = Time.now
+        @state = inst.state
+      end
+      
+      self.save
+      
+      self
+    end
+    
+    def destroy
+      @ec2inst.destroy(@awsid) if running?
+      super
+    end
+    
+    
+    def Machine.generate_machine_data
+      data = {      # Give the machine an identity
+        :region => @@global.region.to_s,
+        :zone => @@global.zone.to_s,
+        :environment => @@global.environment.to_s,
+        :role => @@global.role.to_s,
+        :position => @@global.position.to_s,
+        :hosts => { # Add hosts to the /etc/hosts file 
+          :dbmaster => "127.0.0.1",
+        }
+      } 
+      data
+    end
+    
+    def running?
+      return false if @awsid.nil? || @awsid.empty?
+      @ec2inst.running?(@awsid)
+    end
+      
+  end
+  
+  
+  
+  class Machines
+    include Rudy::MetaData
+        
+    def create(&each_mach)
+      raise MachineGroupAlreadyRunning, current_machine_group if running?
+      raise MachineGroupNotDefined, current_machine_group unless known_machine_group?
+      
+      unless (1..MAX_INSTANCES).member?(current_machine_count)
+        raise "Instance count must be more than 0, less than #{MAX_INSTANCES}"
+      end
+      
+      unless @rgrp.exists?(current_group_name)
+        puts "Creating group: #{current_group_name}"
+        @rgrp.create(current_group_name)
+      end
+      
+      unless @rkey.exists?(root_keypairname)
+        kp_file = File.join(Rudy::CONFIG_DIR, root_keypairname)
+        raise PrivateKeyFileExists, kp_file if File.exists?(kp_file)
+        puts "Creating keypair: #{root_keypairname}"
+        kp = @rkey.create(root_keypairname)
+        puts "Saving #{kp_file}"
+        Rudy::Utils.write_to_file(kp_file, kp.private_key, 'w', 0600)
+      else
+        kp_file = root_keypairpath
+        # This means no keypair file can be found
+        raise PrivateKeyNotFound, root_keypairname if kp_file.nil?
+        # This means we found a keypair in the config but we cannot find the private key file. 
+        raise PrivateKeyNotFound, kp_file if !File.exists?(kp_file)
+      end
+      
+      machines = []
+      current_machine_count.times do  |i|
+        machine = Rudy::Machine.new
+        puts "Starting %s" % machine.name
+        machine.start
+        machines << machine
+      end
+      machines.each { |m| each_mach.call(m) } if each_mach
+      machines
+    end
+    
+    
+    def destroy(&each_mach)
+      raise MachineGroupNotRunning, current_machine_group unless running?
+      raise MachineGroupNotDefined, current_machine_group unless known_machine_group?
+      list.each { |m| each_mach.call(m); } if each_mach
+      list do |mach|
+        puts "Destroying #{mach.name}"
+        mach.destroy
+      end
+    end
+    
+    def list(more=[], less=[], &each_mach)
+      machines = list_as_hash(&each_mach)
+      machines &&= machines.values
+      machines
+    end
+    
+    def list_as_hash(more=[], less=[], &each_mach)
+      query = to_select([:rtype, 'm'], less)
+      list = @sdb.select(query) || {}
+      machines = {}
+      list.each_pair do |n,m|
+        machines[n] = Rudy::Machine.from_hash(m)
+      end
+      machines.each_pair { |n,mach| each_mach.call(mach) } if each_mach
+      machines = nil if machines.empty?
+      machines
+    end
+    
+    def get(rname=nil)
+      Rudy::Machine.from_hash(@sdb.get(Rudy::DOMAIN, rname)) # Returns nil if empty
+    end
+    
+    
+    def running?
+      !list.nil?
+      # TODO: add logic that checks whether the instances are running.
+    end
+    
+
+    
+  end
+  
+end

data/lib/rudy/metadata.rb

@@ -0,0 +1,136 @@
+
+
+module Rudy
+  module MetaData
+    module ObjectBase
+      include Rudy::Huxtable
+      
+      def initialize(*args)
+        a, s, r = @@global.accesskey, @@global.secretkey, @@global.region
+        @sdb = Rudy::AWS::SDB.new(a, s, r)
+        @ec2inst = Rudy::AWS::EC2::Instances.new(a, s, r)
+        @rvol = Rudy::AWS::EC2::Volumes.new(a, s, r)
+        init(*args)
+      end
+      
+      def init(*args); raise "Must override init"; end
+      
+      def valid?; raise "#{self.class} must override 'valid?'"; end
+      
+      def to_query(more=[], less=[])
+        Rudy::AWS::SDB.generate_query build_criteria(more, less)
+      end
+    
+      def to_select(more=[], less=[])
+        Rudy::AWS::SDB.generate_select ['*'], Rudy::DOMAIN, build_criteria(more, less)
+      end
+      
+      def name(identifier, zon, env, rol, pos, *other)
+        pos = pos.to_s.rjust 2, '0'
+        [identifier, zon, env, rol, pos, *other].flatten.compact.join(Rudy::DELIM)
+      end
+      
+      def save(replace=true)
+        replace = true if replace.nil?
+        @sdb ||= Rudy::AWS::SDB.new(@@global.accesskey, @@global.secretkey, @@global.region)
+        @sdb.put(Rudy::DOMAIN, name, self.to_hash, replace) # Always returns nil
+        true
+      end
+    
+      def destroy
+        @sdb ||= Rudy::AWS::SDB.new(@@global.accesskey, @@global.secretkey, @@global.region)
+        @sdb.destroy(Rudy::DOMAIN, name)
+        true
+      end
+    
+      def refresh
+        @sdb ||= Rudy::AWS::SDB.new(@@global.accesskey, @@global.secretkey, @@global.region)
+        h = @sdb.get(Rudy::DOMAIN, name) || {}
+        from_hash(h)
+      end
+      
+      def ==(other)
+        return false unless other.is_a?(self.class)
+        self.name == other.name
+      end
+      
+      def to_s
+        str = ""
+        field_names.each do |key|
+          str << sprintf(" %22s: %s#{$/}", key, self.send(key.to_sym))
+        end
+        str
+      end
+
+      
+    protected
+    
+      # Builds a zipped Array from a list of criteria.
+      # The list of criteria is made up of metadata object attributes. 
+      # The list is constructed by taking the adding +more+, and
+      # subtracting +less+ from <tt>:rtype, :region, :zone, :environment, :role, :position</tt>
+      # 
+      # Returns [[:rtype, value], [:zone, value], ...]
+      def build_criteria(more=[], less=[])
+        criteria = [:rtype, :region, :zone, :environment, :role, :position, *more].compact
+        criteria -= [*less].flatten.uniq.compact
+        values = criteria.collect do |n|
+          self.send(n.to_sym)
+        end
+        criteria.zip(values)
+      end
+      
+    end
+  end
+end
+
+
+
+module Rudy
+  module MetaData
+    include Rudy::Huxtable
+    
+    def initialize(*args)
+      a, s, r = @@global.accesskey, @@global.secretkey, @@global.region
+      @sdb = Rudy::AWS::SDB.new(a, s, r)
+      @rinst = Rudy::AWS::EC2::Instances.new(a, s, r)
+      @rgrp = Rudy::AWS::EC2::Groups.new(a, s, r)
+      @rkey = Rudy::AWS::EC2::KeyPairs.new(a, s, r)
+      init(*args)
+    end
+    
+    def init(*args)
+    end
+    
+    # 20090224-1813-36
+    def format_timestamp(dat)
+      mon, day, hour, min, sec = [dat.mon, dat.day, dat.hour, dat.min, dat.sec].collect { |v| v.to_s.rjust(2, "0") }
+      [dat.year, mon, day, Rudy::DELIM, hour, min, Rudy::DELIM, sec].join
+    end
+    
+  private
+  
+    # Returns a generic zipped Array of metadata
+    # (There is region, zone, environment, role, but no rtype)
+    def build_criteria(more=[], less=[])
+      # TODO: This build_criteria treats "more" differently than the
+      # ObjectBase one. Sort it out! (This way is better)
+      names = [:region, :zone, :environment, :role].compact
+      names -= [*less].flatten.uniq.compact
+      values = names.collect do |n|
+        @@global.send(n.to_sym)
+      end
+      names.zip(values) + more
+    end
+    
+    def to_query(more=[], less=[])
+      Rudy::AWS::SDB.generate_query build_criteria(more, less)
+    end
+  
+    def to_select(more=[], less=[])
+      Rudy::AWS::SDB.generate_select ['*'], Rudy::DOMAIN, build_criteria(more, less)
+    end
+    
+  end
+end
+

data/lib/rudy/routines/helpers/diskhelper.rb

@@ -0,0 +1,101 @@
+
+
+module Rudy; module Routines; 
+  module DiskHelper
+    extend self
+    
+    def execute(routine, machine, rbox)
+      return unless routine
+      raise "Not a Rudy::Machine" unless machine.is_a?(Rudy::Machine)
+      raise "Not a Rye::Box" unless rbox.is_a?(Rye::Box)
+      
+      @machine = machine
+      @rbox = rbox
+      
+      (routine.disks || {}).each_pair do |action, disks|
+        unless DiskHelper.respond_to?(action)  
+          STDERR.puts %Q(DiskHelper: unknown action "#{action}")
+          next
+        end
+        send(action, disks) # create, copy, destroy, ...
+      end
+    end
+    
+    def create(disks)
+      rdisk = Rudy::Disks.new
+      
+      disks.each_pair do |path, props|
+        disk = Rudy::Disk.new(path, props[:size], props[:device], @machine.position)
+        #disk = rdisk.get(disk.name)
+        puts "Creating #{disk.name} "
+        
+        print "Creating volume... "
+        disk.create
+        Rudy::Utils.waiter(2, 60, STDOUT, "done", nil) { 
+          disk.available?
+        }
+        
+        print "Attaching #{disk.awsid} to #{@machine.awsid}... "
+        disk.attach(@machine.awsid)
+        Rudy::Utils.waiter(2, 60, STDOUT, "done", nil) { 
+          disk.attached?
+        }
+        
+        # The device needs some time. 
+        # Otherwise mkfs returns:
+        # "No such file or directory while trying to determine filesystem size"
+        sleep 2 
+        
+        print "Creating ext3 filesystem for #{disk.device}... "
+        @rbox.mkfs(:t, "ext3", :F, disk.device)
+        @rbox.mkdir(:p, disk.path)
+        puts "done"
+        
+        print "Mounting at #{disk.path}... "
+        
+        @rbox.mount(:t, 'ext3', disk.device, disk.path)
+        disk.mounted = true
+        disk.save
+        puts "done"
+        
+      end
+    end
+      
+    def destroy(disks)
+      rdisk = Rudy::Disks.new
+      
+      disks.each_pair do |path, props|
+        adisk = Rudy::Disk.new(path, props[:size], props[:device], @machine.position)
+        disk = rdisk.get(adisk.name)
+        if disk == nil
+          puts "Not found: #{adisk.name}".color(:red)
+          return
+        end
+        
+        puts "Destroying #{disk.name}"
+
+        if disk.mounted?
+          print "Unmounting #{disk.path}... "
+          @rbox.umount(disk.path)
+          sleep 0.5
+          puts "done"
+        end
+        
+        if disk.attached?
+          print "Detaching #{disk.awsid}... "
+          disk.detach 
+          Rudy::Utils.waiter(2, 60, STDOUT, 'done', nil) { 
+            disk.available? 
+          }
+          sleep 0.5
+        end
+        
+        print "Destroying metadata... "
+        disk.destroy
+        puts "done"
+        
+      end
+    end
+      
+  end
+end;end

data/lib/rudy/routines/helpers/scripthelper.rb

@@ -0,0 +1,91 @@
+require 'tempfile'
+
+
+module Rudy; module Routines; 
+  
+  module ScriptHelper
+    extend self
+    
+    @@script_types = [:after, :before, :after_local, :before_local]
+    @@script_config_file = "rudy-config.yml"
+    
+    def before_local(routine, sconf, rbox)
+      execute_command(:before_local, routine, sconf, 'localhost', rbox)
+    end    
+    def after_local(routine, sconf, rbox)
+      execute_command(:after_local, routine, sconf, 'localhost', rbox)
+    end
+  
+    def before(routine, sconf, machine, rbox)
+      raise "ScriptHelper: Not a Rudy::Machine" unless machine.is_a?(Rudy::Machine)
+      execute_command(:before, routine, sconf, machine.name, rbox)
+    end
+    def after(routine, sconf, machine, rbox)
+      raise "ScriptHelper: Not a Rudy::Machine" unless machine.is_a?(Rudy::Machine)
+      execute_command(:after, routine, sconf, machine.name, rbox)
+    end
+    
+  
+  private  
+    
+    # * +timing+ is one of: after, before
+    # * +routine+ a single routine hash (startup, shutdown, etc...)
+    # * +sconf+ is a config hash from machines config (ignored if nil)
+    # * +hostname+ machine hostname that we're working on
+    # * +rbox+ a Rye::Box instance for the machine we're working on
+    def execute_command(timing, routine, sconf, hostname, rbox)
+      raise "ScriptHelper: Not a Rye::Box" unless rbox.is_a?(Rye::Box)
+      raise "ScriptHelper: #{timing}?" unless @@script_types.member?(timing)
+      
+      if sconf && !sconf.empty?
+        tf = Tempfile.new(@@script_config_file)
+        Rudy::Utils.write_to_file(tf.path, sconf.to_hash.to_yaml, 'w')
+      end
+      
+      # We need to explicitly add the rm command for rbox so we
+      # can delete the script config file when we're done. This
+      # add the method on for the instance of rbox we are using. 
+      def rbox.rm(*args); cmd('rm', args); end
+      
+      if routine.is_a?(Caesars::Hash) && routine.has_key?(timing)
+        puts "Connecting to #{hostname}"
+        rbox.connect
+        original_user = rbox.user
+        scripts = [routine[timing]].flatten
+        scripts.each do |script|
+          user, command, *args = script.to_a.flatten.compact
+          rbox.switch_user user # does nothing if it's the same user
+          puts "Creating #{@@script_config_file}"
+          rbox.safe = false
+          puts rbox.echo("'#{sconf.to_hash.to_yaml}' > #{@@script_config_file}")
+          rbox.safe = true
+          rbox.chmod(600, @@script_config_file)
+          puts %Q{Running (as #{user}): #{rbox.preview_command(command, args)}}
+          
+          begin
+            ret = rbox.send(command, args)
+            if ret.exit_code > 0
+              puts "  Exit code: #{ret.exit_code}".color(:red)
+              puts "  STDERR: #{ret.stderr.join("#{$/}  ")}".color(:red)
+              puts "  STDOUT: #{ret.stdout.join("#{$/}  ")}".color(:red)
+            else
+              puts '  ' << ret.stdout.join("#{$/}  ")
+            end
+          rescue Rye::CommandNotFound => ex
+            puts "  CommandNotFound: #{ex.message}".color(:red)
+          end
+          
+          
+          rbox.rm(@@script_config_file)
+        end
+        rbox.switch_user original_user
+      else
+        #puts "Nothing to do"
+      end
+      
+      tf.delete # delete local copy of script config
+      
+    end
+  end
+  
+end;end

data/lib/rudy/routines/release.rb

@@ -1,15 +1,34 @@
-module Rudy
-  module Routines
-    class Release < Rudy::Routines::Base
-      
 
-      def execute()
-        
-      end
+module Rudy; module Routines;
+  class Release < Rudy::Routines::Base
 
+    def execute
+      p find_scm(:release)
+    end
+    
+    
+  private
+    def find_scm(routine)
+      env, rol, att = @@global.environment, @@global.role
 
+      # Look for the source control engine, checking all known scm values.
+      # The available one will look like [environment][role][release][svn]
+      params = nil
+      scm_name = nil
+      SUPPORTED_SCM_NAMES.each do |v|
+        scm_name = v
+        params = @@config.routines.find(env, rol, routine, scm_name)
+        break if params
+      end
+
+      if params
+        klass = eval "Rudy::SCM::#{scm_name.to_s.upcase}"
+        scm = klass.new(:base => params[:base])
+      end
 
+      [scm, params]
 
     end
+    
   end
-end
+end;end

data/lib/rudy/routines/shutdown.rb

@@ -1,42 +1,57 @@
 
 
-module Rudy
-  module Routines
-    class Shutdown < Rudy::Routines::Base
-      
-      def shutdown
-        routine = fetch_routine(:shutdown)
-        rmach = Rudy::Instances.new(:config => @config, :global => @global)
-        rmach.destroy do
-          @logger.puts $/, "Running BEFORE scripts...", $/
-          #instances.each { |inst| @script_runner.execute(inst, :shutdown, :before) }
-
-          @logger.puts $/, "Running DISK routines...", $/
-          routine.disks.each_pair do |action,disks|
+module Rudy; module Routines;
 
-            unless @rdisks.respond_to?(action)
-              @logger.puts("Skipping unknown action: #{action}").color(:blue)
-              next
-            end
+  class Shutdown < Rudy::Routines::Base
+    
+    def execute
+      raise Rudy::PrivateKeyNotFound, root_keypairpath unless has_keypair?(:root)
+      rmach = Rudy::Machines.new
+      routine = fetch_routine_config(:shutdown)
+      rbox_local = Rye::Box.new('localhost')
+      sconf = fetch_script_config
 
-            disks.each_pair do |path,props|
-              props[:path] = path
-              begin
-                @rdisks.send(action, instance, props)
-              rescue => ex
-                @logger.puts ex.message
-                @logger.puts "Continuing..."
-                @logger.puts ex.backtrace if debug?
-              end
-            end
-            
-          end
-        end
+      # Runs "before_local" scripts of routines config. 
+      puts task_separator("BEFORE SCRIPTS (local)")
+      Rudy::Routines::ScriptHelper.before_local(routine, sconf, rbox_local)
+      
+      rmach.destroy do |machine|
+      #rmach.list do |machine|
+        
+        print "Waiting for instance..."
+        isup = Rudy::Utils.waiter(3, 120, STDOUT, "it's up!", 0) {
+          inst = machine.get_instance
+          inst && inst.running?
+        } 
+        machine.update # Add instance info to machine and save it
+        print "Waiting for SSH daemon..."
+        isup = Rudy::Utils.waiter(2, 60, STDOUT, "it's up!", 0) {
+          Rudy::Utils.service_available?(machine.dns_public, 22)
+        }
+        
+        opts = { :keys =>  root_keypairpath, :user => 'root', :debug => nil }
+        rbox = Rye::Box.new(machine.dns_public, opts)
         
+        # Runs "before" scripts of routines config. 
+        puts task_separator("BEFORE SCRIPTS")
+        Rudy::Routines::ScriptHelper.before(routine, sconf, machine, rbox)
+                
+        # Runs "disk" portion of routines config
+        puts task_separator("DISK ROUTINES")
+        Rudy::Routines::DiskHelper.execute(routine, machine, rbox)
+        
+        puts machine_separator(machine.liner_note)
       end
       
+        
+      # Runs "after_local" scripts
+      # NOTE: There "after" (remote) scripts are not run b/c the machines
+      # are no longer running. 
+      puts task_separator("AFTER SCRIPTS (local)")
+      Rudy::Routines::ScriptHelper.after_local(routine, sconf, rbox_local)
+      
     end
+
   end
-end
 
-      
+end; end