sml-rubber 0.9.1

data/lib/rubber/recipes/rubber/static_ips.rb

@@ -0,0 +1,107 @@
+namespace :rubber do
+
+
+  desc <<-DESC
+    Sets up static IPs for the instances configured to have them
+  DESC
+  required_task :setup_static_ips do
+    rubber_cfg.instance.each do |ic|
+      env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+      if env.use_static_ip
+        artifacts = rubber_cfg.instance.artifacts
+        ip = artifacts['static_ips'][ic.name] rescue nil
+
+        # first allocate the static ip if we don't have a global record (artifacts) for it
+        if ! ip
+          logger.info "Allocating static IP for #{ic.full_name}"
+          ip = allocate_static_ip()
+          artifacts['static_ips'][ic.name] = ip
+          rubber_cfg.instance.save
+        end
+
+        # then, associate it if we don't have a record (on instance) of association
+        if ! ic.static_ip
+          logger.info "Associating static ip #{ip} with #{ic.full_name}"
+          associate_static_ip(ip, ic.instance_id)
+
+          instance = cloud.describe_instances(:instance_id => ic.instance_id).first
+          ic.external_host = instance[:external_host]
+          ic.internal_host = instance[:internal_host]
+          ic.external_ip = ip
+          ic.static_ip = ip
+          rubber_cfg.instance.save()
+
+          logger.info "Waiting for static ip to associate"
+          while true do
+            task :_wait_for_static_ip, :hosts => ip do
+              run "echo"
+            end
+            begin
+              _wait_for_static_ip
+            rescue ConnectionError
+              sleep 2
+              logger.info "Failed to connect to static ip #{ip}, retrying"
+              retry
+            end
+            break
+          end
+        end
+
+      end
+    end
+  end
+
+  desc <<-DESC
+    Shows the configured static IPs
+  DESC
+  required_task :describe_static_ips do
+    results = []
+    format = "%-10s %-15s %-30s"
+    results << format % %w[InstanceID IP Alias]
+
+    ips = cloud.describe_addresses()
+    ips.each do |ip_data|
+      instance_id = ip_data[:instance_id]
+      ip = ip_data[:ip]
+
+      local_alias = find_alias(ip, instance_id, false)
+
+      results << format % [instance_id || "Unassigned", ip, local_alias || "Unknown"]
+    end
+    
+    results.each {|r| logger.info r}
+  end
+
+  desc <<-DESC
+    Deallocates the given static ip
+  DESC
+  required_task :destroy_static_ip do
+    ip = get_env('IP', "Static IP (run rubber:describe_static_ips for a list)", true)
+    destroy_static_ip(ip)
+  end
+
+  def allocate_static_ip()
+    ip = cloud.create_static_ip()
+    fatal "Failed to allocate static ip" if ip.nil?
+    return ip
+  end
+
+  def associate_static_ip(ip, instance_id)
+    success = cloud.attach_static_ip(ip, instance_id)
+    fatal "Failed to associate static ip" unless success
+  end
+
+  def destroy_static_ip(ip)
+    logger.info "Releasing static ip: #{ip}"
+    cloud.destroy_static_ip(ip) rescue logger.info("IP was not attached")
+
+    logger.info "Removing ip #{ip} from rubber instances file"
+    artifacts = rubber_cfg.instance.artifacts
+    artifacts['static_ips'].delete_if {|k,v| v == ip}
+    rubber_cfg.instance.each do |ic|
+      ic.static_ip = nil if ic.static_ip == ip
+    end
+    rubber_cfg.instance.save
+  end
+
+end

data/lib/rubber/recipes/rubber/utils.rb

@@ -0,0 +1,195 @@
+namespace :rubber do
+    
+  desc <<-DESC
+    Convenience task for creating a staging instance for the given RUBBER_ENV/RAILS_ENV.
+    By default this task assigns all known roles when creating the instance,
+    but you can specify a different default in rubber.yml:staging_roles
+    At the end, the instance will be up and running
+    e.g. RUBBER_ENV=matt cap create_staging
+  DESC
+  required_task :create_staging do
+    if rubber_cfg.instance.size > 0
+      value = Capistrano::CLI.ui.ask("The #{RUBBER_ENV} environment already has instances, Are you SURE you want to create a staging instance that may interact with them [y/N]?: ")
+      fatal("Exiting", 0) if value !~ /^y/
+    end
+    instance_alias = ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname to use for staging instance", true, RUBBER_ENV)
+    default_roles = rubber_cfg.environment.bind().staging_roles || "*"
+    roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
+    if rubber_cfg.instance[instance_alias]
+      logger.info "Instance already exists, skipping to bootstrap"
+    else
+      rubber.create
+    end
+    rubber.bootstrap
+    # stop everything in case we have a bundled instance with monit, etc starting at boot
+    deploy.stop rescue nil
+    # bootstrap_db does setup/update_code, so since release directory
+    # variable gets reused by cap, we have to just do the symlink here - doing
+    # a update again will fail
+    deploy.symlink
+    deploy.migrate
+    deploy.start
+  end
+
+  desc <<-DESC
+    Destroy the staging instance for the given RUBBER_ENV.
+  DESC
+  task :destroy_staging do
+    ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname of staging instance to be destroyed", true, RUBBER_ENV)
+    rubber.destroy
+  end
+
+  desc <<-DESC
+    Live tail of rails log files for all machines
+    By default tails the rails logs for the current RUBBER_ENV, but one can
+    set FILE=/path/file.*.glob to tails a different set
+  DESC
+  task :tail_logs, :roles => :app do
+    log_file_glob = rubber.get_env("FILE", "Log files to tail", true, "#{current_path}/log/#{RUBBER_ENV}*.log")
+    run "tail -qf #{log_file_glob}" do |channel, stream, data|
+      puts  # for an extra line break before the host name
+      puts data
+      break if stream == :err
+    end
+  end
+
+  # Use instead of task to define a capistrano task that runs serially instead of in parallel
+  # The :groups option specifies how many groups to partition the servers into so that we can
+  # do the task for N (= total/groups) servers at a time.  When multiple roles are supplied,
+  # this tries to be intelligent and slice up each role independently, but runs the slices together
+  # so that things don't take too long, e.g. adding an :api role to some :app servers, when restarting
+  # you don't want to do the api first, then the others as this would take a long time, so instead
+  # it does some :api and some :app, then some more of each
+  #
+  def serial_task(ns, name, options = {}, &block)
+    # first figure out server names for the passed in roles - when no roles
+    # are passed in, use all servers
+    serial_roles = Array(options[:roles])
+    servers = {}
+    if serial_roles.empty?
+      all_servers = []
+      self.roles.each do |rolename, serverdefs|
+        all_servers += serverdefs.collect {|server| server.host}
+      end
+      servers[:_serial_all] = all_servers.uniq.sort
+    else
+      # get servers for each role
+      self.roles.each do |rolename, serverdefs|
+        if serial_roles.include?(rolename)
+          servers[rolename] ||= []
+          servers[rolename] += serverdefs.collect {|server| server.host}
+        end
+      end
+
+      # Remove duplication of servers - roles which come first in list
+      # have precedence, so the servers show up in that group
+      serial_roles.each_with_index do |rolename, i|
+        servers[rolename] ||= []
+        serial_roles[i+1..-1].each do |r|
+          servers[r] -= servers[rolename]
+        end
+        servers[rolename] = servers[rolename].uniq.sort
+      end
+    end
+
+    # group each role's servers into slices, but combine slices across roles
+    slices = []
+    servers.each do |rolename, svrs|
+      next if svrs.size == 0
+      # figure out size of each slice by deviding server count by # of groups
+      slice_size = svrs.size / (options.delete(:groups) || 2)
+      slice_size = 1 if slice_size == 0
+      slice_idx = 0
+      svrs.each_slice(slice_size) do |srv_slice|
+        slices[slice_idx] ||= []
+        slices[slice_idx] += srv_slice
+        slice_idx += 1
+      end
+    end
+    # for each slice, define a new task specific to the hosts in that slice
+    task_syms = []
+    slices.each do |server_group|
+      servers = server_group.map{|s| s.gsub(/\..*/, '')}.join("_")
+      task_sym = "_serial_task_#{name.to_s}_#{servers}".to_sym
+      task_syms << task_sym
+      ns.task task_sym, options.merge(:hosts => server_group), &block
+    end
+
+    # create the top level task that calls all the serial ones
+    ns.task name, options do
+      task_syms.each do |t|
+        ns.send t
+      end
+    end
+  end
+
+  def find_alias(ip, instance_id, do_connect=true)
+    if instance_id
+      instance = rubber_cfg.instance.find {|i| i.instance_id == instance_id }
+      local_alias = instance.full_name if instance
+    end
+    local_alias ||= File.read("/etc/hosts").grep(/#{ip}/).first.split[1] rescue nil
+    if ! local_alias && do_connect
+      task :_get_ip, :hosts => ip do
+        local_alias = "* " + capture("hostname").strip
+      end
+      _get_ip rescue ConnectionError
+    end
+    return local_alias
+  end
+
+  def prepare_script(name, contents)
+    script = "/tmp/#{name}"
+    # this lets us abort a script if a command in the middle of it errors out
+    env = rubber_cfg.environment.bind()
+    contents = "#{env.stop_on_error_cmd}\n#{contents}" if env.stop_on_error_cmd
+    put(contents, script)
+    return script
+  end
+
+  def run_script(name, contents)
+    script = prepare_script(name, contents)
+    run "sh #{script}"
+  end
+
+  def sudo_script(name, contents)
+    script = prepare_script(name, contents)
+    sudo "sh #{script}"
+  end
+
+  def get_env(name, desc, required=false, default=nil)
+    value = ENV.delete(name)
+    msg = "#{desc}"
+    msg << " [#{default}]" if default
+    msg << ": "
+    value = Capistrano::CLI.ui.ask(msg) unless value
+    value = value.size == 0 ? default : value
+    fatal "#{name} is required, pass using environment or enter at prompt" if required && ! value
+    return value
+  end
+
+  def fatal(msg, code=1)
+    logger.info msg
+    exit code
+  end
+
+  # Returns a map of "hostvar_<hostname>" => value for the given config value for each instance host
+  # This is used to run capistrano tasks scoped to the correct role/host so that a config value
+  # specific to a role/host will only be used for that role/host, e.g. the list of packages to
+  # be installed.
+  def get_host_options(cfg_name, &block)
+    opts = {}
+    rubber_cfg.instance.each do | ic|
+      env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+      cfg_value = env[cfg_name]
+      if cfg_value
+        if block
+          cfg_value = block.call(cfg_value)
+        end
+        opts["hostvar_#{ic.full_name}"] = cfg_value if cfg_value && cfg_value.strip.size > 0
+      end
+    end
+    return opts
+  end
+
+end

data/lib/rubber/recipes/rubber/volumes.rb

@@ -0,0 +1,263 @@
+namespace :rubber do
+
+  desc <<-DESC
+    Sets up persistent volumes in the cloud
+    All volumes defined in rubber.yml will be created if neccessary, and attached/mounted on their associated instances
+  DESC
+  required_task :setup_volumes do
+    rubber_cfg.instance.filtered.each do |ic|
+      env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+      created_vols = []
+      vol_specs = env.volumes || []
+      vol_specs.each do |vol_spec|
+        created_vols << setup_volume(ic, vol_spec)
+      end
+      created_vols.compact!
+
+      created_parts = []
+      partition_specs = env.local_volumes || []
+      partition_specs.each do |partition_spec|
+        created_parts << setup_partition(ic, partition_spec)
+      end
+      created_parts.compact!
+      zero_partitions(ic, created_parts)
+      created_vols += created_parts
+      
+      created_vols = created_vols.compact.uniq
+      raid_specs = env.raid_volumes || []
+      raid_specs.each do |raid_spec|
+        # we want to format if we created the ec2 volumes, or if we don't have any
+        # ec2 volumes and are just creating raid array from ephemeral stores
+        format = raid_spec['source_devices'].all? {|dev| created_vols.include?(dev)}
+        setup_raid_volume(ic, raid_spec, format)
+      end
+    end
+  end
+
+  desc <<-DESC
+    Shows the configured persistent volumes
+  DESC
+  required_task :describe_volumes do
+    results = []
+    format = "%-20s %-15s %-15s %-20s"
+    results << format % %w[Id Status Attached Instance]
+
+    volumes = cloud.describe_volumes()
+    volumes.each do |volume|
+      results << format % [volume[:id], volume[:status], volume[:attachment_status], volume[:attachment_instance_id]]
+    end
+
+    results.each {|r| logger.info r}
+  end
+
+  desc <<-DESC
+    Shows the configured persistent volumes
+  DESC
+  required_task :destroy_volume do
+    volume_id = get_env('VOLUME_ID', "Volume ID", true)
+    destroy_volume(volume_id)
+  end
+
+  def create_volume(size, zone)
+    volumeId = cloud.create_volume(size.to_s, zone)
+    fatal "Failed to create volume" if volumeId.nil?
+    return volumeId
+  end
+
+  def attach_volume(vol_id, instance_id, device)
+    cloud.attach_volume(vol_id, instance_id, device)
+  end
+
+  def setup_volume(ic, vol_spec)
+    created = nil
+    key = "#{ic.name}_#{vol_spec['device']}"
+    artifacts = rubber_cfg.instance.artifacts
+    vol_id = artifacts['volumes'][key]
+
+    # first create the volume if we don't have a global record (artifacts) for it
+    if ! vol_id
+      logger.info "Creating volume for #{ic.full_name}:#{vol_spec['device']}"
+      vol_id = create_volume(vol_spec['size'], vol_spec['zone'])
+      artifacts['volumes'][key] = vol_id
+      rubber_cfg.instance.save
+      created = vol_spec['device']
+    end
+
+    # then, attach it if we don't have a record (on instance) of attachment
+    ic.volumes ||= []
+    if ! ic.volumes.include?(vol_id)
+      logger.info "Attaching volume #{vol_id} to #{ic.full_name}:#{vol_spec['device']}"
+      attach_volume(vol_id, ic.instance_id, vol_spec['device'])
+      ic.volumes << vol_id
+      rubber_cfg.instance.save
+
+      print "Waiting for volume to attach"
+      while true do
+        print "."
+        sleep 2
+        volume = cloud.describe_volumes(vol_id).first
+        break if volume[:status] == "in-use"
+      end
+      print "\n"
+
+      # we don't mount/format at this time if we are doing a RAID array
+      if vol_spec['mount'] && vol_spec['filesystem']
+        # then format/mount/etc if we don't have an entry in hosts file
+        task :_setup_volume, :hosts => ic.external_ip do
+          rubber.run_script 'setup_volume', <<-ENDSCRIPT
+            if ! grep -q '#{vol_spec['mount']}' /etc/fstab; then
+              if mount | grep -q '#{vol_spec['mount']}'; then
+                umount '#{vol_spec['mount']}'
+              fi
+              mv /etc/fstab /etc/fstab.bak
+              cat /etc/fstab.bak | grep -v '#{vol_spec['mount']}' > /etc/fstab
+              echo '#{vol_spec['device']} #{vol_spec['mount']} #{vol_spec['filesystem']} noatime 0 0 # rubber volume #{vol_id}' >> /etc/fstab
+
+              #{('yes | mkfs -t ' + vol_spec['filesystem'] + ' ' + vol_spec['device']) if created}
+              mkdir -p '#{vol_spec['mount']}'
+              mount '#{vol_spec['mount']}'
+            fi
+          ENDSCRIPT
+        end
+        _setup_volume
+      end
+
+    end
+    return created
+  end
+
+  def setup_partition(ic, partition_spec)
+    created = nil
+    part_id = partition_spec['partition_device']
+
+    # Only create the partition if we haven't already done so
+    ic.partitions ||= []
+    if ! ic.partitions.include?(part_id)
+      # then format/mount/etc if we don't have an entry in hosts file
+      task :_setup_partition, :hosts => ic.external_ip do
+        rubber.run_script 'setup_partition', <<-ENDSCRIPT
+          if ! fdisk -l 2>&1 | grep -q '#{partition_spec['partition_device']}'; then
+            if grep -q '#{partition_spec['disk_device']}\\b' /etc/fstab; then
+              umount #{partition_spec['disk_device']}
+              mv /etc/fstab /etc/fstab.bak
+              cat /etc/fstab.bak | grep -v '#{partition_spec['disk_device']}\\b' > /etc/fstab
+            fi
+
+            # partition format is: Start (blank is first available),Size(MB due to -uM),Id(83=linux,82=swap,etc),Bootable
+            echo "#{partition_spec['start']},#{partition_spec['size']},#{partition_spec['type']},#{partition_spec['bootable']}" | sfdisk -L -uM #{partition_spec['disk_device']}
+          fi
+        ENDSCRIPT
+      end
+      _setup_partition
+
+      ic.partitions << part_id
+      rubber_cfg.instance.save
+      created = part_id
+
+    end
+
+    return created
+  end
+
+  def zero_partitions(ic, partitions)
+    env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+
+    # don't zero out the ones that we weren't told to
+    partitions.delete_if do |part|
+      spec = env.local_volumes.find {|s| s['partition_device'] == part}
+      ! spec['zero']
+    end
+
+    if partitions.size > 0
+      zero_script = ""
+      partitions.each do |partition|
+        zero_script << "nohup dd if=/dev/zero bs=1M of=#{partition} &> /dev/null &\n"
+      end
+      # then format/mount/etc if we don't have an entry in hosts file
+      task :_zero_partitions, :hosts => ic.external_ip do
+        rubber.run_script 'zero_partitions', <<-ENDSCRIPT
+          # zero out parition for performance (see amazon DevGuide)
+          echo "Zeroing out raid partitions to improve performance, this way take a while"
+          #{zero_script}
+
+          echo "Waiting for partitions to zero out"
+          while true; do
+            if ! ps ax | grep -q "[d]d.*/dev/zero"; then exit; fi
+            echo -n .
+            sleep 1
+          done
+        ENDSCRIPT
+      end
+      _zero_partitions
+    end
+  end
+
+  def setup_raid_volume(ic, raid_spec, create=false)
+    if create
+      mdadm_init = "mdadm --create #{raid_spec['device']} --level #{raid_spec['raid_level']} --raid-devices #{raid_spec['source_devices'].size} #{raid_spec['source_devices'].sort.join(' ')}"
+    else
+      mdadm_init = "mdadm --assemble #{raid_spec['device']} #{raid_spec['source_devices'].sort.join(' ')}"
+    end
+
+    task :_setup_raid_volume, :hosts => ic.external_ip do
+      rubber.run_script 'setup_raid_volume', <<-ENDSCRIPT
+        if ! grep -q '#{raid_spec['device']}' /etc/fstab; then
+          if mount | grep -q '#{raid_spec['mount']}'; then
+            umount '#{raid_spec['mount']}'
+          fi
+          mv /etc/fstab /etc/fstab.bak
+          cat /etc/fstab.bak | grep -v '#{raid_spec['mount']}' > /etc/fstab
+          echo '#{raid_spec['device']} #{raid_spec['mount']} #{raid_spec['filesystem']} noatime 0 0 # rubber raid volume' >> /etc/fstab
+
+          # seems to help devices initialize, otherwise mdadm fails because
+          # device not ready even though ec2 says the volume is attached
+          fdisk -l &> /dev/null
+
+          #{mdadm_init}
+
+          # set reconstruction speed
+          echo $((30*1024)) > /proc/sys/dev/raid/speed_limit_min
+
+          echo 'DEVICE /dev/hd*[0-9] /dev/sd*[0-9]' > /etc/mdadm/mdadm.conf
+          mdadm --detail --scan >> /etc/mdadm/mdadm.conf
+
+          mv /etc/rc.local /etc/rc.local.bak
+          echo "mdadm --assemble --scan" > /etc/rc.local
+          chmod +x /etc/rc.local
+
+          #{('yes | mkfs -t ' + raid_spec['filesystem'] + ' ' + raid_spec['device']) if create}
+          mkdir -p '#{raid_spec['mount']}'
+          mount '#{raid_spec['mount']}'
+        fi
+      ENDSCRIPT
+    end
+    _setup_raid_volume
+  end
+
+  def destroy_volume(volume_id)
+
+    logger.info "Detaching volume #{volume_id}"
+    cloud.detach_volume(volume_id) rescue logger.info("Volume was not attached")
+
+    print "Waiting for volume to detach"
+    while true do
+      print "."
+      sleep 2
+      volume = cloud.describe_volumes(volume_id).first
+      break if !volume || volume[:attachment_status] == "detached"
+    end
+    print "\n"
+
+    logger.info "Deleting volume #{volume_id}"
+    cloud.destroy_volume(volume_id)
+
+    logger.info "Removing volume #{volume_id} from rubber instances file"
+    artifacts = rubber_cfg.instance.artifacts
+    artifacts['volumes'].delete_if {|k,v| v == volume_id}
+    rubber_cfg.instance.each do |ic|
+      ic.volumes.delete(volume_id) if ic.volumes
+    end
+    rubber_cfg.instance.save
+  end
+  
+end