wr0ngway-rubber 1.0.0

data/lib/rubber/recipes/rubber/setup.rb

@@ -0,0 +1,357 @@
+namespace :rubber do
+
+  desc <<-DESC
+    Bootstraps instances by setting timezone, installing packages and gems
+  DESC
+  task :bootstrap do
+    set_timezone
+    link_bash
+    upgrade_packages
+    install_packages
+    setup_volumes
+    setup_gem_sources
+    install_gems
+    deploy.setup
+  end
+
+  desc <<-DESC
+    Sets up aliases for instance hostnames based on contents of instance.yml.
+    Generates /etc/hosts for local/remote machines and sets hostname on
+    remote instances, and sets values in dynamic dns entries
+  DESC
+  required_task :setup_aliases do
+    setup_local_aliases
+    setup_remote_aliases
+    setup_dns_aliases
+  end
+
+  desc <<-DESC
+    Sets up local aliases for instance hostnames based on contents of instance.yml.
+    Generates/etc/hosts for local machine
+  DESC
+  required_task :setup_local_aliases do
+    hosts_file = '/etc/hosts'
+
+    # Generate /etc/hosts contents for the local machine from instance config
+    delim = "## rubber config #{rubber_env.domain} #{RUBBER_ENV}"
+    local_hosts = delim + "\n"
+    rubber_instances.each do |ic|
+      # don't add unqualified hostname in local hosts file since user may be
+      # managing multiple domains with same aliases
+      hosts_data = [ic.full_name, ic.external_host, ic.internal_host].join(' ')
+      local_hosts << ic.external_ip << ' ' << hosts_data << "\n"
+    end
+    local_hosts << delim << "\n"
+
+    # Write out the hosts file for this machine, use sudo
+    filtered = File.read(hosts_file).gsub(/^#{delim}.*^#{delim}\n?/m, '')
+    logger.info "Writing out aliases into local machines #{hosts_file}, sudo access needed"
+    Rubber::Util::sudo_open(hosts_file, 'w') do |f|
+      f.write(filtered)
+      f.write(local_hosts)
+    end
+  end
+
+  desc <<-DESC
+    Sets up aliases in dynamic dns provider for instance hostnames based on contents of instance.yml.
+  DESC
+  required_task :setup_dns_aliases do
+    rubber_instances.each do |ic|
+      update_dyndns(ic)
+    end
+  end
+
+  desc <<-DESC
+    Sets up aliases for instance hostnames based on contents of instance.yml.
+    Generates /etc/hosts for remote machines and sets hostname on remote instances
+  DESC
+  task :setup_remote_aliases do
+    hosts_file = '/etc/hosts'
+
+    # Generate /etc/hosts contents for the remote instance from instance config
+    delim = "## rubber config"
+    delim = "#{delim} #{RUBBER_ENV}"
+    remote_hosts = delim + "\n"
+    rubber_instances.each do |ic|
+      hosts_data = [ic.name, ic.full_name, ic.external_host, ic.internal_host].join(' ')
+      remote_hosts << ic.internal_ip << ' ' << hosts_data << "\n"
+    end
+    remote_hosts << delim << "\n"
+    if rubber_instances.size > 0
+      # write out the hosts file for the remote instances
+      # NOTE that we use "capture" to get the existing hosts
+      # file, which only grabs the hosts file from the first host
+      filtered = (capture "cat #{hosts_file}").gsub(/^#{delim}.*^#{delim}\n?/m, '')
+      filtered = filtered + remote_hosts
+      # Put the generated hosts back on remote instance
+      put filtered, hosts_file
+
+      # Setup hostname on instance so shell, etcs have nice display
+      sudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
+    end
+
+    # TODO
+    # /etc/resolv.conf to add search domain
+    # ~/.ssh/options to setup user/host/key aliases
+  end
+
+  desc <<-DESC
+    Update to the newest versions of all packages/gems.
+  DESC
+  task :update do
+    upgrade_packages
+    update_gems
+  end
+
+  desc <<-DESC
+    Upgrade to the newest versions of all Ubuntu packages.
+  DESC
+  task :upgrade_packages do
+    package_helper(true)
+  end
+
+  desc <<-DESC
+    Upgrade to the newest versions of all rubygems.
+  DESC
+  task :update_gems do
+    gem_helper(true)
+  end
+
+  desc <<-DESC
+    Install extra packages and gems.
+  DESC
+  task :install do
+    install_packages
+    install_gems
+  end
+
+  desc <<-DESC
+    Install Ubuntu packages. Set 'packages' in rubber.yml to \
+    be an array of strings.
+  DESC
+  task :install_packages do
+    package_helper(false)
+  end
+
+  desc <<-DESC
+    Install ruby gems. Set 'gems' in rubber.yml to \
+    be an array of strings.
+  DESC
+  task :install_gems do
+    gem_helper(false)
+  end
+
+  desc <<-DESC
+    Install ruby gems defined in the rails environment.rb
+  DESC
+  after "deploy:symlink", "rubber:install_rails_gems" if Rubber::Util.is_rails?
+  task :install_rails_gems do
+    sudo "sh -c 'cd #{current_path} && RAILS_ENV=#{RUBBER_ENV} rake gems:install'"
+  end
+
+  desc <<-DESC
+    Setup ruby gems sources. Set 'gemsources' in rubber.yml to \
+    be an array of URI strings.
+  DESC
+  task :setup_gem_sources do
+    if rubber_env.gemsources
+      script = prepare_script 'gem_sources_helper', <<-'ENDSCRIPT'
+        ruby - $@ <<-'EOF'
+
+        sources = ARGV
+
+        installed = []
+        `gem sources -l`.grep(/^[^*]/) do |line|
+            line = line.strip
+            installed << line if line.size > 0
+        end
+
+        to_install = sources - installed
+        to_remove = installed - sources
+
+        if to_install.size > 0
+          to_install.each do |source|
+            system "gem sources -a #{source}"
+            fail "Unable to add gem sources" if $?.exitstatus > 0
+          end
+        end
+        if to_remove.size > 0
+          to_remove.each do |source|
+            system "gem sources -r #{source}"
+            fail "Unable to remove gem sources" if $?.exitstatus > 0
+          end
+        end
+
+        'EOF'
+      ENDSCRIPT
+
+      sudo "sh #{script} #{rubber_env.gemsources.join(' ')}"
+    end
+  end
+
+  desc <<-DESC
+    The ubuntu has /bin/sh linking to dash instead of bash, fix this
+    You can override this task if you don't want this to happen
+  DESC
+  task :link_bash do
+    sudo("ln -sf /bin/bash /bin/sh")
+  end
+
+  desc <<-DESC
+    Set the timezone using the value of the variable named timezone. \
+    Valid options for timezone can be determined by the contents of \
+    /usr/share/zoneinfo, which can be seen here: \
+    http://packages.ubuntu.com/cgi-bin/search_contents.pl?searchmode=filelist&word=tzdata&version=gutsy&arch=all&page=1&number=all \
+    Remove 'usr/share/zoneinfo/' from the filename, and use the last \
+    directory and file as the value. For example 'Africa/Abidjan' or \
+    'posix/GMT' or 'Canada/Eastern'.
+  DESC
+  task :set_timezone do
+    opts = get_host_options('timezone')
+    sudo "bash -c 'echo $CAPISTRANO:VAR$ > /etc/timezone'", opts
+    sudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
+    # restart syslog so that times match timezone
+    sudo "/etc/init.d/sysklogd restart"
+  end
+  
+  def update_dyndns(instance_item)
+    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
+    if env.dns_provider
+      provider = Rubber::Dns::get_provider(env.dns_provider, env)
+      provider.update(instance_item.name, instance_item.external_ip)
+    end
+  end
+
+  def destroy_dyndns(instance_item)
+    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
+    if env.dns_provider
+      provider = Rubber::Dns::get_provider(env.dns_provider, env)
+      provider.destroy(instance_item.name)
+    end
+  end
+
+  def package_helper(upgrade=false)
+    opts = get_host_options('packages') do |pkg_list|
+      expanded_pkg_list = []
+      pkg_list.each do |pkg_spec|
+        if pkg_spec.is_a?(Array)
+          expanded_pkg_list << "#{pkg_spec[0]}=#{pkg_spec[1]}"
+        else
+          expanded_pkg_list << pkg_spec
+        end
+      end
+      expanded_pkg_list.join(' ')
+    end
+
+    sudo "apt-get -q update"
+    if upgrade
+      sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes dist-upgrade'"
+    else
+      sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes install $CAPISTRANO:VAR$'", opts
+    end
+  end
+
+  def custom_package(url_base, name, ver, install_test)
+    rubber.run_script "install_#{name}", <<-ENDSCRIPT
+      if [[ #{install_test} ]]; then
+        arch=`uname -m`
+        if [ "$arch" = "x86_64" ]; then
+          src="#{url_base}/#{name}_#{ver}_amd64.deb"
+        else
+          src="#{url_base}/#{name}_#{ver}_i386.deb"
+        fi
+        src_file="${src##*/}"
+        wget -qP /tmp ${src}
+        dpkg -i /tmp/${src_file}
+      fi
+    ENDSCRIPT
+  end
+
+  def handle_gem_prompt(ch, data, str)
+    ch[:data] ||= ""
+    ch[:data] << data
+    if data =~ />\s*$/
+      logger.info data
+      logger.info "The gem command is asking for a number:"
+      choice = STDIN.gets
+      ch.send_data(choice)
+    else
+      logger.info data
+    end
+  end
+
+  # Helper for installing gems,allows one to respond to prompts
+  def gem_helper(update=false)
+    cmd = update ? "update" : "install"
+
+
+    opts = get_host_options('gems') do |gem_list|
+      expanded_gem_list = []
+      gem_list.each do |gem_spec|
+        if gem_spec.is_a?(Array)
+          expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
+        else
+          expanded_gem_list << gem_spec
+        end
+      end
+      expanded_gem_list.join(' ')
+    end
+    
+    if opts.size > 0
+      # Rubygems always installs even if the gem is already installed
+      # When providing versions, rubygems fails unless versions are provided for all gems
+      # This helper script works around these issues by installing gems only if they
+      # aren't already installed, and separates versioned/unversioned into two separate
+      # calls to rubygems
+      script = prepare_script 'gem_helper', <<-'ENDSCRIPT'
+        ruby - $@ <<-'EOF'
+
+        gem_cmd = ARGV[0]
+        gems = ARGV[1..-1]
+        cmd = "gem #{gem_cmd} --no-rdoc --no-ri"
+
+        to_install = {}
+        to_install_ver = {}
+        # gem list passed in, possibly with versions, as "gem1 gem2:1.2 gem3"
+        gems.each do |gem_spec|
+          parts = gem_spec.split(':')
+          if parts[1]
+            to_install_ver[parts[0]] = parts[1]
+          else
+            to_install[parts[0]] = true
+          end
+        end
+
+        installed = {}
+        `gem list --local`.each do |line|
+            parts = line.scan(/(.*) \((.*)\)/).first
+            next unless parts && parts.size == 2
+            installed[parts[0]] = parts[1].split(",")
+        end
+
+        to_install.delete_if {|g, v| installed.has_key?(g) } if gem_cmd == 'install'
+        to_install_ver.delete_if {|g, v| installed.has_key?(g) && installed[g].include?(v) } 
+
+        # rubygems can only do asingle versioned gem at a time so we need
+        # to do the two groups separately
+        # install versioned ones first so unversioned don't pull in a newer version
+        to_install_ver.each do |g, v|
+          system "#{cmd} #{g} -v #{v}"
+          fail "Unable to install versioned gem #{g}:#{v}" if $?.exitstatus > 0
+        end
+        if to_install.size > 0
+          gem_list = to_install.keys.join(' ')
+          system "#{cmd} #{gem_list}"
+          fail "Unable to install gems" if $?.exitstatus > 0
+        end
+
+        'EOF'
+      ENDSCRIPT
+
+      sudo "sh #{script} #{cmd} $CAPISTRANO:VAR$", opts do |ch, str, data|
+        handle_gem_prompt(ch, data, str)
+      end
+    end
+  end
+
+end

data/lib/rubber/recipes/rubber/static_ips.rb

@@ -0,0 +1,107 @@
+namespace :rubber do
+
+
+  desc <<-DESC
+    Sets up static IPs for the instances configured to have them
+  DESC
+  required_task :setup_static_ips do
+    rubber_instances.each do |ic|
+      env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+      if env.use_static_ip
+        artifacts = rubber_instances.artifacts
+        ip = artifacts['static_ips'][ic.name] rescue nil
+
+        # first allocate the static ip if we don't have a global record (artifacts) for it
+        if ! ip
+          logger.info "Allocating static IP for #{ic.full_name}"
+          ip = allocate_static_ip()
+          artifacts['static_ips'][ic.name] = ip
+          rubber_instances.save
+        end
+
+        # then, associate it if we don't have a record (on instance) of association
+        if ! ic.static_ip
+          logger.info "Associating static ip #{ip} with #{ic.full_name}"
+          associate_static_ip(ip, ic.instance_id)
+
+          instance = cloud.describe_instances(ic.instance_id).first
+          ic.external_host = instance[:external_host]
+          ic.internal_host = instance[:internal_host]
+          ic.external_ip = ip
+          ic.static_ip = ip
+          rubber_instances.save()
+
+          logger.info "Waiting for static ip to associate"
+          while true do
+            task :_wait_for_static_ip, :hosts => ip do
+              run "echo"
+            end
+            begin
+              _wait_for_static_ip
+            rescue ConnectionError
+              sleep 2
+              logger.info "Failed to connect to static ip #{ip}, retrying"
+              retry
+            end
+            break
+          end
+        end
+
+      end
+    end
+  end
+
+  desc <<-DESC
+    Shows the configured static IPs
+  DESC
+  required_task :describe_static_ips do
+    results = []
+    format = "%-10s %-15s %-30s"
+    results << format % %w[InstanceID IP Alias]
+
+    ips = cloud.describe_static_ips()
+    ips.each do |ip_data|
+      instance_id = ip_data[:instance_id]
+      ip = ip_data[:ip]
+
+      local_alias = find_alias(ip, instance_id, false)
+
+      results << format % [instance_id || "Unassigned", ip, local_alias || "Unknown"]
+    end
+    
+    results.each {|r| logger.info r}
+  end
+
+  desc <<-DESC
+    Deallocates the given static ip
+  DESC
+  required_task :destroy_static_ip do
+    ip = get_env('IP', "Static IP (run rubber:describe_static_ips for a list)", true)
+    destroy_static_ip(ip)
+  end
+
+  def allocate_static_ip()
+    ip = cloud.create_static_ip()
+    fatal "Failed to allocate static ip" if ip.nil?
+    return ip
+  end
+
+  def associate_static_ip(ip, instance_id)
+    success = cloud.attach_static_ip(ip, instance_id)
+    fatal "Failed to associate static ip" unless success
+  end
+
+  def destroy_static_ip(ip)
+    logger.info "Releasing static ip: #{ip}"
+    cloud.destroy_static_ip(ip) rescue logger.info("IP was not attached")
+
+    logger.info "Removing ip #{ip} from rubber instances file"
+    artifacts = rubber_instances.artifacts
+    artifacts['static_ips'].delete_if {|k,v| v == ip}
+    rubber_instances.each do |ic|
+      ic.static_ip = nil if ic.static_ip == ip
+    end
+    rubber_instances.save
+  end
+
+end

data/lib/rubber/recipes/rubber/utils.rb

@@ -0,0 +1,203 @@
+namespace :rubber do
+    
+  desc <<-DESC
+    Convenience task for creating a staging instance for the given RUBBER_ENV/RAILS_ENV.
+    By default this task assigns all known roles when creating the instance,
+    but you can specify a different default in rubber.yml:staging_roles
+    At the end, the instance will be up and running
+    e.g. RUBBER_ENV=matt cap create_staging
+  DESC
+  required_task :create_staging do
+    if rubber_instances.size > 0
+      value = Capistrano::CLI.ui.ask("The #{RUBBER_ENV} environment already has instances, Are you SURE you want to create a staging instance that may interact with them [y/N]?: ")
+      fatal("Exiting", 0) if value !~ /^y/
+    end
+    instance_alias = ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname to use for staging instance", true, RUBBER_ENV)
+    default_roles = rubber_env.staging_roles || "*"
+    roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
+
+    # some bootstraps update code (bootstrap_db) but if you don't have that role, need to do it here
+    # Since release directory variable gets reused by cap, we have to just do the symlink here - doing
+    # a update again will fail
+    set :rubber_code_was_updated, false
+    after "deploy:update_code" do
+      set :rubber_code_was_updated, true
+    end
+
+    if rubber_instances[instance_alias]
+      logger.info "Instance already exists, skipping to bootstrap"
+    else
+      rubber.create
+    end
+    rubber.bootstrap
+    # stop everything in case we have a bundled instance with monit, etc starting at boot
+    deploy.stop rescue nil
+    if ! rubber_code_was_updated
+      deploy.update_code
+    end
+    deploy.symlink
+    deploy.migrate
+    deploy.start
+  end
+
+  desc <<-DESC
+    Destroy the staging instance for the given RUBBER_ENV.
+  DESC
+  task :destroy_staging do
+    ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname of staging instance to be destroyed", true, RUBBER_ENV)
+    rubber.destroy
+  end
+
+  desc <<-DESC
+    Live tail of rails log files for all machines
+    By default tails the rails logs for the current RUBBER_ENV, but one can
+    set FILE=/path/file.*.glob to tails a different set
+  DESC
+  task :tail_logs, :roles => :app do
+    log_file_glob = rubber.get_env("FILE", "Log files to tail", true, "#{current_path}/log/#{RUBBER_ENV}*.log")
+    run "tail -qf #{log_file_glob}" do |channel, stream, data|
+      puts  # for an extra line break before the host name
+      puts data
+      break if stream == :err
+    end
+  end
+
+  # Use instead of task to define a capistrano task that runs serially instead of in parallel
+  # The :groups option specifies how many groups to partition the servers into so that we can
+  # do the task for N (= total/groups) servers at a time.  When multiple roles are supplied,
+  # this tries to be intelligent and slice up each role independently, but runs the slices together
+  # so that things don't take too long, e.g. adding an :api role to some :app servers, when restarting
+  # you don't want to do the api first, then the others as this would take a long time, so instead
+  # it does some :api and some :app, then some more of each
+  #
+  def serial_task(ns, name, options = {}, &block)
+    # first figure out server names for the passed in roles - when no roles
+    # are passed in, use all servers
+    serial_roles = Array(options[:roles])
+    servers = {}
+    if serial_roles.empty?
+      all_servers = []
+      self.roles.each do |rolename, serverdefs|
+        all_servers += serverdefs.collect {|server| server.host}
+      end
+      servers[:_serial_all] = all_servers.uniq.sort
+    else
+      # get servers for each role
+      self.roles.each do |rolename, serverdefs|
+        if serial_roles.include?(rolename)
+          servers[rolename] ||= []
+          servers[rolename] += serverdefs.collect {|server| server.host}
+        end
+      end
+
+      # Remove duplication of servers - roles which come first in list
+      # have precedence, so the servers show up in that group
+      serial_roles.each_with_index do |rolename, i|
+        servers[rolename] ||= []
+        serial_roles[i+1..-1].each do |r|
+          servers[r] -= servers[rolename]
+        end
+        servers[rolename] = servers[rolename].uniq.sort
+      end
+    end
+
+    # group each role's servers into slices, but combine slices across roles
+    slices = []
+    servers.each do |rolename, svrs|
+      next if svrs.size == 0
+      # figure out size of each slice by deviding server count by # of groups
+      slice_size = svrs.size / (options.delete(:groups) || 2)
+      slice_size = 1 if slice_size == 0
+      slice_idx = 0
+      svrs.each_slice(slice_size) do |srv_slice|
+        slices[slice_idx] ||= []
+        slices[slice_idx] += srv_slice
+        slice_idx += 1
+      end
+    end
+    # for each slice, define a new task specific to the hosts in that slice
+    task_syms = []
+    slices.each do |server_group|
+      servers = server_group.map{|s| s.gsub(/\..*/, '')}.join("_")
+      task_sym = "_serial_task_#{name.to_s}_#{servers}".to_sym
+      task_syms << task_sym
+      ns.task task_sym, options.merge(:hosts => server_group), &block
+    end
+
+    # create the top level task that calls all the serial ones
+    ns.task name, options do
+      task_syms.each do |t|
+        ns.send t
+      end
+    end
+  end
+
+  def find_alias(ip, instance_id, do_connect=true)
+    if instance_id
+      instance = rubber_instances.find {|i| i.instance_id == instance_id }
+      local_alias = instance.full_name if instance
+    end
+    local_alias ||= File.read("/etc/hosts").grep(/#{ip}/).first.split[1] rescue nil
+    if ! local_alias && do_connect
+      task :_get_ip, :hosts => ip do
+        local_alias = "* " + capture("hostname").strip
+      end
+      _get_ip rescue ConnectionError
+    end
+    return local_alias
+  end
+
+  def prepare_script(name, contents)
+    script = "/tmp/#{name}"
+    # this lets us abort a script if a command in the middle of it errors out
+    contents = "#{rubber_env.stop_on_error_cmd}\n#{contents}" if rubber_env.stop_on_error_cmd
+    put(contents, script)
+    return script
+  end
+
+  def run_script(name, contents)
+    script = prepare_script(name, contents)
+    run "sh #{script}"
+  end
+
+  def sudo_script(name, contents)
+    script = prepare_script(name, contents)
+    sudo "sh #{script}"
+  end
+
+  def get_env(name, desc, required=false, default=nil)
+    value = ENV.delete(name)
+    msg = "#{desc}"
+    msg << " [#{default}]" if default
+    msg << ": "
+    value = Capistrano::CLI.ui.ask(msg) unless value
+    value = value.size == 0 ? default : value
+    fatal "#{name} is required, pass using environment or enter at prompt" if required && ! value
+    return value
+  end
+
+  def fatal(msg, code=1)
+    logger.info msg
+    exit code
+  end
+
+  # Returns a map of "hostvar_<hostname>" => value for the given config value for each instance host
+  # This is used to run capistrano tasks scoped to the correct role/host so that a config value
+  # specific to a role/host will only be used for that role/host, e.g. the list of packages to
+  # be installed.
+  def get_host_options(cfg_name, &block)
+    opts = {}
+    rubber_instances.each do | ic|
+      env = rubber_cfg.environment.bind(ic.role_names, ic.name)
+      cfg_value = env[cfg_name]
+      if cfg_value
+        if block
+          cfg_value = block.call(cfg_value)
+        end
+        opts["hostvar_#{ic.full_name}"] = cfg_value if cfg_value && cfg_value.strip.size > 0
+      end
+    end
+    return opts
+  end
+
+end