wr0ngway-rubber 1.0.0

data/lib/rubber/recipes/rubber/instances.rb

@@ -0,0 +1,348 @@
+namespace :rubber do
+
+  desc <<-DESC
+    Create a new EC2 instance with the given ALIAS and ROLES
+  DESC
+  required_task :create do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+    if r == '*'
+      instance_roles = rubber_cfg.environment.known_roles
+      instance_roles = instance_roles.collect {|role| role == "db" ? "db:primary=true" : role }
+    else
+      instance_roles = r.split(",")
+    end
+
+    ir = []
+    instance_roles.each do |r|
+      role = Rubber::Configuration::RoleItem.parse(r)
+
+      # If user doesn't setup a primary db, then be nice and do it
+      if role.name == "db" && role.options["primary"] == nil && rubber_instances.for_role("db").size == 0
+        value = Capistrano::CLI.ui.ask("You do not have a primary db role, should #{instance_alias} be it [y/n]?: ")
+        role.options["primary"] = true if value =~ /^y/
+      end
+
+      ir << role
+    end
+
+    # Add in roles that the given set of roles depends on
+    ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
+
+    create_instance(instance_alias, ir)
+  end
+
+  desc <<-DESC
+    Refresh the host data for a EC2 instance with the given ALIAS.
+    This is useful to run when rubber:create fails after instance creation
+  DESC
+  task :refresh do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
+    refresh_instance(instance_alias)
+  end
+
+  desc <<-DESC
+    Destroy the EC2 instance for the given ALIAS
+  DESC
+  task :destroy do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
+    destroy_instance(instance_alias)
+  end
+
+  desc <<-DESC
+    Destroy ALL the EC2 instances for the current env
+  DESC
+  task :destroy_all do
+    rubber_instances.each do |ic|
+      destroy_instance(ic.name)
+    end
+  end
+
+  desc <<-DESC
+    Adds the given ROLES to the instance named ALIAS
+  DESC
+  required_task :add_role do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+
+    instance_roles = r.split(",")
+
+    ir = []
+    instance_roles.each do |r|
+      role = Rubber::Configuration::RoleItem.parse(r)
+      ir << role
+    end
+
+    # Add in roles that the given set of roles depends on
+    ir = Rubber::Configuration::RoleItem.expand_role_dependencies(ir, get_role_dependencies)
+
+    instance = rubber_instances[instance_alias]
+    fatal "Instance does not exist: #{instance_alias}" unless instance
+
+    instance.roles = (instance.roles + ir).uniq
+    rubber_instances.save()
+  end
+
+  desc <<-DESC
+    Removes the given ROLES from the instance named ALIAS
+  DESC
+  required_task :remove_role do
+    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    r = get_env('ROLES', "Instance roles (e.g. web,app,db:primary=true)", true)
+
+    instance_roles = r.split(",")
+
+    ir = []
+    instance_roles.each do |r|
+      role = Rubber::Configuration::RoleItem.parse(r)
+      ir << role
+    end
+
+    instance = rubber_instances[instance_alias]
+    fatal "Instance does not exist: #{instance_alias}" unless instance
+
+    instance.roles = (instance.roles - ir).uniq
+    rubber_instances.save()
+  end
+
+  desc <<-DESC
+    List all your EC2 instances
+  DESC
+  required_task :describe do
+    results = []
+    format = "%-10s %-10s %-10s %-15s %-30s"
+    results << format % %w[InstanceID State Zone IP Alias\ (*=unknown)]
+
+    instances = cloud.describe_instances()
+    instances.each do |instance|
+      local_alias = find_alias(instance[:external_ip], instance[:id], instance[:state] == 'running')
+      results << format % [instance[:id], instance[:state], instance[:zone], instance[:external_ip] || "NoIP", local_alias || "Unknown"]
+    end
+    results.each {|r| logger.info r}
+  end
+
+  desc <<-DESC
+    Describes the availability zones
+  DESC
+  required_task :describe_zones do
+    results = []
+    format = "%-20s %-15s"
+    results << format % %w[Name State]
+
+    zones = cloud.describe_availability_zones()
+    zones.each do |zone|
+      results << format % [zone[:name], zone[:state]]
+    end
+
+    results.each {|r| logger.info r}
+  end
+
+
+  set :print_ip_command, "ifconfig eth0 | awk 'NR==2 {print $2}' | awk -F: '{print $2}'"
+
+  # Creates a new ec2 instance with the given alias and roles
+  # Configures aliases (/etc/hosts) on local and remote machines
+  def create_instance(instance_alias, instance_roles)
+    fatal "Instance already exists: #{instance_alias}" if rubber_instances[instance_alias]
+
+    role_names = instance_roles.collect{|x| x.name}
+    env = rubber_cfg.environment.bind(role_names, instance_alias)
+
+    # We need to use security_groups during create, so create them up front
+    setup_security_groups(instance_alias, role_names)
+    security_groups = get_assigned_security_groups(instance_alias, role_names)
+
+    ami = env.cloud_providers[env.cloud_provider].image_id
+    ami_type = env.cloud_providers[env.cloud_provider].image_type
+    availability_zone = env.availability_zone
+    logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
+    instance_id = cloud.create_instance(ami, ami_type, security_groups, availability_zone)
+
+    logger.info "Instance #{instance_id} created"
+
+    instance_item = Rubber::Configuration::InstanceItem.new(instance_alias, env.domain, instance_roles, instance_id, security_groups)
+    rubber_instances.add(instance_item)
+    rubber_instances.save()
+
+
+    print "Waiting for instance to start"
+    while true do
+      print "."
+      sleep 2
+      instance = cloud.describe_instances(instance_id).first
+
+      if instance[:state] == "running"
+        print "\n"
+        logger.info "Instance running, fetching hostname/ip data"
+        instance_item.external_host = instance[:external_host]
+        instance_item.external_ip = instance[:external_ip]
+        instance_item.internal_host = instance[:internal_host]
+        instance_item.zone = instance[:zone]
+        rubber_instances.save()
+
+        # setup amazon elastic ips if configured to do so
+        setup_static_ips
+
+        # Need to setup aliases so ssh doesn't give us errors when we
+        # later try? to connect to same ip but using alias
+        setup_local_aliases
+
+        # re-load the roles since we may have just defined new ones
+        load_roles() unless env.disable_auto_roles
+
+        # Connect to newly created instance and grab its internal ip
+        # so that we can update all aliases
+
+        task :_get_ip, :hosts => instance_item.external_ip do
+          instance_item.internal_ip = capture(print_ip_command).strip
+          rubber_instances.save()
+        end
+
+        # even though instance is running, sometimes ssh hasn't started yet,
+        # so retry on connect failure
+        begin
+          _get_ip
+        rescue ConnectionError
+          sleep 2
+          logger.info "Failed to connect to #{instance_alias} (#{instance_item.external_ip}), retrying"
+          retry
+        end
+
+        # Add the aliases for this instance to all other hosts
+        setup_remote_aliases
+        setup_dns_aliases
+
+        break
+      end
+    end
+  end
+
+  # Refreshes a ec2 instance with the given alias
+  # Configures aliases (/etc/hosts) on local and remote machines
+  def refresh_instance(instance_alias)
+    instance_item = rubber_instances[instance_alias]
+
+    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+
+    env = rubber_cfg.environment.bind(instance_item.role_names, instance_alias)
+    
+    instance = cloud.describe_instances(instance_item.instance_id).first
+
+    if instance[:state] == "running"
+      logger.info "\nInstance running, fetching hostname/ip data"
+      instance_item.external_host = instance[:external_host]
+      instance_item.external_ip = instance[:external_ip]
+      instance_item.internal_host = instance[:internal_host]
+      instance_item.zone = instance[:zone]
+
+      # setup amazon elastic ips if configured to do so
+      setup_static_ips
+      
+      # Need to setup aliases so ssh doesn't give us errors when we
+      # later try to connect to same ip but using alias
+      setup_local_aliases
+
+      # re-load the roles since we may have just defined new ones
+      load_roles() unless env.disable_auto_roles
+
+      # Connect to newly created instance and grab its internal ip
+      # so that we can update all aliases
+      task :_get_ip, :hosts => instance_item.external_ip do
+        instance_item.internal_ip = capture(print_ip_command).strip
+        rubber_instances.save()
+      end
+
+      # even though instance is running, sometimes ssh hasn't started yet,
+      # so retry on connect failure
+      begin
+        _get_ip
+      rescue ConnectionError
+        sleep 2
+        logger.info "Failed to connect to #{instance_alias} (#{instance_item.external_ip}), retrying"
+        retry
+      end
+
+
+      # Add the aliases for this instance to all other hosts
+      setup_remote_aliases
+      setup_dns_aliases
+    end
+  end
+
+
+  # Destroys the given ec2 instance
+  def destroy_instance(instance_alias)
+    instance_item = rubber_instances[instance_alias]
+    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+
+    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
+
+    value = Capistrano::CLI.ui.ask("About to DESTROY #{instance_alias} (#{instance_item.instance_id}) in mode #{RUBBER_ENV}.  Are you SURE [yes/NO]?: ")
+    fatal("Exiting", 0) if value != "yes"
+
+    if instance_item.static_ip
+      value = Capistrano::CLI.ui.ask("Instance has a static ip, do you want to release it? [y/N]?: ")
+      destroy_static_ip(instance_item.static_ip) if value =~ /^y/
+    end
+
+    if instance_item.volumes
+      value = Capistrano::CLI.ui.ask("Instance has persistent volumes, do you want to destroy them? [y/N]?: ")
+      if value =~ /^y/
+        instance_item.volumes.clone.each do |volume_id|
+          destroy_volume(volume_id)
+        end
+      end
+    end
+
+    logger.info "Destroying instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
+
+    cloud.destroy_instance(instance_item.instance_id)
+
+    rubber_instances.remove(instance_alias)
+    rubber_instances.save()
+
+    # re-load the roles since we just removed some and setup_remote_aliases
+    # shouldn't hit removed ones
+    load_roles() unless env.disable_auto_roles
+
+    setup_aliases
+    destroy_dyndns(instance_item)
+    cleanup_known_hosts(instance_item) unless env.disable_known_hosts_cleanup
+  end
+
+
+  # delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias
+  def cleanup_known_hosts(instance_item)
+    logger.info "Cleaning ~/.ssh/known_hosts"
+    File.open(File.expand_path('~/.ssh/known_hosts'), 'r+') do |f|
+        out = ""
+        f.each do |line|
+          line = case line
+            when /^ec2-/; ''
+            when /#{instance_item.full_name}/; ''
+            when /#{instance_item.external_host}/; ''
+            when /#{instance_item.external_ip}/; ''
+            else line;
+          end
+          out << line
+        end
+        f.pos = 0
+        f.print out
+        f.truncate(f.pos)
+    end
+  end
+
+  def get_role_dependencies
+    # convert string format of role_dependencies from rubber.yml into
+    # objects for use by expand_role_dependencies
+    deps = {}
+    rubber_env.role_dependencies.each do |k, v|
+      rhs = Array(v).collect {|r| Rubber::Configuration::RoleItem.parse(r)}
+      deps[Rubber::Configuration::RoleItem.parse(k)] = rhs
+    end if rubber_env.role_dependencies
+    return deps
+  end
+  
+end

data/lib/rubber/recipes/rubber/load_balancers.rb

@@ -0,0 +1,44 @@
+namespace :rubber do
+
+  #desc <<-DESC
+  #  Sets up the network load balancers
+  #DESC
+  #required_task :setup_load_balancers do
+  #  setup_load_balancers()
+  #end
+  #
+  #desc <<-DESC
+  #  Describes the network load balancers
+  #DESC
+  #required_task :describe_load_balancers do
+  #  lbs = cloud.describe_load_balancers()
+  #  pp lbs
+  #end
+
+  def setup_load_balancers
+    # OPTIONAL: Automatically provision and assign instances to a Cloud provided
+    # load balancer.
+    #load_balancers:
+    #  my_lb_name:
+    #    listeners:
+    #      - protocol: http
+    #        port: 80
+    #        instance_port: 8080
+    #      - protocol: tcp
+    #        port: 443
+    #        instance_port: 8080
+    #    target_roles: [app]
+    #
+    #isolate_load_balancers: true
+
+
+
+    # get remote lbs
+    # for each local not in remote, add it
+    #   get all zones for all instances for roles, and make sure in lb
+    #   warn if lb not balanced (count of instances per zone is equal)
+    # for each local that is in remote, sync listeners and zones
+    # for each remote not in local, remove it
+  end
+
+end

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -0,0 +1,189 @@
+namespace :rubber do
+
+  desc <<-DESC
+    Sets up the network security groups
+    All defined groups will be created, and any not defined will be removed.
+    Likewise, rules within a group will get created, and those not will be removed
+  DESC
+  required_task :setup_security_groups do
+    setup_security_groups()
+  end
+
+  desc <<-DESC
+    Describes the network security groups
+  DESC
+  required_task :describe_security_groups do
+    groups = cloud.describe_security_groups()
+    groups.each do |group|
+      puts "#{group[:name]}, #{group[:description]}"
+      group[:permissions].each do |perm|
+        puts "  protocol: #{perm[:protocol]}"
+        puts "  from_port: #{perm[:from_port]}"
+        puts "  to_port: #{perm[:to_port]}"
+        puts "  source_groups: #{perm[:source_groups].collect {|g| g[:name]}.join(", ") }" if perm[:source_groups]
+        puts "  source_ips: #{perm[:source_ips].join(", ") }" if perm[:source_ips]
+        puts "\n"
+      end if group[:permissions]
+      puts "\n"
+    end
+  end
+
+
+  def get_assigned_security_groups(host=nil, roles=[])
+    env = rubber_cfg.environment.bind(roles, host)
+    security_groups = env.assigned_security_groups
+    if env.auto_security_groups
+      security_groups << host
+      security_groups += roles
+    end
+    security_groups = security_groups.uniq.compact
+    security_groups = security_groups.collect {|x| isolate_group_name(x) } if env.isolate_security_groups
+    return security_groups
+  end
+
+  def setup_security_groups(host=nil, roles=[])
+    env = rubber_cfg.environment.bind(roles, host)
+    security_group_defns = env.security_groups
+    if env.auto_security_groups
+      sghosts = (rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
+      sgroles = (rubber_instances.all_roles + roles).uniq.compact
+      security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
+      sync_security_groups(security_group_defns)
+    else
+      sync_security_groups(security_group_defns)
+    end
+  end
+
+  def inject_auto_security_groups(groups, hosts, roles)
+    hosts.each do |name|
+      group_name = name
+      groups[group_name] ||= {'description' => "Rubber automatic security group for host: #{name}", 'rules' => []}
+    end
+    roles.each do |name|
+      group_name = name
+      groups[group_name] ||= {'description' => "Rubber automatic security group for role: #{name}", 'rules' => []}
+    end
+    return groups
+  end
+
+  def isolate_prefix
+    return "#{rubber_env.app_name}_#{RUBBER_ENV}_"
+  end
+
+  def isolate_group_name(group_name)
+    new_name = "#{isolate_prefix}#{group_name}"
+    return new_name
+  end
+
+  def isolate_groups(groups)
+    renamed = {}
+    groups.each do |name, group|
+      new_name = name =~ /^#{isolate_prefix}/ ? name : isolate_group_name(name)
+      new_group =  Marshal.load(Marshal.dump(group))
+      new_group['rules'].each do |rule|
+        old_ref_name = rule['source_group_name']
+        if old_ref_name && old_ref_name !~ /^#{isolate_prefix}/
+          rule['source_group_name'] = isolate_group_name(old_ref_name)
+        end
+      end
+      renamed[new_name] = new_group
+    end
+    return renamed
+  end
+
+  def sync_security_groups(groups)
+    return unless groups
+
+    groups = Rubber::Util::stringify(groups)
+    groups = isolate_groups(groups) if rubber_env.isolate_security_groups
+    group_keys = groups.keys.clone()
+    
+    # For each group that does already exist in cloud
+    cloud_groups = cloud.describe_security_groups()
+    cloud_groups.each do |cloud_group|
+      group_name = cloud_group[:name]
+
+      # skip those groups that don't belong to this project/env
+      next if rubber_env.isolate_security_groups && group_name !~ /^#{isolate_prefix}/
+
+      if group_keys.delete(group_name)
+        # sync rules
+        logger.debug "Security Group already in cloud, syncing rules: #{group_name}"
+        group = groups[group_name]
+        rules = group['rules'].clone
+        rule_maps = []
+
+        # first collect the rule maps from the request (group/user pairs are duplicated for tcp/udp/icmp,
+        # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
+        cloud_group[:permissions].each do |rule|
+          if rule[:source_groups]
+            rule.source_groups.each do |source_group|
+              rule_map = {:source_group_name => source_group[:name], :source_group_account => source_group[:account]}
+              rule_map = Rubber::Util::stringify(rule_map)
+              rule_maps << rule_map unless rule_maps.include?(rule_map)
+            end
+          else
+            rule_map = Rubber::Util::stringify(rule)
+            rule_maps << rule_map unless rule_maps.include?(rule_map)
+          end
+        end if cloud_group[:permissions]
+        # For each rule, if it exists, do nothing, otherwise remove it as its no longer defined locally
+        rule_maps.each do |rule_map|
+          if rules.delete(rule_map)
+            # rules match, don't need to do anything
+            # logger.debug "Rule in sync: #{rule_map.inspect}"
+          else
+            # rules don't match, remove them from cloud and re-add below
+            answer = Capistrano::CLI.ui.ask("Rule '#{rule_map.inspect}' exists in cloud, but not locally, remove from cloud? [y/N]?: ")
+            rule_map = Rubber::Util::symbolize_keys(rule_map)
+            if rule_map[:source_group_name]
+              cloud.remove_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+            else
+              rule_map[:source_ips].each do |source_ip|
+                cloud.remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+              end if rule_map[:source_ips] && answer =~ /^y/
+            end
+          end
+        end
+
+        rules.each do |rule_map|
+          # create non-existing rules
+          logger.debug "Missing rule, creating: #{rule_map.inspect}"
+          rule_map = Rubber::Util::symbolize_keys(rule_map)
+          if rule_map[:source_group_name]
+            cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+          else
+            rule_map[:source_ips].each do |source_ip|
+              cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+            end if rule_map[:source_ips]
+          end
+        end
+      else
+        # delete group
+        answer = Capistrano::CLI.ui.ask("Security group '#{group_name}' exists in cloud but not locally, remove from cloud? [y/N]: ")
+        cloud.destroy_security_group(group_name) if answer =~ /^y/
+      end
+    end
+
+    # For each group that didnt already exist in cloud
+    group_keys.each do |group_name|
+      group = groups[group_name]
+      logger.debug "Creating new security group: #{group_name}"
+      # create each group
+      cloud.create_security_group(group_name, group['description'])
+      # create rules for group
+      group['rules'].each do |rule_map|
+        logger.debug "Creating new rule: #{rule_map.inspect}"
+        rule_map = Rubber::Util::symbolize_keys(rule_map)
+        if rule_map[:source_group_name]
+          cloud.add_security_group_rule(group_name, nil, nil, nil, {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+        else
+          rule_map[:source_ips].each do |source_ip|
+            cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+          end if rule_map[:source_ips]
+        end
+      end
+    end
+  end
+
+end