awsborn 0.7.0 → 0.8.0

data/README.mdown

@@ -77,12 +77,12 @@ equally well be added as options to the specific servers.
 
 ### Defining a cluster
 
-The `cluster` method accepts two commands, `domain` and `server`.  `domain` is
-optional and is just a way to avoid repetition in the `server` commands.  `server`
+The `cluster` method accepts three commands, `domain`, `load_balancer` and `server`.  `domain` is
+optional and is just a way to avoid repetition in the `server` commands.  `load_balancer` is also optional and take a name and a hash. `server`
 takes a name (which can be used as a key in the cluster, e.g. `log_servers[:log_a]`)
 and a hash:
 
-Mandatory keys:
+Mandatory keys for server:
 
 * `:zone` -- the availability zone for the server.  One of `:us_east_1a`, `:us_east_1b`,
   `:us_east_1c`, `:us_east_1d`, `:us_west_1a`, `:us_west_1b`, `:eu_west_1a`, `:eu_west_1b`,
@@ -92,7 +92,7 @@ Mandatory keys:
   `[volume-id, :format]`, in which case `the_server.format_disk_on_device?(device)`
   will return `true`.  See `contrib/cookbooks/ec2-ebs` for example usage.
 
-Optional keys:
+Optional keys for server:
 
 * `:ip` -- a domain name which translates to an elastic ip.  If the domain name does not
   contain a full stop (dot) and `domain` has been specified above, the domain is added.
@@ -106,6 +106,40 @@ Keys that override server type settings:
 * `:bootstrap_script`
 * `:image_id`
 
+Mandatory keys for load_balancer:
+
+* `:region` -- the region covered by the load balancer. One of `:us_east_1`, `:us_west_1`, `:eu_west_1`, `:ap_southeast_1`
+
+Optional keys for load_balancer:
+
+* `:only` -- A list of server names to which the load balancer is
+	restricted
+* `:except` -- A list of servers that the load balancer should ignore
+* `:listerners` -- A list of listeners definitions for the load
+	balancer. For instance: `{ :protocol => :http, :load_balancer_port => 80,  :instance_port => 80 }`
+* `:sticky_cookies` -- A list of sticky cookies policies that the load balancer
+	should take care of. There are three kinds of available policies: `:disabled`, `:app_generated` (the cookie is generated by the app) or `:lb_generated` (the cookie is generated by the load balancer).
+
+Example:
+
+    cluster "test" do
+      domain 'yourdimain.net'
+      load_balancer 'test-balancer',
+        :region => :eu_west_1,
+        :only => [:cluster_test_1, :cluster_test_2],
+        :listeners => [
+          { :protocol => :http, :load_balancer_port => 80,  :instance_port => 80},
+          { :protocol => :tcp,  :load_balancer_port => 443, :instance_port => 443}
+        ],
+        :sticky_cookies => [
+          { :ports => [80], :policy => :disabled }
+          #{ :ports => [80], :policy => :app_generated, :cookie_name => 'some_cookie' }
+          #{ :ports => [80], :policy => :lb_generated, :expiration_period => 10 * 60 }
+        ]
+      server :cluster_test_1, :zone => :eu_west_1a, :disk => {:sdf => "vol-12345678"}, :instance_type => :m1_small
+      server :cluster_test_2, :zone => :eu_west_1a, :disk => {:sdf => "vol-09876543"}, :instance_type => :m1_small
+    end
+
 ### Launching a cluster
 
 The `launch` method on the cluster checks to see if each server is running by checking

data/VERSION

@@ -1 +1 @@
-0.7.0
+0.8.0

data/awsborn.gemspec

@@ -0,0 +1,80 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{awsborn}
+  s.version = "0.8.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["David Vrensk"]
+  s.date = %q{2011-08-11}
+  s.description = %q{Awsborn lets you define and launch a server cluster on Amazon EC2.}
+  s.email = %q{david@icehouse.se}
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.mdown"
+  ]
+  s.files = [
+    ".document",
+    "LICENSE",
+    "README.mdown",
+    "Rakefile",
+    "VERSION",
+    "awsborn.gemspec",
+    "contrib/chef-bootstrap.sh",
+    "contrib/cookbooks/ec2-ebs/recipes/default.rb",
+    "lib/awsborn.rb",
+    "lib/awsborn/aws_constants.rb",
+    "lib/awsborn/awsborn.rb",
+    "lib/awsborn/ec2.rb",
+    "lib/awsborn/elb.rb",
+    "lib/awsborn/extensions/object.rb",
+    "lib/awsborn/extensions/proc.rb",
+    "lib/awsborn/git_branch.rb",
+    "lib/awsborn/keychain.rb",
+    "lib/awsborn/known_hosts_updater.rb",
+    "lib/awsborn/load_balancer.rb",
+    "lib/awsborn/rake.rb",
+    "lib/awsborn/server.rb",
+    "lib/awsborn/server_cluster.rb",
+    "spec/aws_constants_spec.rb",
+    "spec/ec2_spec.rb",
+    "spec/elb_spec.rb",
+    "spec/load_balancer_spec.rb",
+    "spec/server_spec.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/icehouse/awsborn}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Awsborn defines servers as instances with a certain disk volume, which makes it easy to restart missing servers.}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<right_aws>, [">= 2.1.0"])
+      s.add_runtime_dependency(%q<json_pure>, [">= 1.2.3"])
+      s.add_runtime_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.6.0"])
+      s.add_development_dependency(%q<webmock>, [">= 1.3.0"])
+    else
+      s.add_dependency(%q<right_aws>, [">= 2.1.0"])
+      s.add_dependency(%q<json_pure>, [">= 1.2.3"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 2.6.0"])
+      s.add_dependency(%q<webmock>, [">= 1.3.0"])
+    end
+  else
+    s.add_dependency(%q<right_aws>, [">= 2.1.0"])
+    s.add_dependency(%q<json_pure>, [">= 1.2.3"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 2.6.0"])
+    s.add_dependency(%q<webmock>, [">= 1.3.0"])
+  end
+end
+

data/lib/awsborn/aws_constants.rb

@@ -0,0 +1,59 @@
+module Awsborn
+  module AwsConstants
+
+    AVAILABILITY_ZONES = %w[
+      us-east-1a us-east-1b us-east-1c us-east-1d
+      us-west-1a us-west-1b
+      eu-west-1a eu-west-1b
+      ap-southeast-1a ap-southeast-1b
+      ap-northeast-1a ap-northeast-1b
+    ]
+    REGIONS = AVAILABILITY_ZONES.map{|z| z.sub(/[a-z]$/,'') }.uniq
+    INSTANCE_TYPES_32_BIT = %w[m1.small c1.medium t1.micro]
+    INSTANCE_TYPES_64_BIT = %w[m1.large m1.xlarge m2.xlarge m2.2xlarge m2.4xlarge c1.xlarge cc1.4xlarge t1.micro]
+    INSTANCE_TYPES = (INSTANCE_TYPES_32_BIT + INSTANCE_TYPES_64_BIT).uniq
+    SYMBOL_CONSTANT_MAP = (AVAILABILITY_ZONES + INSTANCE_TYPES).inject({}) { |memo,str| memo[str.tr('-.','_').to_sym] = str; memo }
+
+    def endpoint_for_zone_and_service (zone, service)
+      region = zone_to_awz_region(zone)
+      case service
+      when :ec2 then "https://#{region}.ec2.amazonaws.com"
+      when :elb then "https://#{region}.elasticloadbalancing.amazonaws.com"
+      end
+    end
+
+    def zone_to_awz_region (zone)
+      region = zone.to_s.sub(/[a-z]$/,'').tr('_','-')
+      raise UnknownConstantError, "Unknown region: #{region} for zone: #{zone}" unless REGIONS.include? region
+      region
+    end
+
+    def symbol_to_aws_zone (symbol)
+      zone = symbol.to_s.tr('_','-')
+      raise UnknownConstantError, "Unknown availability zone: #{zone}" unless AVAILABILITY_ZONES.include? zone
+      zone
+    end
+
+    def aws_zone_to_symbol (zone)
+      raise UnknownConstantError, "Unknown availability zone: #{zone}" unless AVAILABILITY_ZONES.include? zone
+      zone.to_s.tr('-','_').to_sym
+    end
+
+    def symbol_to_aws_instance_type (symbol)
+      type = symbol.to_s.tr('_','.')
+      raise UnknownConstantError, "Unknown instance type: #{type}" unless INSTANCE_TYPES.include? type
+      type
+    end
+
+    def aws_instance_type_to_symbol (type)
+      raise UnknownConstantError, "Unknown instance type: #{type}" unless INSTANCE_TYPES.include? type
+      type.to_s.tr('.','_').to_sym
+    end
+
+    def awz_constant (symbol)
+      SYMBOL_CONSTANT_MAP[symbol] || raise(UnknownConstantError, "Unknown constant: #{symbol}")
+    end
+  end
+
+  class UnknownConstantError < Exception; end
+end

data/lib/awsborn/ec2.rb

@@ -1,17 +1,11 @@
 module Awsborn
   class Ec2
     extend Forwardable
+    include Awsborn::AwsConstants
     def_delegators :Awsborn, :logger
 
     attr_accessor :instance_id
 
-    class << self
-      def endpoint_for_zone (zone)
-        zone = zone.to_s.sub(/[a-z]$/,'').tr('_','-')
-        "https://#{zone}.ec2.amazonaws.com"
-      end
-    end
-
     def connection
       unless @connection
         env_ec2_url = ENV['EC2_URL']
@@ -28,7 +22,7 @@ module Awsborn
     end
 
     def initialize (zone)
-      @endpoint = self.class.endpoint_for_zone(zone)
+      @endpoint = endpoint_for_zone_and_service(zone, :ec2)
     end
 
     KeyPair = Struct.new :name, :path

data/lib/awsborn/elb.rb

@@ -0,0 +1,127 @@
+module Awsborn
+  class Elb
+    extend Forwardable
+    def_delegators :Awsborn, :logger
+    include Awsborn::AwsConstants
+    attr_accessor :region, :endpoint
+
+    def connection
+      unless @connection
+        env_ec2_url = ENV['ELB_URL']
+        begin
+          ENV['ELB_URL'] = @endpoint
+          @connection = RightAws::ElbInterface.new(
+            Awsborn.access_key_id,
+            Awsborn.secret_access_key,
+            :logger => Awsborn.logger
+          )
+        ensure
+          ENV['ELB_URL'] = env_ec2_url
+        end
+      end
+      @connection
+    end
+
+    def initialize (zone)
+      @region = zone_to_awz_region(zone)
+      @endpoint = endpoint_for_zone_and_service(@region, :elb)
+    end
+
+    def describe_load_balancer (balancer_name)
+      connection.describe_load_balancers(balancer_name).first
+    end
+
+    def running? (balancer_name)
+      describe_load_balancer(balancer_name)
+      true
+    rescue RightAws::AwsError
+      false
+    end
+
+    def dns_name (balancer_name)
+      describe_load_balancer(balancer_name)[:dns_name]
+    end
+
+    def instances (balancer_name)
+      describe_load_balancer(balancer_name)[:instances]
+    end
+
+    def zones (balancer_name)
+      describe_load_balancer(balancer_name)[:availability_zones]
+    end
+
+    def create_load_balancer (balancer_name)
+      logger.debug "Creating load balancer #{balancer_name}"
+      connection.create_load_balancer(balancer_name, [@region+'a'], [])
+    end
+
+    def set_load_balancer_listeners (balancer_name, listeners)
+      logger.debug "Setting listeners on load balancer #{balancer_name}"
+      description = describe_load_balancer(balancer_name)
+      previous_ports = description[:listeners].map{|i| i[:instance_port]}
+      connection.delete_load_balancer_listeners(balancer_name, *previous_ports) unless previous_ports.empty?
+      connection.create_load_balancer_listeners(balancer_name, listeners)
+    end
+
+    def set_lb_cookie_policy (balancer_name, ports, expiration_period)
+      logger.debug "Setting cookie policies for ports #{ports.inspect} on load balancer #{balancer_name}"
+      policy_name = "lb-#{balancer_name}-#{expiration_period}".tr('_','-')
+      unless existing_lb_cookie_policies(balancer_name).include?(policy_name)
+        connection.create_lb_cookie_stickiness_policy(balancer_name, policy_name, expiration_period)
+      end
+      ports.each do |port|
+        connection.set_load_balancer_policies_of_listener(balancer_name, port, policy_name)
+      end
+    end
+
+    def set_app_cookie_policy (balancer_name, ports, cookie_name)
+      logger.debug "Setting cookie policies for ports #{ports.inspect} on load balancer #{balancer_name}"
+      policy_name = "app-#{balancer_name}-#{cookie_name}".tr('_','-')
+      unless existing_app_cookie_policies(balancer_name).include?(policy_name)
+        connection.create_app_cookie_stickiness_policy(balancer_name, policy_name, cookie_name)
+      end
+      ports.each do |port|
+        connection.set_load_balancer_policies_of_listener(balancer_name, port, policy_name)
+      end
+    end
+
+    def remove_all_cookie_policies(balancer_name)
+      description = describe_load_balancer(balancer_name)
+      description[:listeners].each do |listener|
+        connection.set_load_balancer_policies_of_listener(balancer_name, listener[:load_balancer_port])
+      end
+    end
+
+    def register_instances (balancer_name, instances)
+      logger.debug "Registering instances #{instances.inspect} on load balancer #{balancer_name}"
+      connection.register_instances_with_load_balancer(balancer_name, *instances)
+    end
+
+    def deregister_instances (balancer_name, instances)
+      logger.debug "De-registering instances #{instances.inspect} on load balancer #{balancer_name}"
+      connection.deregister_instances_with_load_balancer(balancer_name, *instances)
+    end
+
+    def enable_zones (balancer_name, zones)
+      logger.debug "Enabling zones #{zones.inspect} on load balancer #{balancer_name}"
+      connection.enable_availability_zones_for_load_balancer(balancer_name, *zones)
+    end
+
+    def disable_zones (balancer_name, zones)
+      logger.debug "Disabling zones #{zones.inspect} on load balancer #{balancer_name}"
+      connection.disable_availability_zones_for_load_balancer(balancer_name, *zones)
+    end
+
+    protected
+
+    def existing_app_cookie_policies (balancer_name)
+      description = describe_load_balancer(balancer_name)
+      description[:app_cookie_stickiness_policies].map {|p| p[:policy_name]}.uniq
+    end
+
+    def existing_lb_cookie_policies (balancer_name)
+      description = describe_load_balancer(balancer_name)
+      description[:lb_cookie_stickiness_policies].map {|p| p[:policy_name]}.uniq
+    end
+  end
+end

data/lib/awsborn/load_balancer.rb

@@ -0,0 +1,132 @@
+module Awsborn
+  class LoadBalancer
+
+    include Awsborn::AwsConstants
+    attr_accessor :name, :only, :except, :region, :listeners, :sticky_cookies
+
+    DEFAULT_LISTENERS = [ { :protocol => :http, :load_balancer_port => 80, :instance_port => 80} ]
+    def initialize (name, options={})
+      @name = name
+      @only   = options[:only] || []
+      @except = options[:except] || []
+      @region = zone_to_awz_region(options[:region])
+      @listeners = options[:listeners] || DEFAULT_LISTENERS
+      @sticky_cookies = options[:sticky_cookies] || []
+      launch unless running?
+    end
+
+    def elb
+      @elb ||= Elb.new(@region)
+    end
+
+    def dns_name
+      elb.dns_name(@name)
+    end
+
+    def instances
+      elb.instances(@name)
+    end
+
+    def instances= (new_instances)
+      previous_instances = self.instances
+      register_instances(new_instances - previous_instances)
+      deregister_instances(previous_instances - new_instances)
+      self.instances
+    end
+
+    def zones
+      elb.zones(@name)
+    end
+
+    def zones= (new_zones)
+      previous_zones = self.zones
+      enable_zones(new_zones - previous_zones)
+      disable_zones(previous_zones - new_zones)
+      self.zones
+    end
+
+    def description
+      elb.describe_load_balancer(@name)
+    end
+
+    def running?
+      elb.running?(@name)
+    end
+
+    def launch
+      elb.create_load_balancer(@name)
+    end
+
+
+    def update_listeners
+      elb.set_load_balancer_listeners(@name, @listeners)
+    end
+
+    def update_sticky_cookies
+      elb.remove_all_cookie_policies(@name)
+      @sticky_cookies.each do |sc|
+        raise "Option :ports is missing for #{sc.inspect}" if sc[:ports].nil?
+
+        case sc[:policy]
+        when :disabled
+          set_disabled_cookie_policy(sc[:ports])
+        when :lb_generated
+          set_lb_cookie_policy(sc[:ports], sc[:expiration_period])
+        when :app_generated
+          set_app_cookie_policy(sc[:ports], sc[:cookie_name])
+        else
+          raise "unknown policy #{sc.inspect}. Accepted policies => :disabled, :lb_generated, :app_generated"
+        end
+      end
+    end
+
+    def update_with (new_servers)
+      servers_to_be_balanced = new_servers
+      servers_to_be_balanced =
+        servers_to_be_balanced.select{|s| @only.include?(s.name)}    unless @only.empty?
+      servers_to_be_balanced =
+        servers_to_be_balanced.reject{|s| @except.include?(s.name)}  unless @except.empty?
+
+      self.instances = servers_to_be_balanced.map{|s| s.instance_id }
+      self.zones     = servers_to_be_balanced.map{|s| symbol_to_aws_zone(s.zone) }.uniq
+
+      update_listeners
+      update_sticky_cookies
+
+      self.description
+    end
+
+    protected
+
+    def set_disabled_cookie_policy(ports)
+      # Do nothing
+    end
+
+    def set_app_cookie_policy(ports, cookie_name)
+      raise ":cookie_name is missing" if cookie_name.nil?
+      elb.set_app_sticky_cookie(@name, ports, cookie_name)
+    end
+
+    def set_lb_cookie_policy(ports, expiration_period)
+      raise ":expiration_period is missing" if expiration_period.nil?
+      elb.set_lb_sticky_cookie(@name, ports, expiration_period.to_i)
+    end
+
+    def register_instances (instances)
+      elb.register_instances(@name, instances) unless instances.empty?
+    end
+
+    def deregister_instances (instances)
+      elb.deregister_instances(@name, instances) unless instances.empty?
+    end
+
+    def enable_zones (zones)
+      elb.enable_zones(@name, zones) unless zones.empty?
+    end
+
+    def disable_zones (zones)
+      elb.disable_zones(@name, zones) unless zones.empty?
+    end
+
+  end
+end