ey_cloud_awareness 0.1.12 → 0.1.13

data/README.rdoc

@@ -8,6 +8,18 @@ This gem makes it a little easier to live on the EngineYard cloud:
 
 We use it over at http://brighterplanet.com.
 
+== Don't read this
+
+This gem depends on
+
+  /etc/chef/dna.json
+
+being READABLE and containing certain attributes as named by EngineYard. You might have to add:
+
+  `sudo chmod a+r /etc/chef/dna.json`
+
+to your before_migrate.rb script.
+
 == Quick start
 
 Put this in <tt>config/environment.rb</tt>:
@@ -139,23 +151,50 @@ Or whatever you want:
   >> all_app_instances = EngineYardCloudInstance.app      
   => [#<EngineYardCloudInstance:0xb5e12f70 @instance_id="i-50cf5838">, #<EngineYardCloudInstance:0xb5e12f5c @instance_id="i-dc9207b4">, #<EngineYardCloudInstance:0xb5e12f34 @instance_id="i-b2d84fda">]
   >> all_app_instances.first.dns_name
-  => "ec2-67-202-43-40.compute-1.amazonaws.com"
+  => "ec2-67-201-47-30.compute-1.amazonaws.com"
   >> pp all_app_instances.first.to_hash
-  {:dns_name=>"ec2-67-202-43-40.compute-1.amazonaws.com",
-   :instance_role=>"app",
-   :aws_groups=>["ey-my_production-1256085955-3205-13340"],
-   :aws_instance_id=>"i-50cf5838",
-   :private_dns_name=>"domU-12-31-39-01-99-D3.compute-1.internal",
-   :aws_state=>"running"}
-  => nil
-
-== A note on EngineYard dependence
-
-This gem depends on
-
-  /etc/chef/dna.json
-
-being present and containing certain attributes as named by EngineYard. Please let me know if something changes.
+  => {:block_device_mapping=>
+        [{:ebs=>
+           {:status=>"attached",
+            :volumeId=>"vol-26172ee8",
+            :deleteOnTermination=>"false",
+            :attachTime=>"2010-04-07T21:09:37.000Z"},
+          :deviceName=>"/dev/sdz2"},
+         {:ebs=>
+           {:status=>"attached",
+            :volumeId=>"vol-26172ee8",
+            :deleteOnTermination=>"false",
+            :attachTime=>"2010-04-07T21:09:37.000Z"},
+          :deviceName=>"/dev/sdz1"}],
+        :launch_time=>"2010-04-07T21:08:10.000Z",
+        :instance_type=>"c1.medium",
+        :private_dns_name=>"domU-44-44-44-44-A4-02.compute-1.internal",
+        :instance_state=>{:code=>"16", :name=>"running"},
+        :ami_launch_index=>"0",
+        :users=>
+        [{:password=>"hoppAugEv",
+          :username=>"deploy",
+          :uid=>"1000",
+          :comment=>"",
+          :gid=>"1000"}],
+        :environment=>
+        {:framework_env=>"production",
+         :name=>"my_staging",
+         :stack=>"nginx_passenger"},
+        :instance_id=>"i-50cf5838",
+        :group_id=>"ey-my_staging-XXXXXXXXXXXXXXXXXXXX",
+        :root_device_type=>"instance-store",
+        :private_ip_address=>"10.201.102.201",
+        :kernel_id=>"aki-9b00e5f2",
+        :placement=>{:availabilityZone=>"us-east-1a"},
+        :product_codes=>nil,
+        :image_id=>"ami-7044a419",
+        :reason=>nil,
+        :dns_name=>"ec2-67-201-47-30.compute-1.amazonaws.com",
+        :ip_address=>"199.99.99.99",
+        :architecture=>"i386",
+        :instance_role=>"solo",
+        :monitoring=>{:state=>"disabled"}}
 
 == A note on caching and network needs
 
@@ -163,7 +202,7 @@ I tried to be smart about caching the results of network calls.
 
 Stuff like the current instance id, which is pulled from an EC2 metadata server, is stored in
 
-  CURRENT_INSTANCE_ID_CACHE_PATH = defined?(RAILS_ROOT) ? "#{RAILS_ROOT}/config/engine_yard_cloud_instance_id" : '/etc/engine_yard_cloud_instance_id'
+  CURRENT_INSTANCE_ID_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_instance_id'
 
 Please let me know if this causes problems.
 

data/Rakefile

@@ -6,14 +6,14 @@ begin
   Jeweler::Tasks.new do |gem|
     gem.name = "ey_cloud_awareness"
     gem.summary = %Q{Make your EngineYard cloud instances aware of each other.}
-    gem.description = %Q{Make your EngineYard cloud instances aware of each other.}
+    gem.description = %Q{Pull metadata from EC2 and EngineYard so that your EngineYard Cloud instances know about each other.}
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/ey_cloud_awareness"
     gem.authors = ["Seamus Abshere"]
     # gem.rubyforge_project = "ey_cloud_awareness"
     gem.add_dependency 'json', '>=1.2.3'
     gem.add_dependency 'activesupport', '>=2.3.4'
-    gem.add_dependency 'right_aws', '1.10.0' # static because of my hack
+    gem.add_dependency 'amazon-ec2', '>=0.9.10'
     gem.add_development_dependency "rspec", ">= 1.2.9"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-0.1.12
+0.1.13

data/ey_cloud_awareness.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ey_cloud_awareness}
-  s.version = "0.1.12"
+  s.version = "0.1.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere"]
-  s.date = %q{2010-04-07}
-  s.description = %q{Make your EngineYard cloud instances aware of each other.}
+  s.date = %q{2010-04-08}
+  s.description = %q{Pull metadata from EC2 and EngineYard so that your EngineYard Cloud instances know about each other.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -49,18 +49,18 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<json>, [">= 1.2.3"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_runtime_dependency(%q<right_aws>, ["= 1.10.0"])
+      s.add_runtime_dependency(%q<amazon-ec2>, [">= 0.9.10"])
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
     else
       s.add_dependency(%q<json>, [">= 1.2.3"])
       s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_dependency(%q<right_aws>, ["= 1.10.0"])
+      s.add_dependency(%q<amazon-ec2>, [">= 0.9.10"])
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
     end
   else
     s.add_dependency(%q<json>, [">= 1.2.3"])
     s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-    s.add_dependency(%q<right_aws>, ["= 1.10.0"])
+    s.add_dependency(%q<amazon-ec2>, [">= 0.9.10"])
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
   end
 end

data/lib/engine_yard_cloud_instance.rb

@@ -1,7 +1,7 @@
 class EngineYardCloudInstance
   CURRENT_INSTANCE_ID_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_instance_id'
-  CURRENT_SECURITY_GROUPS_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_security_groups'
-  INSTANCE_DESCRIPTIONS_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_instance_descriptions.yml'
+  CURRENT_SECURITY_GROUP_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_security_group'
+  INSTANCE_DESCRIPTIONS_CACHE_PATH = File.expand_path '~/.ey_cloud_awareness/engine_yard_cloud_ec2_instance_descriptions.yml'
   DNA_PATH = '/etc/chef/dna.json'
   
   attr_reader :instance_id
@@ -104,49 +104,52 @@ class EngineYardCloudInstance
       raise "[EY CLOUD AWARENESS GEM] Can't clear if we used from_hash" if self.proxy
       @_data = nil
       @_dna = nil
-      cached_instance_descriptions true
-      cached_current_security_groups true
+      cached_ec2_instance_descriptions true
+      cached_current_security_group true
       cached_current_instance_id true
     end
     
     def data
       return @_data if @_data
       raise "[EY CLOUD AWARENESS GEM] Can't calculate data if we used from_hash" if self.proxy
-      hash = Hash.new
-      cached_instance_descriptions.each do |instance_description|
-        next unless Set.new(Array.wrap(cached_current_security_groups)).superset? Set.new(instance_description[:aws_groups])
-        hash[instance_description[:aws_instance_id]] ||= Hash.new
-        current = hash[instance_description[:aws_instance_id]]
-        # using current as a pointer
+      @_data = Hash.new
+      cached_ec2_instance_descriptions.each do |ec2_instance_description|
+        @_data[ec2_instance_description['instanceId']] ||= Hash.new
+        member = @_data[ec2_instance_description['instanceId']]
+        # using instance as a pointer
         if dna[:instance_role] == 'solo'
-          current[:instance_role] = 'solo'
-        elsif dna[:db_host] == instance_description[:dns_name] or dna[:db_host] == instance_description[:private_dns_name]
-          current[:instance_role] = 'db_master'
-        elsif Array.wrap(dna[:db_slaves]).include? instance_description[:private_dns_name]
-          current[:instance_role] = 'db_slave'
-        elsif Array.wrap(dna[:utility_instances]).include? instance_description[:private_dns_name]
-          current[:instance_role] = 'utility'
-        elsif dna[:master_app_server][:private_dns_name] == instance_description[:private_dns_name]
-          current[:instance_role] = 'app_master'
-        elsif instance_description[:aws_state] == 'running'
-          current[:instance_role] = 'app'
+          member[:instance_role] = 'solo'
+        elsif dna[:db_host] == ec2_instance_description['dnsName'] or dna[:db_host] == ec2_instance_description['privateDnsName']
+          member[:instance_role] = 'db_master'
+        elsif Array.wrap(dna[:db_slaves]).include? ec2_instance_description['privateDnsName']
+          member[:instance_role] = 'db_slave'
+        elsif Array.wrap(dna[:utility_instances]).include? ec2_instance_description['privateDnsName']
+          member[:instance_role] = 'utility'
+        elsif dna[:master_app_server][:private_dns_name] == ec2_instance_description['privateDnsName']
+          member[:instance_role] = 'app_master'
+        elsif ec2_instance_description['instanceState']['name'] == 'running'
+          member[:instance_role] = 'app'
         else
-          current[:instance_role] = 'unknown'
+          member[:instance_role] = 'unknown'
         end
-        current[:private_dns_name] = instance_description[:private_dns_name]
-        current[:dns_name] = instance_description[:dns_name]
-        current[:aws_state] = instance_description[:aws_state]
-        current[:aws_groups] = instance_description[:aws_groups]
-        current[:aws_instance_id] = instance_description[:aws_instance_id]
-        current[:users] = dna[:users]
-        current[:environment] = dna[:environment]
+        member[:group_id] = cached_current_security_group
+        member[:users] = dna[:users]
+        member[:environment] = dna[:environment]
         @_environment ||= dna[:environment]
+
+        ec2_instance_description.each do |raw_k, raw_v|
+          k = raw_k.underscore.to_sym
+          next if member.keys.include? k
+          member[k] = raw_v
+        end
       end
-      @_data = hash.recursive_symbolize_keys!
+      @_data.recursive_symbolize_keys!
+      @_data
     end
     
     def dna
       raise "[EY CLOUD AWARENESS GEM] Can't see DNA if we used from_hash" if self.proxy
+      raise "[EY CLOUD AWARENESS GEM] Can't read DNA from #{DNA_PATH}! You should put 'sudo chmod a+r /etc/chef/dna.json' your your before_migrate.rb!" unless File.readable?(DNA_PATH)
       @_dna ||= JSON.load(IO.read(DNA_PATH)).recursive_symbolize_keys!
     end
     
@@ -167,33 +170,37 @@ class EngineYardCloudInstance
     end
   
 
-    def cached_current_security_groups(refresh = false)
-      raise "[EY CLOUD AWARENESS GEM] Can't call current_security_groups if we used from_hash" if self.proxy
-      if refresh or !File.readable?(CURRENT_SECURITY_GROUPS_CACHE_PATH)
-        @_cached_current_security_groups = open("http://169.254.169.254/latest/meta-data/security-groups").gets
+    def cached_current_security_group(refresh = false)
+      raise "[EY CLOUD AWARENESS GEM] Can't call current_security_group if we used from_hash" if self.proxy
+      if refresh or !File.readable?(CURRENT_SECURITY_GROUP_CACHE_PATH)
+        @_cached_current_security_group = open('http://169.254.169.254/latest/meta-data/security-groups').gets
+        raise "[EY CLOUD AWARENESS GEM] Don't know how to deal with (possibly) multiple security group: #{@_cached_current_security_group}" if @_cached_current_security_group =~ /,;/
         begin
-          FileUtils.mkdir_p File.dirname(CURRENT_SECURITY_GROUPS_CACHE_PATH)
-          File.open(CURRENT_SECURITY_GROUPS_CACHE_PATH, 'w') { |f| f.write @_cached_current_security_groups }
+          FileUtils.mkdir_p File.dirname(CURRENT_SECURITY_GROUP_CACHE_PATH)
+          File.open(CURRENT_SECURITY_GROUP_CACHE_PATH, 'w') { |f| f.write @_cached_current_security_group }
         rescue Errno::EACCES
-          $stderr.puts "[EY CLOUD AWARENESS GEM] Not caching current security groups because #{CURRENT_SECURITY_GROUPS_CACHE_PATH} can't be written to"
+          $stderr.puts "[EY CLOUD AWARENESS GEM] Not caching current security group because #{CURRENT_SECURITY_GROUP_CACHE_PATH} can't be written to"
         end
       end
-      @_cached_current_security_groups ||= IO.read(CURRENT_SECURITY_GROUPS_CACHE_PATH)
+      @_cached_current_security_group ||= IO.read(CURRENT_SECURITY_GROUP_CACHE_PATH)
     end
     
-    def cached_instance_descriptions(refresh = false)
-      raise "[EY CLOUD AWARENESS GEM] Can't call cached_instance_descriptions if we used from_hash" if self.proxy
+    def cached_ec2_instance_descriptions(refresh = false)
+      raise "[EY CLOUD AWARENESS GEM] Can't call cached_ec2_instance_descriptions if we used from_hash" if self.proxy
       if refresh or !File.readable?(INSTANCE_DESCRIPTIONS_CACHE_PATH)
-        ec2 = RightAws::Ec2.new dna[:aws_secret_id], dna[:aws_secret_key]
-        @_cached_instance_descriptions = ec2.describe_instances.map(&:recursive_symbolize_keys!)
+        ec2 = AWS::EC2::Base.new :access_key_id => dna[:aws_secret_id], :secret_access_key => dna[:aws_secret_key]
+        @_cached_ec2_instance_descriptions = ec2.describe_instances
+        @_cached_ec2_instance_descriptions.recursive_kill_xml_item_keys!
+        @_cached_ec2_instance_descriptions = @_cached_ec2_instance_descriptions['reservationSet'].select { |hash| cached_current_security_group.include? hash['groupSet'].first['groupId'] }
+        @_cached_ec2_instance_descriptions.map! { |hash| hash['instancesSet'].first }
         begin
           FileUtils.mkdir_p File.dirname(INSTANCE_DESCRIPTIONS_CACHE_PATH)
-          File.open(INSTANCE_DESCRIPTIONS_CACHE_PATH, 'w') { |f| f.write @_cached_instance_descriptions.to_yaml }
+          File.open(INSTANCE_DESCRIPTIONS_CACHE_PATH, 'w') { |f| f.write @_cached_ec2_instance_descriptions.to_yaml }
         rescue Errno::EACCES
           $stderr.puts "[EY CLOUD AWARENESS GEM] Not caching instance data because #{INSTANCE_DESCRIPTIONS_CACHE_PATH} can't be written to"
         end
       end
-      @_cached_instance_descriptions ||= YAML.load(IO.read(INSTANCE_DESCRIPTIONS_CACHE_PATH))
+      @_cached_ec2_instance_descriptions ||= YAML.load(IO.read(INSTANCE_DESCRIPTIONS_CACHE_PATH))
     end
   end
 end

data/lib/ey_cloud_awareness.rb

@@ -3,13 +3,20 @@ require 'set'
 require 'fileutils'
 require 'json'
 require 'yaml'
-require 'right_aws' # See aws-s3 compatibility hack below
+require 'AWS' # aka amazon-ec2
 require 'active_support'
-begin; require 'active_support/core_ext/class/inheritable_attributes'; rescue MissingSourceFile; end
-begin; require 'active_support/inflector/inflections'; rescue MissingSourceFile; end
-begin; require 'active_support/core_ext/string/inflections'; rescue MissingSourceFile; end
-begin; require 'active_support/core_ext/hash/keys'; rescue MissingSourceFile; end
-begin; require 'active_support/core_ext/array/wrap'; rescue MissingSourceFile; end
+require 'active_support/version'
+%w{
+  active_support/core_ext/string
+  active_support/core_ext/class/inheritable_attributes
+  active_support/inflector/inflections
+  active_support/core_ext/string/inflections
+  active_support/core_ext/hash/keys
+  active_support/core_ext/array/wrap
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ActiveSupport::VERSION::MAJOR == 3
+    
 require 'engine_yard_cloud_instance'
 
 class Hash
@@ -28,36 +35,39 @@ class Hash
     self
   end
   
-  def deep_copy
-    Marshal.load Marshal.dump(self)
+  XML_ITEM_KEYS = [ :item, 'item' ]
+  
+  # :sam => { :item => [{ :foo => :bar }] }
+  # into
+  # :sam => [{:foo => :bar}]
+  def kill_xml_item_keys!
+    if keys.length == 1 and XML_ITEM_KEYS.include?(keys.first)
+      raise ArgumentError, "You need to call kill_xml_item_keys! on { :foo => { :items => [...] } } not on { :items => [...] }"
+    end
+    keys.each do |key|
+      if self[key].is_a?(Hash) and self[key].keys.length == 1 and XML_ITEM_KEYS.include?(self[key].keys.first)
+        # self[:sam] = self[:sam]["item"] (using values.first because we don't know if it's :item or "item")
+        self[key] = delete(key).values.first
+      end
+    end
+    self
   end
-end
-
-# sabshere 11/05/09
-# Compatibility hack for aws-s3/right_aws
-# Apologies for rescuing instead of directly checking arity
-# I couldn't figure out how to do that because we don't have Module#instance_method
-# ... and the class has overridden Object#method (=> "GET"/"POST"/etc)
-module Net
-  class HTTPGenericRequest
-    def exec(sock, ver, path, send_only = nil)   #:nodoc: internal use only
-      if @body
-        begin
-          send_request_with_body sock, ver, path, @body, send_only
-        rescue ArgumentError
-          $stderr.puts "[EY CLOUD AWARENESS GEM] Rescued from #{$!} because we thought it might have to do with aws-s3/right_aws incompatibility"
-          send_request_with_body sock, ver, path, @body
-        end
-      elsif @body_stream
-        begin
-          send_request_with_body_stream sock, ver, path, @body_stream, send_only
-        rescue ArgumentError
-          $stderr.puts "[EY CLOUD AWARENESS GEM] Rescued from #{$!} because we thought it might have to do with aws-s3/right_aws incompatibility"
-          send_request_with_body_stream sock, ver, path, @body_stream
-        end
-      else
-        write_header sock, ver, path
+  
+  def recursive_kill_xml_item_keys!
+    kill_xml_item_keys!
+    values.select { |v| v.is_a?(Hash) }.each do |hsh|
+      hsh.recursive_kill_xml_item_keys!
+    end
+    # burst thru at least one level of arrays
+    values.select { |v| v.is_a?(Array) }.each do |ary|
+      ary.each do |v|
+        v.recursive_kill_xml_item_keys! if v.is_a?(Hash)
       end
     end
+    self
+  end
+  
+  def deep_copy
+    Marshal.load Marshal.dump(self)
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 12
-  version: 0.1.12
+  - 13
+  version: 0.1.13
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-07 00:00:00 -04:00
+date: 2010-04-08 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -46,17 +46,17 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: right_aws
+  name: amazon-ec2
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
         segments: 
-        - 1
-        - 10
         - 0
-        version: 1.10.0
+        - 9
+        - 10
+        version: 0.9.10
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -73,7 +73,7 @@ dependencies:
         version: 1.2.9
   type: :development
   version_requirements: *id004
-description: Make your EngineYard cloud instances aware of each other.
+description: Pull metadata from EC2 and EngineYard so that your EngineYard Cloud instances know about each other.
 email: seamus@abshere.net
 executables: []