build-cloud 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 44ac926d7827973fce157398265b248b9f2263df
-  data.tar.gz: 4d4ae065f177a4c2cd16840c5501c3d90b7f27fa
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZThjZDkyYzRlNjlkOTA5ZDJlOTkyY2RmYTQ3MTI1YzhiMTg3NWFmOA==
+  data.tar.gz: !binary |-
+    MWM3ZGRjZWUxOTgxM2I5MTdkMjI2Y2ZhZTVjNjBmYjk4OTM0Zjk4MA==
 SHA512:
-  metadata.gz: 63aa3403dddf30aa6623d1b006953fb00fed715e1099eda8070d1f5a7bc27ba7549e86347c0490da46d310232a52aa20cd6d73c44adf2341e6093176993cdd97
-  data.tar.gz: 18380bcc47e5ac2c635210e59f0d90c67aaf94c00be3609d3f2eab5a2589db2740e28455791fcf3cc652032d66ea4c8afeb634ecb3d616395639376da2dcb9fe
+  metadata.gz: !binary |-
+    M2IxNDhlN2VhNTQyM2IyNDhlNjk4NzhlM2RkYzVkYjRhYmNkODg1NTQxY2Mz
+    ZTljOWRkNTcxZGZlODExMzg3NzkzMGVlZDYxNzY5YzE1OTkxOTZhY2E2ZTgy
+    NTYzODA5MmZmOTgxZjI2ZmU2MzQzN2MzNWNiYjhkMjNhYTJjNDc=
+  data.tar.gz: !binary |-
+    MGQzMTg5YzYyZTA0NWI5NGIzYjAzZmViOGFmOGE5ODlmNTc2NjgzNGUxM2Jl
+    NTFjNjRiMTJhMmUzY2ZmNjY4MDFiYWNhM2JlNDliZDJhYjUzMDBiMGE5MTg1
+    NmI1YjkxYzJjMzk2MjczNjE5MDBjYWE2ODAzMzQ1NTRlYWMwZmY=

data/README.md

@@ -1,6 +1,6 @@
 # Build::Cloud
 
-TODO: Write a gem description
+Tools for building resources in AWS}
 
 ## Installation
 
@@ -18,7 +18,15 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+See the command line help for `build-cloud`.
+
+## Changelog
+
+2014-09-15 - version 0.0.3 - now accepts multiple files to `--config`. The second and subsequent files are merged into the first YAML file in order, ahead of any files specified in a `:include` list in the first YAML file.
+
+2014-06-23 - version 0.0.2 - now accepts an array of files in the `:include` key in the given YAML config file. The files are merged in to the config file in the order that they are given.
+
+2014-05-27 - version 0.0.1 - initial version.
 
 ## Contributing
 

data/bin/build-cloud

@@ -25,6 +25,7 @@ require 'build-cloud/cacheparametergroup'
 require 'build-cloud/iamrole'
 require 'build-cloud/s3bucket'
 require 'build-cloud/instance'
+require 'build-cloud/ebsvolume'
 require 'build-cloud/networkinterface'
 require 'build-cloud/internetgateway'
 
@@ -32,7 +33,7 @@ options = {}
 optparse = OptionParser.new do |opts|
 
     options[:config] = nil
-    opts.on('-c', '--config=file', 'Config file path') do |c|
+    opts.on('-c', '--config file1,file2', Array, 'Config files - second and subsequent files merged into first.') do |c|
         options[:config] = c
     end
 
@@ -72,12 +73,12 @@ optparse = OptionParser.new do |opts|
     end
 
     options[:find_type] = []
-    opts.on('--find-type=t', 'Find objects by type') do |t|
+    opts.on('--find-type t', 'Find objects by type') do |t|
         options[:find_type].push(t.to_sym)
     end
 
     options[:find] = {}
-    opts.on('--find=f', 'Find objects by key match') do |f|
+    opts.on('--find f', 'Find objects by key match') do |f|
         args = f.split('=')
         options[:find][ args[0].to_sym ] = args[1]
     end

data/build-cloud.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "build-cloud"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.3"
   spec.authors       = ["The Scale Factory"]
   spec.email         = ["info@scalefactory.com"]
   spec.summary       = %q{Tools for building resources in AWS}

data/lib/build-cloud.rb

@@ -15,14 +15,28 @@ class BuildCloud
 
         @log = options[:logger] or Logger.new( STDERR )
         @mock = options[:mock] or false
-        @config = YAML::load( File.open( options[:config] ) )
         
-        if include_file = @config.delete(:include)
-            include_path = File.join( File.dirname( options[:config] ), include_file)
+        first_config_file = options[:config].shift
+
+        @config = YAML::load( File.open( first_config_file ) )
+
+        include_files = options[:config]
+        if include_yaml = @config.delete(:include)
+            if include_yaml.is_a?(Array)
+                include_files.concat(include_yaml)
+            else
+                include_files.push(include_yaml)
+            end
+        end
+        
+        include_files.each do |include_file|
+
+            include_path = File.join( File.dirname( first_config_file ), include_file)
 
             if File.exists?( include_path )
-               included_conf = YAML::load( File.open( include_path ) )
-               @config = @config.merge( included_conf )
+                @log.info( "Including YAML file #{include_path}" )
+                included_conf = YAML::load( File.open( include_path ) )
+                @config = @config.merge( included_conf )
             end
 
         end
@@ -99,6 +113,7 @@ class BuildCloud
             :iam_roles             => BuildCloud::IAMRole,
             :s3_buckets            => BuildCloud::S3Bucket,
             :instances             => BuildCloud::Instance,
+            :ebs_volumes           => BuildCloud::EBSVolume,
         }
 
     end
@@ -125,6 +140,7 @@ class BuildCloud
             :as_groups,
             :r53_record_sets,
             :s3_buckets,
+            :ebs_volumes,
             :instances,
         ]
     end

data/lib/build-cloud/ebsvolume.rb

@@ -0,0 +1,82 @@
+class BuildCloud::EBSVolume
+
+    include ::BuildCloud::Component
+
+    @@objects = []
+
+    def self.get_id_by_name( name )
+
+        volume = self.search( :name => name ).first
+
+        unless volume
+            raise "Couldn't get an EBSVolume object for #{name} - is it defined?"
+        end
+
+        volume_fog = volume.read
+
+        unless volume_fog
+            raise "Couldn't get an EBSVolume fog object for #{name} - is it created?"
+        end
+
+        volume_fog.id
+
+    end
+
+    def initialize ( fog_interfaces, log, options = {} )
+
+        @compute = fog_interfaces[:compute]
+        @log     = log
+        @options = options
+
+        @log.debug( options.inspect )
+
+        required_options(:name, :availability_zone, :size)
+
+    end
+
+    def create
+        
+        return if exists?
+
+        @log.info( "Creating volume #{@options[:name]}" )
+
+        options = @options.dup
+
+        volume = @compute.volumes.new(options)
+        volume.save
+
+        attributes = {}
+        attributes[:resource_id] = volume.id.to_s
+        attributes[:key] = 'Name'
+        attributes[:value] = @options[:name]
+        volume_tag = @compute.tags.new( attributes )
+        volume_tag.save
+
+        @log.debug( volume.inspect )
+
+        if @options[:instance_name]
+            instance_id = BuildCloud::Instance.get_id_by_name( options[:instance_name] )
+            attach_response = @compute.attach_volume(instance_id, volume.id, options[:device])
+            @log.debug( attach_response.inspect )
+        end
+
+    end
+
+    def read
+        @compute.volumes.select { |v| v.tags['Name'] == @options[:name]}.first
+    end
+
+    alias_method :fog_object, :read
+
+    def delete
+
+        return unless exists?
+
+        @log.info( "Deleting volume #{@options[:name]}" )
+
+        fog_object.destroy
+
+    end
+
+end
+

data/lib/build-cloud/instance.rb

@@ -1,3 +1,6 @@
+require 'erb'
+require 'timeout'
+
 class BuildCloud::Instance
 
     include ::BuildCloud::Component
@@ -30,10 +33,11 @@ class BuildCloud::Instance
 
         @log.debug( options.inspect )
 
-        required_options(:image_id, :flavor_id, :private_ip_address, :name)
+        required_options(:image_id, :flavor_id, :name)
         require_one_of(:security_group_ids, :security_group_names, :network_interfaces)
         require_one_of(:subnet_id, :subnet_name, :network_interfaces)
         #require_one_of(:network_interfaces, :private_ip_address)
+        require_one_of(:user_data, :user_data_file, :user_data_template)
         require_one_of(:vpc_id, :vpc_name)
 
     end
@@ -92,7 +96,39 @@ class BuildCloud::Instance
             options.delete(:subnet_id)
         end
 
-        options[:user_data] = JSON.generate( @options[:user_data] )
+        if options[:user_data]
+
+            options[:user_data] = JSON.generate( options[:user_data] )
+
+        elsif options[:user_data_file]
+
+            user_data_file_path = File.join( Dir.pwd, options[:user_data_file])
+
+            if File.exists?( user_data_file_path )
+                options[:user_data] = File.read( user_data_file_path )
+                options.delete(:user_data_file)
+            else
+                @log.error("config lists a :user_data_file that doesn't exist at #{options[:user_data_file]}")
+            end
+
+        elsif options[:user_data_template]
+
+            variable_hash = options[:user_data_variables]
+            user_data_template_path = File.join( Dir.pwd, options[:user_data_template])
+
+            if File.exists?( user_data_template_path )
+                template = File.read( user_data_template_path )
+                ### We set 'trim_mode' to '-', which supresses end of line white space on lines ending in '-%>'
+                ### see http://ruby-doc.org/stdlib-2.1.2/libdoc/erb/rdoc/ERB.html#method-c-new
+                buffer = ERB.new(template,nil,'-').result(binding)
+                options[:user_data] = buffer
+                options.delete(:user_data_variables)
+                options.delete(:user_data_template)
+            else
+                @log.error("config lists a :user_data_template that doesn't exist at #{options[:user_data_template]}")
+            end
+
+        end
 
         options[:network_interfaces].each { |iface|
             if ! iface[:network_interface_name].nil?
@@ -107,17 +143,61 @@ class BuildCloud::Instance
         instance = @ec2.servers.new( options )
         instance.save
 
-        options[:tags].each do | tag |
-            attributes = {}
-            attributes[:resource_id] = instance.id.to_s
-            attributes[:key] = tag[:key]
-            attributes[:value] = tag[:value]
-            new_tag = @ec2.tags.new( attributes )
-            new_tag.save
-        end unless options[:tags].empty? or options[:tags].nil?
-
         @log.debug( instance.inspect )
 
+        if options[:ebs_volumes]
+
+            instance = @ec2.servers.get(instance.id)
+            instance_state = instance.state
+
+            begin
+                Timeout::timeout(30) {
+                    until instance_state == 'running'
+                        @log.info( "instance not ready yet: #{instance_state}" )
+                        sleep 3
+                        instance = @ec2.servers.get(instance.id)
+                        instance_state = instance.state
+                    end
+                    @log.debug("Instance state: #{instance_state}")
+                }
+            rescue Timeout::Error
+                @log.error("Waiting on availability for instance: #{instance.id}, timed out")
+            end
+
+            instance_id = instance.id
+            options[:ebs_volumes].each do |vol|
+                vol_id = BuildCloud::EBSVolume.get_id_by_name( vol[:name] )
+                attach_response = @ec2.attach_volume(instance_id, vol_id, vol[:device])
+                @log.debug( attach_response.inspect )
+                volume_state = @ec2.volumes.get(vol_id).state
+
+                begin
+                    Timeout::timeout(30) {
+                        until volume_state == 'in-use'
+                            @log.info( "Volume not attached yet: #{volume_state}" )
+                            sleep 3
+                            volume_state = @ec2.volumes.get(vol_id).state
+                        end
+                        @log.debug("Volume state: #{volume_state}")
+                    }
+                rescue Timeout::Error
+                    @log.error("Operation to attach volume: #{vol[:name]}, timed out")
+                end
+
+                if vol[:delete_on_termination] and volume_state == 'in-use'
+                    request_resp = @ec2.modify_instance_attribute(
+                        instance_id,
+                        { "BlockDeviceMapping.DeviceName" => "#{vol[:device]}",
+                          "BlockDeviceMapping.Ebs.DeleteOnTermination" => true }
+                    )
+                    unless request_resp.body["return"]
+                        @log.error("Failed to set delete_on_termination for volume: #{vol[:name]}")
+                    end
+                    @log.debug( request_resp.inspect )
+                end
+            end
+        end
+
     end
 
     def read

data/lib/build-cloud/internetgateway.rb

@@ -18,7 +18,7 @@ class BuildCloud::InternetGateway
             raise "Couldn't get an InternetGateway fog object for #{name} - is it created?"
         end
 
-        internet_gateway_fog.internet_gateway_id
+        internet_gateway_fog.id
 
     end
 

data/lib/build-cloud/launchconfiguration.rb

@@ -14,8 +14,9 @@ class BuildCloud::LaunchConfiguration
 
         @log.debug( options.inspect )
 
-        required_options(:id, :image_id, :key_name, :user_data, :instance_type)
+        required_options(:id, :image_id, :key_name, :instance_type)
         require_one_of(:security_groups, :security_group_names)
+        require_one_of(:user_data, :user_data_file, :user_data_template)
 
     end
 
@@ -39,7 +40,39 @@ class BuildCloud::LaunchConfiguration
 
         end
 
-        options[:user_data] = JSON.generate( @options[:user_data] )
+        if options[:user_data]
+
+            options[:user_data] = JSON.generate( @options[:user_data] )
+
+        elsif options[:user_data_file]
+
+            user_data_file_path = File.join( Dir.pwd, options[:user_data_file])
+
+            if File.exists?( user_data_file_path )
+                options[:user_data] = File.read( user_data_file_path )
+                options.delete(:user_data_file)
+            else
+                @log.error("config lists a :user_data_file that doesn't exist at #{options[:user_data_file]}")
+            end
+
+        elsif options[:user_data_template]
+
+            variable_hash = options[:user_data_variables]
+            user_data_template_path = File.join( Dir.pwd, options[:user_data_template])
+
+            if File.exists?( user_data_template_path )
+                template = File.read( user_data_template_path )
+                ### We set 'trim_mode' to '-', which supresses end of line white space on lines ending in '-%>'
+                ### see http://ruby-doc.org/stdlib-2.1.2/libdoc/erb/rdoc/ERB.html#method-c-new
+                buffer = ERB.new(template,nil,'-').result(binding)
+                options[:user_data] = buffer
+                options.delete(:user_data_variables)
+                options.delete(:user_data_template)
+            else
+                @log.error("config lists a :user_data_template that doesn't exist at #{options[:user_data_template]}")
+            end
+
+        end
 
         launch_config = @as.configurations.new( options )
         launch_config.save

data/lib/build-cloud/networkinterface.rb

@@ -68,6 +68,7 @@ class BuildCloud::NetworkInterface
 
         interface = @compute.network_interfaces.new(options)
         interface.save
+        wait_until_ready
 
         attributes = {}
         attributes[:resource_id] = interface.network_interface_id
@@ -81,17 +82,21 @@ class BuildCloud::NetworkInterface
         end
 
         if options[:assign_new_public_ip]
-            ip = @compute.addresses.create
-            public_ip = ip.public_ip
-            allocation_id = ip.allocation_id
-            @compute.associate_address(nil, nil, interface.network_interface_id, allocation_id )
+            ### Need to use the request here, in order to support older accounts which still have
+            ### 'EC2 Classic' regions/azs assigned
+            ip = @compute.allocate_address(domain='vpc')
+            public_ip = ip.body['publicIp']
+            allocation_id = ip.body['allocationId']
+            @compute.associate_address(nil, public_ip, interface.network_interface_id, allocation_id )
+
+            @log.info( "Assigned new public IP #{public_ip}" )
         end
 
         unless options[:existing_public_ip].nil?
             ip = @compute.addresses.get(options[:existing_public_ip])
             public_ip = ip.public_ip
             allocation_id = ip.allocation_id
-            @compute.associate_address(nil, nil, interface.network_interface_id, allocation_id )
+            @compute.associate_address(nil, ip.public_ip, interface.network_interface_id, allocation_id )
         end
 
         @log.debug( interface.inspect )

data/lib/build-cloud/route.rb

@@ -28,17 +28,17 @@ class BuildCloud::Route
 
         options[:tags] = { 'Name' => options.delete(:name) }
 
-        unless options[:network_interface_name]
+        unless options[:network_interface_name].nil?
             options[:network_interface_id] = BuildCloud::NetworkInterface.get_id_by_name( options[:network_interface_name] )
             options.delete(:network_interface_name)
         end
 
-        unless options[:internet_gateway_name]
+        if options[:internet_gateway_name]
             options[:internet_gateway_id] = BuildCloud::InternetGateway.get_id_by_name( options[:internet_gateway_name] )
             options.delete(:internet_gateway_name)
         end
 
-        unless options[:route_table_name]
+        if options[:route_table_name]
             options[:route_table_id] = BuildCloud::RouteTable.get_id_by_name( options[:route_table_name] )
             options.delete(:route_table_name)
         end
@@ -46,14 +46,14 @@ class BuildCloud::Route
         route_table_id = options[:route_table_id]
         destination_cidr_block = options[:destination_cidr_block]
         internet_gateway_id = options[:internet_gateway_id]
-        network_interface_id = options[:network_interface_id]
+        network_interface_id = options[:network_interface_id] ||= nil
 
         # Using requests instead of model here, because the model
         #  doesn't support associations.
 
         begin
 
-            if create_route(route_table_id, destination_cidr_block, internet_gateway_id, nil, network_interface_id)
+            if @compute.create_route(route_table_id, destination_cidr_block, internet_gateway_id, nil, network_interface_id)
                 @log.debug("route created successfully")
             else 
                 @log.debug("failed to create route")
@@ -67,12 +67,10 @@ class BuildCloud::Route
 
     def read
         rt = @compute.route_tables.select { |rt| rt.tags['Name'] == @options[:name] }.first 
-        @log.info( rt.inspect )
-        @log.info( rt.routes ) unless rt.routes.empty?
-
-        route = rt.routes.select { |t| t['destinationCidrBlock'] == @options[:destination_cidr_block]}
-        @log.info( route.inspect )
-
+        @log.debug( rt.inspect )
+        route = rt.routes.select { |t| t['destinationCidrBlock'] == @options[:destination_cidr_block]}.first unless rt.nil?
+        @log.debug( route.inspect )
+        return true unless route.nil?
     end
 
     alias_method :fog_object, :read
@@ -81,9 +79,11 @@ class BuildCloud::Route
 
         return unless exists?
 
-        @log.info("Deleting route #{@options[:name]}")
+        options = @options.dup
+
+        @log.info("Deleting route #{options[:name]}")
 
-        unless options[:route_table_name]
+        unless options[:route_table_name].nil?
             options[:route_table_id] = BuildCloud::RouteTable.get_id_by_name( options[:route_table_name] )
             options.delete(:route_table_name)
         end
@@ -93,7 +93,7 @@ class BuildCloud::Route
 
         begin
 
-            if delete_route(route_table_id, destination_cidr_block)
+            if @compute.delete_route(route_table_id, destination_cidr_block)
                 @log.debug("route deleted successfully")
             else 
                 @log.debug("failed to delet route")

data/lib/build-cloud/routetable.rb

@@ -18,7 +18,7 @@ class BuildCloud::RouteTable
             raise "Couldn't get a RouteTable fog object for #{name} - is it created?"
         end
 
-        route_table_fog.route_table_id
+        route_table_fog.id
 
     end
 
@@ -73,6 +73,9 @@ class BuildCloud::RouteTable
         rt.save
         @log.debug(rt.inspect)
 
+        ### sometimes tag creation will fail unless we wait until the API catches up:
+        wait_until_ready
+
         @compute.create_tags( rt.id, options[:tags] )
        
         options[:subnet_ids].each do |s|

data/lib/build-cloud/subnet.rb

@@ -53,14 +53,18 @@ class BuildCloud::Subnet
         subnet = @compute.subnets.new( options )
         subnet.save
 
-        options[:tag_set].each do | tag |
-            attributes = {}
-            attributes[:resource_id] = subnet.subnet_id.to_s
-            attributes[:key] = tag[:key]
-            attributes[:value] = tag[:value]
-            new_tag = @compute.tags.new( attributes )
-            new_tag.save
-        end unless options[:tag_set].empty? or options[:tag_set].nil?
+        if options[:tag_set]
+
+            options[:tag_set].each do | tag |
+                attributes = {}
+                attributes[:resource_id] = subnet.subnet_id.to_s
+                attributes[:key] = tag[:key]
+                attributes[:value] = tag[:value]
+                new_tag = @compute.tags.new( attributes )
+                new_tag.save
+            end unless options[:tag_set].empty? or options[:tag_set].nil?
+
+        end
 
         @log.debug( subnet.inspect )
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: build-cloud
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - The Scale Factory
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-27 00:00:00.000000000 Z
+date: 2014-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,42 +28,42 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.22.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.22.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.12.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.12.6
 description: 
@@ -90,6 +90,7 @@ files:
 - lib/build-cloud/dbparametergroup.rb
 - lib/build-cloud/dbparameters.rb
 - lib/build-cloud/dbsubnetgroup.rb
+- lib/build-cloud/ebsvolume.rb
 - lib/build-cloud/iamrole.rb
 - lib/build-cloud/instance.rb
 - lib/build-cloud/internetgateway.rb
@@ -115,17 +116,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Tools for building resources in AWS