syncwrap 1.5.2 → 2.0.0

data/lib/syncwrap/amazon_ec2.rb

@@ -0,0 +1,236 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'time'
+
+require 'syncwrap/amazon_ws'
+require 'syncwrap/host'
+
+module SyncWrap
+
+  # Amazon EC2 host provider. Supports importing, creating, and
+  # terminating hosts.
+  #
+  # == Synopsis
+  #
+  # Add the following to your sync.rb
+  #
+  #   ec2 = AmazonEC2.new( space )
+  #
+  # Then add profiles via #profile, as needed.
+  #
+  class AmazonEC2
+    include AmazonWS
+
+    # FIXME: Interim strategy: use AmazonWS and defer deciding final
+    # organization.
+
+    # The json configuration file path, parsed and passed to
+    # AWS::config. This file should contain a json object with the
+    # minimal required keys: access_key_id, secret_access_key. A
+    # relative path is interpreted relative to the :sync_file_path
+    # option if provided on construction.  (default: private/aws.json)
+    attr_accessor :aws_config
+
+    def initialize( space, opts = {} )
+      super()
+      @profiles = {}
+      @space = space
+      @aws_config = 'private/aws.json'
+      opts = opts.dup
+      sync_file_path = opts.delete( :sync_file_path )
+      opts.each { |name, val| send( name.to_s + '=', val ) }
+
+      # Look up aws_config (i.e. default private/aws.json) relative to
+      # the sync file path.
+      if @aws_config
+        if sync_file_path
+          aws_configure( File.expand_path( @aws_config, sync_file_path ) )
+        else
+          aws_configure( @aws_config )
+        end
+      end
+    end
+
+    # Define a host profile by Symbol key and Hash value.
+    #
+    # Profiles may inherit properties from a :base_profile, either
+    # specified by that key, or the :default key profile. The
+    # base_profile must be defined in advance (above in the sync
+    # file). When merging profile to any base_profile, the :roles
+    # property is concatenated via set union. All other properties are
+    # overwritten.
+    #
+    # FIXME: All other profile properties are as currently defined by
+    # #aws_create_instance.
+    def profile( key, profile )
+      profile = profile.dup
+      base = profile.delete( :base_profile ) || :default
+      base_profile = @profiles[ base ]
+      if base_profile
+        profile = base_profile.merge( profile )
+        if base_profile[ :roles ] && profile[ :roles ]
+          profile[ :roles ] = ( base_profile[ :roles ] | profile[ :roles ] )
+        end
+      end
+
+      @profiles[ key ] = profile
+    end
+
+    def import_hosts( regions, output_file )
+      hlist = import_host_props( regions )
+      unless hlist.empty?
+
+        hlist.map! do |props|
+          props[:name] ||= props[:id].to_s
+          Host.new( space, props )
+        end
+
+        time = Time.now.utc
+        cmt = "\n# Import of AWS #{regions.join ','} on #{time.iso8601}"
+        append_host_definitions( hlist, cmt, output_file )
+      end
+    end
+
+    def create_hosts( count, profile_key, name, output_file )
+      profile = @profiles[ profile_key ].dup or
+        raise "Profile #{profile_key} not registered"
+
+      # FIXME: Support profiles overrides.
+      # Also add some targeted CLI overrides (like for :availability_zones)
+
+      if profile[ :user_data ] == :ec2_user_sudo
+        profile[ :user_data ] = ec2_user_data
+      end
+
+      dname = profile.delete( :default_name )
+      name ||= dname
+
+      count.times do
+        hname = if count == 1
+                  if space.host_names.include?( name )
+                    raise "Host #{name} already exists!"
+                  end
+                  name
+                else
+                  find_name( name )
+                end
+        props = aws_create_instance( hname, profile )
+        host = space.host( props )
+        append_host_definitions( [ host ], nil, output_file )
+        host[ :just_created ] = true #after so this isn't written
+      end
+    end
+
+    def terminate_hosts( names, delete_attached_storage, sync_file )
+      names.each do |name|
+        if space.host_names.include?( name )
+          host = space.host( name )
+          raise "Host #{name} missing :id" unless host[:id]
+          raise "Host #{name} missing :region" unless host[:region]
+          aws_terminate_instance( host, delete_attached_storage )
+          delete_host_definition( host, sync_file )
+        else
+          raise "Host #{name} not found in Space, sync file."
+        end
+      end
+    end
+
+    private
+
+    attr_reader :space
+
+    def find_name( prefix )
+      host_names = space.host_names
+      i = 1
+      name = nil
+      loop do
+        name = "%s-%2d" % [ prefix, i ]
+        break if ! host_names.include?( name )
+        i += 1
+      end
+      name
+    end
+
+    def append_host_definitions( hosts, comment, output_file )
+      File.open( output_file, "a" ) do |out|
+        out.puts comment if comment
+
+        hosts.each do |host|
+          props = host.props.dup
+          props.delete( :name )
+
+          roles = ( host.roles - [:all] ).map { |s| ':' + s.to_s }.join ', '
+          roles << ',' unless roles.empty?
+          props = host.props.map do |key,val|
+            "#{key}: #{val.inspect}" unless key == :name
+          end.compact.join ",\n      "
+
+          out.puts "host( '#{host.name}', #{roles}"
+          out.puts "      #{props} ) # :auto-generated"
+        end
+      end
+    end
+
+    def delete_host_definition( host, sync_file )
+      lines = IO.readlines( sync_file )
+      out_lines = []
+      state = :find
+      lines.each do |line|
+        if state == :find
+          if line =~ /^host\( '#{host.name}',/
+            state = :just_found
+          else
+            out_lines << line
+          end
+        end
+        if state == :just_found || state == :found
+          if state == :found && line =~ /^host\(/
+            state = :no_end
+            break
+          end
+          state = :found
+          if line =~ /^\s*[^#].*\) # :auto-generated$/
+            state = :deleted
+          end
+        elsif state == :deleted
+          out_lines << line
+        end
+      end
+
+      if state == :deleted
+        File.open( sync_file, "w" ) do |out|
+          out.puts out_lines
+        end
+      else
+        $stderr.puts( "WARNING: #{sync_file} entry not deleted (#{state})" )
+      end
+    end
+
+    def ec2_user_data( user = 'ec2-user' )
+      #FIXME: Utility module for this (+ Users sudoers)?
+      script = <<-SH
+        #!/bin/sh -e
+        echo '#{user} ALL=(ALL) NOPASSWD:ALL'  > /etc/sudoers.d/#{user}
+        echo 'Defaults:#{user} !requiretty'   >> /etc/sudoers.d/#{user}
+        chmod 440 /etc/sudoers.d/#{user}
+      SH
+      script.split( "\n" ).map { |l| l.strip }.join( "\n" )
+    end
+
+  end
+
+end

data/lib/syncwrap/amazon_ws.rb

@@ -0,0 +1,308 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'aws-sdk'
+require 'resolv'
+require 'json'
+
+module SyncWrap
+
+  # Supports host provisioning in EC2 via AWS APIs, creating and
+  # attaching EBS volumes, and creating Route53 record sets.
+  module AmazonWS
+
+    # Default options for Route53 record set creation
+    attr_accessor :route53_default_rs_options
+
+    # DNS Resolver options for testing Route53 (default: Use public
+    # google name servers to avoid local negative caching)
+    attr_accessor :resolver_options
+
+    def initialize
+      @default_instance_options = {
+        ebs_volumes:        0,
+        ebs_volume_options: { size: 16 }, #gb
+        lvm_volumes:        [ [ 1.00, '/data' ] ],
+        security_groups:    [ :default ],
+        instance_type:      'm1.medium',
+        region:             'us-east-1'
+      }
+      @route53_default_rs_options = {
+        ttl:  300,
+        wait: true
+      }
+
+      @resolver_options = {
+        nameserver: [ '8.8.8.8', '8.8.4.4' ]
+      }
+
+      super
+    end
+
+    protected
+
+    def aws_configure( json_file )
+      AWS.config( JSON.parse( IO.read( json_file ),
+                              symbolize_names: true ) )
+    end
+
+    # Create a security_group given name and options. :region is the
+    # only required option, :description is good to have. Currently
+    # this is a no-op if the security group already exists.
+    def aws_create_security_group( name, opts = {} )
+      opts = opts.dup
+      region = opts.delete( :region )
+      ec2 = AWS::EC2.new.regions[ region ]
+      unless ec2.security_groups.find { |sg| sg.name == name }
+        sg = ec2.security_groups.create( name, opts )
+
+        # FIXME: Allow ssh on the "default" region named group
+        if name == region
+          sg.authorize_ingress(:tcp, 22)
+        end
+      end
+    end
+
+    # Create an instance, using name as the Name tag and assumed
+    # host name. For options see
+    # {AWS::EC2::InstanceCollection.create}[http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/InstanceCollection.html#create-instance_method]
+    # with the following additions/differences:
+    #
+    # :count:: must be 1 or unspecified.
+    # :region:: Default 'us-east-1'
+    # :security_groups:: As per aws-sdk, but the special :default value
+    #                    is replaced with a single security group with
+    #                    same name as the :region.
+    # :ebs_volumes:: The number of EBS volumes to create an attach to this instance.
+    # :ebs_volume_options:: A nested Hash of options, as per
+    #                       {AWS::EC2::VolumeCollection.create}[http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/VolumeCollection.html#create-instance_method]
+    #                       with custom default :size 16 GB, and the same
+    #                       :availibility_zone as the instance.
+    # :lvm_volumes:: Ignored here.
+    # :roles:: Array of role Strings or Symbols (applied as Roles tag)
+    def aws_create_instance( name, opts = {} )
+      opts = deep_merge_hashes( @default_instance_options, opts )
+      region = opts.delete( :region )
+      opts.delete( :lvm_volumes ) #unused here
+
+      ec2 = AWS::EC2.new.regions[ region ]
+
+      iopts = opts.dup
+      iopts.delete( :ebs_volumes )
+      iopts.delete( :ebs_volume_options )
+      iopts.delete( :roles )
+
+      if iopts[ :count ] && iopts[ :count ] != 1
+        raise ":count #{iopts[ :count ]} != 1 is not supported"
+      end
+
+      iopts[ :security_groups ].map! do |sg|
+        sg == :default ? region : sg
+      end
+
+      iopts[ :security_groups ].each do |sg|
+        aws_create_security_group( sg, region: region )
+      end
+
+      inst = ec2.instances.create( iopts )
+
+      inst.add_tag( 'Name', value: name )
+
+      if opts[ :roles ]
+        inst.add_tag( 'Roles', value: opts[ :roles ].join(' ') )
+      end
+
+      wait_for_running( inst )
+
+      # FIXME: Split method
+      # FIXME: Support alternative syntax, i.e
+      # { ebs_volumes: [ [4, size: 48], [2, size: 8] ] }
+
+      if opts[ :ebs_volumes ] > 0
+        vopts = { availability_zone: inst.availability_zone }.
+          merge( opts[ :ebs_volume_options ] )
+
+        attachments = opts[ :ebs_volumes ].times.map do |i|
+          vol = ec2.volumes.create( vopts )
+          wait_until( vol.id, 0.5 ) { vol.status == :available }
+          vol.attach_to( inst, "/dev/sdh#{i+1}" ) #=> Attachment
+        end
+
+        wait_until( "volumes to attach" ) do
+          !( attachments.any? { |a| a.status == :attaching } )
+        end
+      end
+      #FIXME: end
+
+      instance_to_props( region, inst )
+    end
+
+    # Create a Route53 DNS CNAME from iprops :name to :internet_name.
+    # Options are per {AWS::Route53::ResourceRecordSetCollection.create}[http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Route53/ResourceRecordSetCollection.html#create-instance_method]
+    # (currently undocumented) with the following additions:
+    #
+    # :domain_name:: name of the hosted zone and suffix for
+    #                CNAME. Should terminate in a DOT '.'
+    # :wait::        If true, wait for CNAME to resolve
+    def route53_create_host_cname( iprops, opts = {} )
+      opts = deep_merge_hashes( @route53_default_rs_options, opts )
+      dname = dot_terminate( opts.delete( :domain_name ) )
+      do_wait = opts.delete( :wait )
+      rs_opts = opts.
+        merge( resource_records: [ {value: iprops[:internet_name]} ] )
+
+      r53 = AWS::Route53.new
+      zone = r53.hosted_zones.find { |hz| hz.name == dname } or
+        raise "Route53 Hosted Zone name #{dname} not found"
+      long_name = [ iprops[:name], dname ].join('.')
+      zone.rrsets.create( long_name, 'CNAME', rs_opts )
+      wait_for_dns_resolve( long_name, dname ) if do_wait
+    end
+
+    def wait_for_dns_resolve( long_name,
+                              domain,
+                              rtype = Resolv::DNS::Resource::IN::CNAME )
+
+      ns_addr = Resolv::DNS.open( @resolver_options ) do |rvr|
+        ns_n = rvr.getresource( domain, Resolv::DNS::Resource::IN::SOA ).mname
+        rvr.getaddress( ns_n ).to_s
+      end
+
+      sleep 3 # Initial wait
+
+      wait_until( "#{long_name} to resolve", 3.0 ) do
+        Resolv::DNS.open( nameserver: ns_addr ) do |rvr|
+          rvr.getresources( long_name, rtype ).first
+        end
+      end
+    end
+
+    # Terminate an instance and wait for it to be terminated. If
+    # requested, /dev/sdh# attached EBS volumes which are not
+    # otherwise marked for :delete_on_termination will _also_ be
+    # terminated.  The minimum required properties in iprops are
+    # :region and :id.
+    #
+    # _WARNING_: data _will_ be lost!
+    def aws_terminate_instance( iprops, delete_attached_storage = false )
+      ec2 = AWS::EC2.new.regions[ iprops[ :region ] ]
+      inst = ec2.instances[ iprops[ :id ] ]
+      unless inst.exists?
+        raise "Instance #{iprops[:id]} does not exist in #{iprops[:region]}"
+      end
+
+      ebs_volumes = []
+      if delete_attached_storage
+        ebs_volumes = inst.block_devices.map do |dev|
+          ebs = dev[ :ebs ]
+          if ebs && dev[:device_name] =~ /dh\d+$/ && !ebs[:delete_on_termination]
+            ebs[ :volume_id ]
+          end
+        end.compact
+      end
+
+      inst.terminate
+      wait_until( "termination of #{inst.id}", 2.0 ) { inst.status == :terminated }
+
+      ebs_volumes = ebs_volumes.map do |vid|
+        volume = ec2.volumes[ vid ]
+        if volume.exists?
+          volume
+        else
+          puts "WARN: #{volume} doesn't exist"
+          nil
+        end
+      end.compact
+
+      ebs_volumes.each do |vol|
+        wait_until( "deletion of vol #{vol.id}" ) do
+          vol.status == :available || vol.status == :deleted
+        end
+        vol.delete if vol.status == :available
+      end
+
+    end
+
+    def wait_for_running( inst )
+      wait_until( "instance #{inst.id} to run", 2.0 ) { inst.status != :pending }
+      stat = inst.status
+      raise "Instance #{inst.id} has status #{stat}" unless stat == :running
+      nil
+    end
+
+    # Find running or pending instances in each region String and
+    # convert to a HostList.
+    def import_host_props( regions )
+      regions.inject([]) do |insts, region|
+        ec2 = AWS::EC2.new.regions[ region ]
+
+        found = ec2.instances.map do |inst|
+          next unless [ :running, :pending ].include?( inst.status )
+          instance_to_props( region, inst )
+        end
+
+        insts + found.compact
+      end
+
+    end
+
+    def instance_to_props( region, inst )
+      tags = inst.tags.to_h
+
+      { id:      inst.id,
+        region:  region,
+        ami:     inst.image_id,
+        name:    tags[ 'Name' ],
+        internet_name:  inst.dns_name,
+        internet_ip:    inst.ip_address,
+        internal_ip:    inst.private_ip_address,
+        instance_type:  inst.instance_type,
+        roles:   decode_roles( tags[ 'Roles' ] ) }
+    end
+
+    def decode_roles( roles )
+      ( roles || "" ).split( /\s+/ ).map { |r| r.to_sym }
+    end
+
+    # Wait until block returns truthy, sleeping for freq seconds
+    # between attempts. Writes desc and a sequence of DOTs on a single
+    # line until complete.
+    def wait_until( desc, freq = 1.0 )
+      $stdout.write( "Waiting for " + desc )
+      until (ret = yield) do
+        $stdout.write '.'
+        sleep freq
+      end
+      ret
+    ensure
+      puts
+    end
+
+    def dot_terminate( name )
+      ( name =~ /\.$/ ) ? name : ( name + '.' )
+    end
+
+    def deep_merge_hashes( h1, h2 )
+      h1.merge( h2 ) do |key, v1, v2|
+        if v1.is_a?( Hash ) && v2.is_a?( Hash )
+          deep_merge_hashes( v1, v2 )
+        else
+          v2
+        end
+      end
+    end
+  end
+end

data/lib/syncwrap/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2013 David Kellum
+# Copyright (c) 2011-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -15,5 +15,7 @@
 #++
 
 module SyncWrap
-  VERSION='1.5.2'
+  VERSION='2.0.0'
+
+  GEM_ROOT = File.dirname(File.dirname(File.dirname(__FILE__))) # :nodoc:
 end