aws_helper 0.0.1

data/README.md

@@ -0,0 +1,42 @@
+# aws_helper
+
+Aws Helper for an instance
+
+Allows functions on EBS volumes, snapshots, IP addresses and more 
+* initially snapshots are supported
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'aws_helper'
+    
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install aws_helper
+
+## Usage
+
+Snapshot EBS attached to device /dev/sdf
+
+    export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
+    export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
+    export HTTP_PROXY=http://myproxy:port
+    aws_helper snap /dev/sdf --description zzzzzzzzz
+
+Prune so only keep 7 snapshots: 
+
+    export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
+    export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
+    aws_helper snap_prune /dev/sdf --snapshots_to_keep=7
+    
+NOTE: Best Practice is for your server to have an IAM role then you don't
+need to specify AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+    
+Other functions to follow    
+
+

data/aws_helper.gemspec

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'awshelper/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'aws_helper'
+  s.version       = Awshelper::VERSION
+  s.authors       = ['Neill Turner']
+  s.email         = ['neillwturner@gmail.com']
+  s.homepage      = 'https://github.com/neillturner/aws_helper'
+  s.summary       = 'Aws Helper for an instance'
+  candidates = Dir.glob('{lib}/**/*') +  ['README.md', 'aws_helper.gemspec']
+  candidates = candidates +  Dir.glob("bin/*")
+  s.files = candidates.sort
+  s.platform      = Gem::Platform::RUBY
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
+  s.add_dependency('right_aws')
+  s.add_dependency('thor')  
+  s.rubyforge_project = '[none]'
+  s.description = <<-EOF
+== DESCRIPTION:
+
+Aws Helper for an instance 
+
+== FEATURES:
+
+Allows functions on EBS volumes, snapshots, IP addresses and more 
+
+EOF
+
+end

data/bin/aws_helper

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "awshelper/cli"
+
+Awshelper::CLI.start(ARGV)

data/lib/awshelper.rb

@@ -0,0 +1,2 @@
+module Awshelper
+end

data/lib/awshelper/cli.rb

@@ -0,0 +1,302 @@
+require 'thor'
+require 'awshelper'
+require 'awshelper/ec2'
+require 'syslog'
+
+module Awshelper
+  class CLI < Thor
+    include Thor::Actions
+
+    include Awshelper::Ec2
+    
+#def ebs_create(volume_id, snapshot_id, most_recent_snapshot)
+#  #TO DO 
+#  raise "Cannot create a volume with a specific id (EC2 chooses volume ids)" if volume_id
+#  if snapshot_id =~ /vol/
+#    new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
+#  end
+#
+#  #nvid = volume_id_in_node_data
+#  #if nvid
+#  #  # volume id is registered in the node data, so check that the volume in fact exists in EC2
+#  #  vol = volume_by_id(nvid)
+#  #  exists = vol && vol[:aws_status] != "deleting"
+#  #  # TODO: determine whether this should be an error or just cause a new volume to be created. Currently erring on the side of failing loudly
+#  #  raise "Volume with id #{nvid} is registered with the node but does not exist in EC2. To clear this error, remove the ['aws']['ebs_volume']['#{new_resource.name}']['volume_id'] entry from this node's data." unless exists
+#  #else
+#    # Determine if there is a volume that meets the resource's specifications and is attached to the current
+#    # instance in case a previous [:create, :attach] run created and attached a volume but for some reason was
+#    # not registered in the node data (e.g. an exception is thrown after the attach_volume request was accepted
+#    # by EC2, causing the node data to not be stored on the server)
+#    if new_resource.device && (attached_volume = currently_attached_volume(instance_id, new_resource.device))
+#      Chef::Log.debug("There is already a volume attached at device #{new_resource.device}")
+#      compatible = volume_compatible_with_resource_definition?(attached_volume)
+#      raise "Volume #{attached_volume[:aws_id]} attached at #{attached_volume[:aws_device]} but does not conform to this resource's specifications" unless compatible
+#      Chef::Log.debug("The volume matches the resource's definition, so the volume is assumed to be already created")
+#      converge_by("update the node data with volume id: #{attached_volume[:aws_id]}") do
+#        node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = attached_volume[:aws_id]
+#        node.save unless Chef::Config[:solo]
+#      end
+#    else
+#      # If not, create volume and register its id in the node data
+#      converge_by("create a volume with id=#{new_resource.snapshot_id} size=#{new_resource.size} availability_zone=#{new_resource.availability_zone} and update the node data with created volume's id") do
+#      nvid = create_volume(new_resource.snapshot_id,
+#                           new_resource.size,
+#                           new_resource.availability_zone,
+#                           new_resource.timeout,
+#                           new_resource.volume_type,
+#                           new_resource.piops)
+#        node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = nvid
+#        node.save unless Chef::Config[:solo]
+#      end
+#    end
+#  #end
+#end
+
+#def ebs_attach(device, volume_id, timeout)
+#  # determine_volume returns a Hash, not a Mash, and the keys are
+#  # symbols, not strings.
+#  vol = determine_volume(device, volume_id)
+#  if vol[:aws_status] == "in-use"
+#    if vol[:aws_instance_id] != instance_id
+#      raise "Volume with id #{vol[:aws_id]} exists but is attached to instance #{vol[:aws_instance_id]}"
+#    else
+#      Chef::Log.debug("Volume is already attached")
+#    end
+#  else
+#      # attach the volume
+#      attach_volume(vol[:aws_id], instance_id, device, timeout)
+#  end
+#end
+
+#def ebs_detach(device, volume_id, timeout)
+#  vol = determine_volume(device, volume_id)
+#  detach_volume(vol[:aws_id], timeout)
+#end
+
+desc "snap DEVICE [VOLUME_ID]", "Take a snapshot of a EBS Disk."
+option :description 
+
+long_desc <<-LONGDESC
+  'snap DEVICE [VOLUME_ID] --description xxxxxx' 
+  \x5 Take a snapshot of a EBS Disk by specifying device and/or volume_id.
+  \x5 All commands rely on environment variables 
+    \x5   export AWS_ACCESS_KEY_ID ='xxxxxxxxxx'
+    \x5   export AWS_SECRET_ACCESS_KEY ='yyyyyy'
+  \x5 For example 
+    \x5    aws_helper snap /dev/sdf
+  \x5 will snap shot the EBS disk attach to device /dev/xvdj  
+LONGDESC
+
+def snap(device, volume_id=nil)
+  vol = determine_volume(device, volume_id)
+  snap_description = options[:description] if options[:description] 
+  snap_description = "Created by aws_helper(#{instance_id}/#{local_ipv4}) for #{ami_id} from #{vol[:aws_id]}" if !options[:description]
+  snapshot = ec2.create_snapshot(vol[:aws_id],snap_description)
+  log("Created snapshot of #{vol[:aws_id]} as #{snapshot[:aws_id]}")
+end
+
+desc "snap_prune DEVICE [VOLUME_ID]", "Prune the number of snapshots."
+option :snapshots_to_keep, :type => :numeric, :required => true  
+
+long_desc <<-LONGDESC
+  'snap_prune DEVICE [VOLUME_ID] --snapshots_to_keep=<numeric>' 
+  \x5 Prune the number of snapshots of a EBS Disk by specifying device and/or volume_id and the no to keep.
+   \x5 All commands rely on environment variables 
+    \x5    export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
+    \x5    export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
+  \x5 For example 
+    \x5    aws_helper snap_prune /dev/sdf --snapshots_to_keep=7
+  \x5 will keep the last 7 snapshots of the EBS disk attach to device /dev/xvdj  
+LONGDESC
+
+def snap_prune(device, volume_id=nil)
+  snapshots_to_keep = options[:snapshots_to_keep]
+  vol = determine_volume(device, volume_id)
+  old_snapshots = Array.new
+  log("Checking for old snapshots")
+  ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+    if snapshot[:aws_volume_id] == vol[:aws_id]
+      log("Found old snapshot #{snapshot[:aws_id]} (#{snapshot[:aws_volume_id]}) #{snapshot[:aws_started_at]}")
+      old_snapshots << snapshot
+    end
+  end
+  if old_snapshots.length > snapshots_to_keep
+    old_snapshots[snapshots_to_keep, old_snapshots.length].each do |die|
+      log("Deleting old snapshot #{die[:aws_id]}")
+      ec2.delete_snapshot(die[:aws_id])
+    end
+  end
+end
+
+private
+
+def log(message,type="info")
+  # $0 is the current script name
+  puts message
+  Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "info"
+  Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "err"
+end
+
+# Pulls the volume id from the volume_id attribute or the node data and verifies that the volume actually exists
+def determine_volume(device, volume_id)
+  vol = currently_attached_volume(instance_id, device)
+  vol_id = volume_id || ( vol ? vol[:aws_id] : nil )
+  log("volume_id attribute not set and no volume is attached at the device #{device}",'err') unless vol_id
+  raise "volume_id attribute not set and no volume is attached at the device #{device}" unless vol_id
+
+  # check that volume exists
+  vol = volume_by_id(vol_id)
+  log("No volume with id #{vol_id} exists",'err') unless vol
+  raise "No volume with id #{vol_id} exists" unless vol
+
+  vol
+end
+
+# Retrieves information for a volume
+def volume_by_id(volume_id)
+  ec2.describe_volumes.find{|v| v[:aws_id] == volume_id}
+end
+
+# Returns the volume that's attached to the instance at the given device or nil if none matches
+def currently_attached_volume(instance_id, device)
+  ec2.describe_volumes.find{|v| v[:aws_instance_id] == instance_id && v[:aws_device] == device}
+end
+
+# Returns true if the given volume meets the resource's attributes
+#def volume_compatible_with_resource_definition?(volume)
+#  if new_resource.snapshot_id =~ /vol/
+#    new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
+#  end
+#  (new_resource.size.nil? || new_resource.size == volume[:aws_size]) &&
+#  (new_resource.availability_zone.nil? || new_resource.availability_zone == volume[:zone]) &&
+#  (new_resource.snapshot_id.nil? || new_resource.snapshot_id == volume[:snapshot_id])
+#end
+
+# TODO: support tags in deswcription 
+#def tag_value(instance,tag_key)
+#       options = ec2.describe_tags({:filters => {:resource_id   => instance }} )
+# end     
+
+# Creates a volume according to specifications and blocks until done (or times out)
+def create_volume(snapshot_id, size, availability_zone, timeout, volume_type, piops)
+  availability_zone ||= instance_availability_zone
+
+  # Sanity checks so we don't shoot ourselves.
+  raise "Invalid volume type: #{volume_type}" unless ['standard', 'io1', 'gp2'].include?(volume_type)
+
+  # PIOPs requested. Must specify an iops param and probably won't be "low".
+  if volume_type == 'io1'
+    raise 'IOPS value not specified.' unless piops >= 100
+  end
+
+  # Shouldn't see non-zero piops param without appropriate type.
+  if piops > 0
+    raise 'IOPS param without piops volume type.' unless volume_type == 'io1'
+  end
+
+  create_volume_opts = { :volume_type => volume_type }
+  # TODO: this may have to be casted to a string.  rightaws vs aws doc discrepancy.
+  create_volume_opts[:iops] = piops if volume_type == 'io1'
+
+  nv = ec2.create_volume(snapshot_id, size, availability_zone, create_volume_opts)
+  Chef::Log.debug("Created new volume #{nv[:aws_id]}#{snapshot_id ? " based on #{snapshot_id}" : ""}")
+
+  # block until created
+  begin
+    Timeout::timeout(timeout) do
+      while true
+        vol = volume_by_id(nv[:aws_id])
+        if vol && vol[:aws_status] != "deleting"
+          if ["in-use", "available"].include?(vol[:aws_status])
+            Chef::Log.info("Volume #{nv[:aws_id]} is available")
+            break
+          else
+            Chef::Log.debug("Volume is #{vol[:aws_status]}")
+          end
+          sleep 3
+       else
+          raise "Volume #{nv[:aws_id]} no longer exists"
+        end
+      end
+    end
+  rescue Timeout::Error
+    raise "Timed out waiting for volume creation after #{timeout} seconds"
+  end
+
+  nv[:aws_id]
+end
+
+# Attaches the volume and blocks until done (or times out)
+def attach_volume(volume_id, instance_id, device, timeout)
+  Chef::Log.debug("Attaching #{volume_id} as #{device}")
+  ec2.attach_volume(volume_id, instance_id, device)
+
+  # block until attached
+  begin
+    Timeout::timeout(timeout) do
+      while true
+        vol = volume_by_id(volume_id)
+        if vol && vol[:aws_status] != "deleting"
+          if vol[:aws_attachment_status] == "attached"
+            if vol[:aws_instance_id] == instance_id
+              Chef::Log.info("Volume #{volume_id} is attached to #{instance_id}")
+              break
+            else
+              raise "Volume is attached to instance #{vol[:aws_instance_id]} instead of #{instance_id}"
+            end
+          else
+            Chef::Log.debug("Volume is #{vol[:aws_status]}")
+          end
+          sleep 3
+        else
+          raise "Volume #{volume_id} no longer exists"
+        end
+      end
+    end
+  rescue Timeout::Error
+    raise "Timed out waiting for volume attachment after #{timeout} seconds"
+  end
+end
+
+# Detaches the volume and blocks until done (or times out)
+def detach_volume(volume_id, timeout)
+  vol = volume_by_id(volume_id)
+  if vol[:aws_instance_id] != instance_id
+    Chef::Log.debug("EBS Volume #{volume_id} is not attached to this instance (attached to #{vol[:aws_instance_id]}). Skipping...")
+    return
+  end
+  Chef::Log.debug("Detaching #{volume_id}")
+  orig_instance_id = vol[:aws_instance_id]
+  ec2.detach_volume(volume_id)
+
+  # block until detached
+  begin
+    Timeout::timeout(timeout) do
+      while true
+        vol = volume_by_id(volume_id)
+        if vol && vol[:aws_status] != "deleting"
+          if vol[:aws_instance_id] != orig_instance_id
+            Chef::Log.info("Volume detached from #{orig_instance_id}")
+            break
+          else
+            Chef::Log.debug("Volume: #{vol.inspect}")
+          end
+        else
+          Chef::Log.debug("Volume #{volume_id} no longer exists")
+          break
+        end
+        sleep 3
+      end
+    end
+  rescue Timeout::Error
+    raise "Timed out waiting for volume detachment after #{timeout} seconds"
+  end
+end
+
+
+end
+
+end
+
+

data/lib/awshelper/ec2.rb

@@ -0,0 +1,141 @@
+#
+# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'open-uri'
+require 'syslog'
+require 'right_aws'
+
+
+  module Awshelper
+    module Ec2
+      def find_snapshot_id(volume_id="", find_most_recent=false)
+        snapshot_id = nil
+        snapshots = if find_most_recent
+          ec2.describe_snapshots.sort { |a,b| a[:aws_started_at] <=> b[:aws_started_at] }
+        else
+          ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }
+        end
+        snapshots.each do |snapshot|
+          if snapshot[:aws_volume_id] == volume_id
+            snapshot_id = snapshot[:aws_id]
+          end
+        end
+        log("Cannot find snapshot id!",'err') unless snapshot_id
+        raise "Cannot find snapshot id!" unless snapshot_id
+        log("Snapshot ID is #{snapshot_id}")
+        snapshot_id
+      end
+
+      def ec2
+        @@ec2 ||= create_aws_interface
+        #(RightAws::Ec2)
+      end
+
+      def instance_id
+        @@instance_id ||= query_instance_id
+      end
+
+      def ami_id
+        @@ami_id ||= query_ami_id
+      end
+
+      def local_ipv4
+        @@local_ipv4 ||= query_local_ipv4
+      end      
+ 
+      def instance_availability_zone
+        @@instance_availability_zone ||= query_instance_availability_zone
+      end
+      
+      private
+
+      def create_aws_interface  
+      
+        region = instance_availability_zone
+        region = region[0, region.length-1]
+        
+        aws_access_key = ENV['AWS_ACCESS_KEY_ID']
+        aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+
+        if aws_access_key and aws_secret_access_key
+          RightAws::Ec2.new(aws_access_key,aws_secret_access_key, {:region => region}) # :logger => Chef::Log,
+        else
+          log("No env var AWS_ACCESS_KEY_ID and 'AWS_SECRET_ACCESS_KEY so trying role credentials")
+          creds = query_role_credentials
+          RightAws::Ec2.new(creds['AccessKeyId'], creds['SecretAccessKey'], { :region => region, :token => creds['Token']})
+        end
+      end
+
+      def query_role
+        r = open("http://169.254.169.254/latest/meta-data/iam/security-credentials/",options = {:proxy => proxy}).readlines.first
+        r
+      end
+
+      def query_role_credentials(role = query_role)
+        log("Instance has no IAM role.",'err') if role.to_s.empty?
+        fail "Instance has no IAM role." if role.to_s.empty?
+        creds = open("http://169.254.169.254/latest/meta-data/iam/security-credentials/#{role}",options = {:proxy => proxy}){|f| JSON.parse(f.string)}
+        log("Retrieved instance credentials for IAM role #{role}")
+        creds
+      end
+
+      def query_instance_id
+        instance_id = open('http://169.254.169.254/latest/meta-data/instance-id',options = {:proxy => proxy}){|f| f.gets}
+        log("Cannot find instance id!",'err') unless instance_id
+        raise "Cannot find instance id!" unless instance_id
+        log("Instance ID is #{instance_id}")
+        instance_id
+      end
+      
+      def query_ami_id
+        ami_id = open('http://169.254.169.254/latest/meta-data/ami-id',options = {:proxy => proxy}){|f| f.gets}
+        log("Cannot find ami id!",'err') unless ami_id
+        raise "Cannot find instance id!" unless ami_id
+        log("Aim ID is #{ami_id}")
+        ami_id
+      end 
+      
+      def query_local_ipv4
+        local_ipv4 = open('http://169.254.169.254/latest/meta-data/local-ipv4',options = {:proxy => proxy}){|f| f.gets}
+        log("Cannot find local_ipv4!",'err') unless local_ipv4
+        raise "Cannot find local_ipv4!" unless local_ipv4
+        log("local_ipv4 is #{local_ipv4}")
+        local_ipv4
+      end      
+
+      def query_instance_availability_zone
+        availability_zone = open('http://169.254.169.254/latest/meta-data/placement/availability-zone/', options = {:proxy => proxy}){|f| f.gets}
+        log("Cannot find availability zone!",'err') unless availability_zone
+        raise "Cannot find availability zone!" unless availability_zone
+        log("Instance's availability zone is #{availability_zone}")
+        availability_zone
+      end
+      
+      def log(message,type="info")
+        # $0 is the current script name
+        puts message
+        Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "info"
+        Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "err"
+      end
+      
+      def proxy
+         p = ENV['HTTP_PROXY'] if ENV['HTTP_PROXY']
+         p
+      end   
+    end
+  end
+

data/lib/awshelper/elb.rb

@@ -0,0 +1,12 @@
+require File.join(File.dirname(__FILE__), 'ec2')
+
+  module AwsHelper
+    module Elb
+      include AwsHelper::Ec2
+
+      def elb
+        @@elb ||= create_aws_interface(RightAws::ElbInterface)
+      end
+    end
+  end
+end

data/lib/awshelper/version.rb

@@ -0,0 +1,3 @@
+module Awshelper
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: aws_helper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Neill Turner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: right_aws
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "== DESCRIPTION:\n\nAws Helper for an instance \n\n== FEATURES:\n\nAllows
+  functions on EBS volumes, snapshots, IP addresses and more \n\n"
+email:
+- neillwturner@gmail.com
+executables:
+- aws_helper
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- aws_helper.gemspec
+- bin/aws_helper
+- lib/awshelper.rb
+- lib/awshelper/cli.rb
+- lib/awshelper/ec2.rb
+- lib/awshelper/elb.rb
+- lib/awshelper/version.rb
+homepage: https://github.com/neillturner/aws_helper
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: ! '[none]'
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Aws Helper for an instance
+test_files: []