zepplen_aws 0.0.1 → 0.0.2

data/README.md

@@ -0,0 +1,7 @@
+aws-tools
+=========
+
+Ruby AWS tools for common tasks
+
+Tools
+ 1. Automatic Route53 DNS Creation: zepplen_dns

data/bin/zepplen_dns

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby1.9.1
+require 'zepplen_aws'
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+	opts.banner = "Zepplen Tools: Route53 DNS Update"
+
+	options[:config_file] = nil
+	opts.on('--config-file FILE', 'YAML config file with options to load') do |file|
+		options[:config_file] = file
+	end
+
+	options[:aws_access_key_id] = nil
+	opts.on('--access-key-id AWS_ACCESS_KEY', 'AWS Access Key ID (Leave blank to use IAM Role auth)') do |key|
+		options[:aws_access_key_id] = key
+	end
+
+	options[:aws_secret_access_key] = nil
+	opts.on('--secret-access-key AWS_SECRET_KEY', 'AWS Secret Key (Leave blank to use IAM Role auth)') do |key|
+		options[:aws_secret_access_key] = key
+	end
+
+	#TODO: Figure out a good way to set this.... aws-sdk does not seem to have a good method at the moment
+	options[:aws_region] = nil
+	opts.on('--region REGION', 'AWS Region(s) to process') do |region|
+		options[:aws_region] = region
+	end
+
+	options[:ec2_tags] = nil
+	opts.on('--ec2-tags NAME-1,NAME-2', Array, 'EC2 tag name(s) to derive DNS from (name collisions resolved by order specified)') do |tag|
+		options[:ec2_tags] = tag
+	end
+
+	options[:elb] = false
+	opts.on('--elb', 'Enalbe ELB DNS (ELB names take precidence over EC2 tags if enabled)') do
+		options[:elb] = true
+	end
+
+	options[:public_zones] = nil
+	opts.on('--public-zones ZONE_ID,ZONE_NAME-2', Array, 'Rout53 zone to populate with public DNS/IP') do |zone|
+		options[:public_zones] = zone
+	end
+
+	options[:private_zones] = nil
+	opts.on('--private-zones ZONE_ID,ZONE_NAME-2', Array, 'Rout53 zone to populate with private DNS/IP') do |zone|
+		options[:private_zones] = zone
+	end
+
+	options[:public_zone_names] = nil
+	opts.on('--public-zone-names ZONE_NAME-1,ZONE_NAME-2', Array, 'Rout53 zone to populate with public DNS/IP  (Use --public-zone if multiple zones exist with the same name)') do |zone|
+		options[:public_zone_names] = zone
+	end
+
+	options[:private_zone_names] = nil
+	opts.on('--private-zone-names ZONE_NAME,ZONE_NAME-2', Array, 'Rout53 zone to populate with private DNS/IP (Use --private-zone if multiple zones exist with the same name)') do |zone|
+		options[:private_zone_names] = zone
+	end
+
+	options[:wildcards] = false
+	opts.on('--wildcards', 'Also create wildcard DNS entries') do
+		options[:wildcards] = true
+	end
+
+	options[:dns_type] = 'CNAME'
+	opts.on('--dns-record-type [TYPE]', [:CNAME, :A], 'Type of DNS entry to create (CNAME, ARECORD). ELBs will always get CNAME records') do |type|
+		options[:dns_type] = type
+	end 
+
+	options[:ttl] = 100
+	opts.on('--ttl SECONDS', Integer, 'DNS TTL') do |seconds|
+		options[:ttl] = seconds
+	end
+
+	options[:commit] = false
+	opts.on('--commit', 'Commit the changes') do
+		options[:commit] = true
+	end
+
+end.parse!
+
+#TODO: Add checks to validate the parameters
+ZepplenAWS::Env.options = options
+auto_dns = ZepplenAWS::AutoDNS.new()
+auto_dns.run!

data/lib/zepplen_aws/auto_dns.rb

@@ -0,0 +1,236 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+
+	# Utalized by the zepplen_dns script to update Route53 zones with dns entries based on
+	# the infrastructure.
+	#
+	# This class makes heavy use of the Env options to retrieve it's parameters.
+	class AutoDNS
+		require 'colorize'
+
+		def initialize()
+			@dns_precidence = 0
+			@dns_pool = []
+			@new_dns_pools = nil
+			@live_dns_pools = nil
+			@dns_entries = {}
+			@dns_entries[:public] = {}
+			@dns_entries[:private] = {}
+			@zones = {}
+			@zones[:public] = {}
+			@zones[:private] = {}
+			@options = Env::options
+			@ec2 = AWS::EC2.new()
+			@elb = AWS::ELB.new()
+			@route53 = AWS::Route53.new()
+			populate_zones()
+		end
+
+		# Process the existing infrusturecte as defined in the Env Options, and update
+		# Route 53 with the deltas.
+		def run!()
+			if(@options[:elb])
+				process_elbs()
+			end
+			next_precience()
+			if(@options[:ec2_tags])
+				@options[:ec2_tags].each do |tag_name|
+					process_ec2_tag_name(tag_name)
+					next_precience()
+				end
+			end
+			reduce_dns_pool()
+			process_zone_updates()
+		end
+
+		private
+
+		def populate_zones()
+			if(@options[:public_zones])
+				zone_id_search(@options[:public_zones], @zones[:public])
+			end
+			if(@options[:private_zones])
+				zone_id_search(@options[:private_zones], @zones[:private])
+			end
+			if(@options[:private_zone_names])
+				zone_name_search(@options[:private_zone_names], @zones[:private])
+			end
+		end
+
+		def zone_id_search(zone_ids, zone_pool)
+			zone_ids.each do |zone_id|
+				zone_pool[zone_id] = @route53.hosted_zones[zone_id]
+			end
+		end
+
+		def zone_name_search(zone_names, zone_pool)
+			@route53.hosted_zones.each do |hosted_zone|
+				zone_names.each do |zone_name|
+					if(hosted_zone.name == zone_name)
+						zone_pool[hosted_zone.id] = hosted_zone
+					end
+				end
+			end
+		end
+
+		def process_zone_updates()
+			@zones.each_pair do |pool_type, zones|
+				zones.each_pair do |zone_id, zone|
+					process_updates(pool_type, zone)
+				end
+			end
+		end
+
+		def process_updates(pool_type, zone)
+			if(@new_dns_pools.has_key?(pool_type))
+				live_zone_pool = {}
+				zone.resource_record_sets.each do |record|
+					if(record.type == 'CNAME' || record.type == 'A')
+						name = record.name.gsub(/^\\052/, '*')
+						live_zone_pool[name] = {:name => name, :type => record.type, :ttl => record.ttl , :value => record.resource_records}
+					end
+				end
+				to_add, to_delete = diff_zones(live_zone_pool, @new_dns_pools[pool_type], zone)
+				log_actions_to_take(to_add, to_delete, zone, pool_type)
+				if(@options[:commit])
+					change_batch = ::AWS::Route53::ChangeBatch.new(zone.id)
+					to_delete.each_pair do |key, dns_data|
+						change_batch << ::AWS::Route53::DeleteRequest.new(dns_data[:name], dns_data[:type], :ttl => dns_data[:ttl], :resource_records => dns_data[:value])
+					end
+					to_add.each_pair do |key, dns_data|
+						change_batch << ::AWS::Route53::CreateRequest.new("#{dns_data[:name]}.#{zone.name}", dns_data[:type], :ttl => dns_data[:ttl], :resource_records => dns_data[:value])
+					end
+					if(change_batch.length > 0)
+						change_batch.call
+					end
+				else
+					puts "Not in --commit mode".red
+				end
+			end
+		end
+
+		def log_actions_to_take(to_add, to_delete, zone, pool_type)
+			puts "Updating #{pool_type.to_s} #{zone.name}:#{zone.id}".green
+			if(to_delete.length > 0)
+				puts "Deleting:".yellow
+				to_delete.each_pair do |key, dns_data|
+					puts "\t#{dns_data[:name]}:".yellow
+					puts "\t\tType: #{dns_data[:type]}".yellow
+					puts "\t\tTTL: #{dns_data[:ttl]}".yellow
+					puts "\t\tValue: #{dns_data[:value]}".yellow
+				end
+			else
+				puts "Nothing To Delete".yellow
+			end
+			if(to_add.length > 0)
+				puts "Adding:".light_blue
+				to_add.each_pair do |key, dns_data|
+					puts "\t#{dns_data[:name]}.#{zone.name}:".light_blue
+					puts "\t\tType: #{dns_data[:type]}".light_blue
+					puts "\t\tTTL: #{dns_data[:ttl]}".light_blue
+					puts "\t\tValue: #{dns_data[:value]}".light_blue
+				end
+			else
+				puts "Nothing To Add".light_blue
+			end
+		end
+
+		def diff_zones(live_zone, new_zone, zone)
+			live_zone_agg = generate_zone_aggrigate(live_zone, '')
+			new_zone_agg = generate_zone_aggrigate(new_zone, ".#{zone.name}")
+			to_add_keys = new_zone_agg.keys - live_zone_agg.keys
+			to_del_keys = live_zone_agg.keys - new_zone_agg.keys
+			to_add = new_zone_agg.select{|k,v| to_add_keys.include?(k)}
+			to_del = live_zone_agg.select{|k,v| to_del_keys.include?(k)}
+			return [to_add, to_del]
+		end
+
+		def generate_zone_aggrigate(zone, domain)
+			zone_agg = {}
+			zone.each_pair do |name, zone_data|
+				key = "#{name}#{domain}|#{zone_data[:type]}|#{zone_data[:ttl]}|#{zone_data[:value].join(',')}"
+				zone_agg[key] = zone_data
+			end
+			return zone_agg
+		end
+
+		def reduce_dns_pool()
+			@new_dns_pools = {}
+			@dns_pool.each do |dns_set|
+				dns_set.each_pair do |key, dns_entries|
+					if(!@new_dns_pools.has_key?(key))
+						@new_dns_pools[key] = {}
+					end
+					dns_entries.each_pair do |name, dns_data|
+						if(!@new_dns_pools[key].has_key?(name))
+								@new_dns_pools[key][name] = dns_data
+						end
+					end
+				end
+			end
+		end
+
+		def next_precience()
+			@dns_precidence += 1
+			@dns_pool << @dns_entries
+			@dns_entries = {}
+			@dns_entries[:public] = {}
+			@dns_entries[:private] = {}
+		end
+
+		def process_ec2_tag_name(tag_name)
+			@ec2.instances.filter('instance-state-name', 'running').each do |instance|
+				if(instance.tags.has_key?(tag_name))
+					instance.tags[tag_name].split(',').each do |tag_value|
+						if(@options[:dns_type] == :CNAME)
+							add_dns_entry(tag_value, @options[:dns_type], @options[:ttl], instance.dns_name, instance.private_dns_name, @options[:wildcards])
+						else
+							add_dns_entry(tag_value, @options[:dns_type], @options[:ttl], instance.ip_address, instance.private_ip_address, @options[:wildcards])
+						end
+					end
+				end
+			end
+		end
+
+		def process_elbs()
+			@elb.load_balancers.each do |load_balancer|
+				add_dns_entry(load_balancer.name, :CNAME, @options[:ttl], load_balancer.dns_name, load_balancer.dns_name, @options[:wildcards])
+			end
+		end
+
+		def add_dns_entry(name, type, ttl, public_value, private_value, wildcards)
+			name = sanatize_name(name)
+			if(@dns_entries[:public].has_key?(name))
+				puts "Duplicate name found at same precidence level: #{name}"
+			else
+				@dns_entries[:public][name] = {:name => name, :type => type.to_s.upcase, :ttl => ttl , :value => [{:value => public_value}]}
+				@dns_entries[:private][name] = {:name => name, :type => type.to_s.upcase, :ttl => ttl , :value => [{:value => private_value}]}
+				if(wildcards)
+					name = "*.#{name}"
+					@dns_entries[:public][name] = {:name => name, :type => type.to_s.upcase, :ttl => ttl , :value => [{:value => public_value}]}
+					@dns_entries[:private][name] = {:name => name, :type => type.to_s.upcase, :ttl => ttl , :value => [{:value => private_value}]}
+				end
+			end
+		end
+
+		def sanatize_name(name)
+			name = name.gsub(/[^a-zA-Z0-9]+/, '-')
+			name = name.gsub(/^-*|-*$/, '')
+			return name.downcase
+		end
+		
+	end
+end

data/lib/zepplen_aws/aws/ec2.rb

@@ -0,0 +1,29 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+	module AWS
+		class EC2
+
+			def initialize()
+				@object = ::AWS::EC2.new()
+			end
+
+			def method_missing(method, *args)
+				@object.public_send(method, *args)
+			end
+
+		end
+	end
+end

data/lib/zepplen_aws/aws/elb.rb

@@ -0,0 +1,29 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+	module AWS
+		class ELB
+
+			def initialize()
+				@object = ::AWS::ELB.new()
+			end
+
+			def method_missing(method, *args)
+				@object.public_send(method, *args)
+			end
+
+		end
+	end
+end

data/lib/zepplen_aws/aws/route53.rb

@@ -0,0 +1,29 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+	module AWS
+		class Route53
+
+			def initialize()
+				@object = ::AWS::Route53.new()
+			end
+
+			def method_missing(method, *args)
+				@object.public_send(method, *args)
+			end
+
+		end
+	end
+end

data/lib/zepplen_aws/aws.rb

@@ -0,0 +1,36 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+
+	# Wrapper module for the aws-sdk Gem
+	#
+	# This module enables pasing authentication from the ENV to the AWS Libs.
+	# It also acts as a (very small) layer of abstraction between the 3rd party lib
+	# and our code.
+	module AWS
+		require 'aws-sdk'
+
+		autoload :EC2, 'zepplen_aws/aws/ec2'
+		autoload :ELB, 'zepplen_aws/aws/elb'
+		autoload :Route53, 'zepplen_aws/aws/route53'
+
+		def self.init!()
+			::AWS.config(:access_key_id => Env.options[:aws_access_key_id], :secret_access_key => Env.options[:aws_secret_access_key])
+		end
+
+	end
+end
+
+ZepplenAWS::AWS.init!

data/lib/zepplen_aws/env.rb

@@ -0,0 +1,107 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+module ZepplenAWS
+
+	# = Manage Environment
+	#
+	# This is a utility class used by the code to manage bassic nessisities of the Environment.
+	#
+	# == Options
+	#
+	# This class will automatically load parameters from multiple sources. AWS Keys are attempted
+	# to be retrieved from Environmental Variables (AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY).
+	# We can also load configuration from a YAML file. Our first attempt to locate a config file
+	# is by making use of the AWS_CONFIG_FILE Environmental Variable. If the varialbe points to a
+	# readable file, we will load configs from that. Beyond that we try to load configs from the
+	# following locations in order:
+	# 1. $HOME/.zepplen_aws.yaml
+	# 2. /etc/zepplen_aws/config.yaml
+	#
+	# If options are passed to us from another source
+	#   ZepplenAWS::Env.options = options
+	# these options will override any configuration pulled from the ENV or config files.
+	#
+	# To use AWS IAM Roles for authentication, leave the :aws_access_key_id and :aws_secret_access_key
+	# parameter empty.
+	class Env
+
+		# Allows user to inject options into the Environment
+		#
+		# @example Injecting Options With a Hash
+		#   ZepplenAWS::Env.options = options
+		#
+		# @param [Hash] options Options to inject into Environment
+		def self.options=(options)
+			if(options.has_key?(:config_file) && options[:config_file])
+				load_yaml(options[:config_file])
+			end
+			@options.merge!(options) do |key, old, new|
+				new == nil ? old : new
+			end
+			return nil
+		end
+
+		# Allows user to retrive the current option set
+		#
+		# @return [Hash] Current Options
+		def self.options()
+			return @options
+		end
+
+		# Allows user access to single option
+		#
+		# @param [Any] Key to retrieve
+		#
+		# @return [Any] Request option
+		def self.[](key)
+			return @options[key]
+		end
+
+
+		# Not intented for general use
+		#
+		# This function initializes the Env class, and will be called automatically uppon gem inclusion
+		def self.init!()
+			@options = {}
+			if(ENV.has_key?('AWS_ACCESS_KEY'))
+				@options[:aws_access_key_id] = ENV['AWS_ACCESS_KEY']
+			end
+			if(ENV.has_key?('AWS_SECRET_ACCESS_KEY'))
+				@options[:aws_secret_access_key] = ENV['AWS_SECRET_ACCESS_KEY']
+			end
+			if(ENV.has_key?('AWS_CONFIG_FILE') && File.readable?(ENV['AWS_CONFIG_FILE']))
+				load_yaml(ENV['AWS_CONFIG_FILE'])
+			elsif(File.readable?("#{ENV['HOME']}/.zepplen_aws.yaml"))
+				load_yaml("#{ENV['HOME']}/.zepplen_aws.yaml")
+			elsif(File.readable?('/etc/zepplen_aws/config.yaml'))
+				load_yaml('/etc/zepplen_aws/config.yaml')
+			end
+		end
+
+		# Not intended for general use
+		#
+		# This function import configuration from a YAML file
+		def self.load_yaml(file)
+			if(!File.readable?(file))
+				raise "Config File UnReadable: #{file}"
+			end
+			options = YAML::load_file(file)
+			@options.merge!(options) do |key, old, new|
+				new == nil ? old : new
+			end
+		end
+
+	end
+end

data/lib/zepplen_aws.rb

@@ -0,0 +1,25 @@
+#Copyright 2013 Mark Trimmer
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+require 'yaml'
+require 'colorize'
+require 'zepplen_aws/env'
+
+module ZepplenAWS
+	autoload :AWS, 'zepplen_aws/aws'
+	autoload :AutoDNS, 'zepplen_aws/auto_dns'
+end
+
+ZepplenAWS::Env.init!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zepplen_aws
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &14981440 !ruby/object:Gem::Requirement
+  requirement: &17350880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -21,10 +21,10 @@ dependencies:
         version: 1.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *14981440
+  version_requirements: *17350880
 - !ruby/object:Gem::Dependency
   name: colorize
-  requirement: &14979800 !ruby/object:Gem::Requirement
+  requirement: &17350200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,14 +32,24 @@ dependencies:
         version: 0.5.8
   type: :runtime
   prerelease: false
-  version_requirements: *14979800
+  version_requirements: *17350200
 description: AWS tools for common needs
 email: zepplen.aws@gmail.com
-executables: []
+executables:
+- zepplen_dns
 extensions: []
 extra_rdoc_files: []
-files: []
-homepage: 
+files:
+- lib/zepplen_aws/env.rb
+- lib/zepplen_aws/aws.rb
+- lib/zepplen_aws/auto_dns.rb
+- lib/zepplen_aws/aws/elb.rb
+- lib/zepplen_aws/aws/ec2.rb
+- lib/zepplen_aws/aws/route53.rb
+- lib/zepplen_aws.rb
+- bin/zepplen_dns
+- README.md
+homepage: https://github.com/zepplen/aws-tools
 licenses: []
 post_install_message: 
 rdoc_options: []