carinadigital-hiera-cloudformation 0.0.2.1 → 0.0.2.2

data/README.md

@@ -1,13 +1,17 @@
-# Hiera::Cloudformation
+# Multiregion Hiera::Cloudformation
 
-This backend for Hiera can retrieve information from:
+This a a fork of the hiera-cloudformaiton project (https://github.com/fanduel/hiera-cloudformation).
+It adds the ability to lookup stack outputs in multiple regions from a single install along with other
+small fixes.
+
+This a backend for Hiera can retrieve information from:
 
 * the outputs of a CloudFormation stack
 * the metadata of a resource in a stack
 
 ## Installation
 
-    gem install hiera-cloudformation
+    gem install carinadigital-hiera-cloudformation
 
 ## Usage
 
@@ -29,16 +33,15 @@ If you do not add these keys to your configuration file, the access keys will be
 the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables, or from an IAM
 instance role (if you are running Hiera on an EC2 instance with an IAM role assigned).
 
-The AWS region to use can also be configured in the `:cloudformation` section of hiera.yaml.
 You can also tell the backend to convert string literals "true", "false", "3.14", etc to Boolean
 or Number types with the `:parse_metadata` configuration option; this may be useful as
 CloudFormation will convert Booleans and Numbers in the template JSON metadata into Strings when
 retrieved from a stack resource:
 
     :cloudformation:
-      :region: 'us-west-1'
       :parse_metadata: true
 
+The AWS region to use can also be configured in the `:cloudformation` section of hiera.yaml.
 For use in multiple AWS regions, the region can be set by an interpolated variable.
 
     :cloudformation:

data/Rakefile

@@ -20,10 +20,10 @@ require 'rake/testtask'
 
 spec = Gem::Specification.new do |gem|
 	gem.name          = "carinadigital-hiera-cloudformation"
-	gem.version       = '0.0.2.1'
+	gem.version       = '0.0.2.2'
 	gem.authors       = ["carinadigital"]
 	gem.email         = ["carinadigital@gmail.com"]
-	gem.summary       = %q{CloudFormation backend for Hiera}
+	gem.summary       = %q{Multiregion CloudFormation backend for Hiera}
 	gem.description   = %q{Forked from hiera-cloudformation. Multiregion cloudFormation quieries of stack outputs or resource metadata for Hiera.}
 	gem.homepage      = "https://github.com/carinadigital/hiera-cloudformation"
 	gem.license       = "Apache License (2.0)"

data/lib/hiera/backend/cloudformation_backend.rb

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-	http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,217 +15,265 @@ limitations under the License.
 =end
 
 class Hiera
-	module Backend
-		class Cloudformation_backend
-			TIMEOUT = 60  # 1 minute timeout for AWS API response caching
-
-			def initialize
-				begin
-					require 'aws'
-					require 'timedcache'
-					require 'json'
-				rescue LoadError
-					require 'rubygems'
-					require 'aws'
-					require 'timedcache'
-					require 'json'
-				end
-
-				if Config.include?(:cloudformation) && !Config[:cloudformation].nil? then
-					if Config[:cloudformation].fetch(:parse_metadata, false) then
-						debug("Will convert CloudFormation stringified metadata back to numbers or booleans.")
-						@parse_metadata = true
-					else
-						@parse_metadata = false
-					end
-
-					@aws_config = {}
-					if Config[:cloudformation].include?(:access_key_id) && Config[:cloudformation].include?(:secret_access_key) then
-						debug("Found AWS access key #{Config[:cloudformation][:access_key_id]} from configuration")
-						@aws_config[:access_key_id] = Config[:cloudformation][:access_key_id]
-						@aws_config[:secret_access_key] = Config[:cloudformation][:secret_access_key]
-					end
-					if Config[:cloudformation].include?(:region) then
-						debug("Found AWS region #{Config[:cloudformation][:region]} from configuration")
-						@aws_config[:region] = Config[:cloudformation][:region]
-					end
-
-				else
-					debug("No configuration found, will fall back to env variables or IAM role")
-					@cf = AWS::CloudFormation.new
-				end
-
-				@output_cache = TimedCache.new
-				@resource_cache = TimedCache.new
-
-				debug("Hiera cloudformation backend loaded")
-			end
-
-
-			def create_connection(scope)
-
-				# If we already have a connection object then return early.
-				if defined? @cf then
-					return
-				end
-
-				# Interpolate the value from hiera.yaml
-				if @aws_config.include?(:region)
-					@aws_config[:region] = Backend.parse_answer(@aws_config[:region], scope)
-
-	 		        # Make an array of valid AWS regions.
-	 		        aws_region_names = []
-	 				AWS.regions.each do |region|
-                        aws_region_names.push(region.name)
+    module Backend
+        class Cloudformation_backend
+            TIMEOUT = 60  # 1 minute timeout for AWS API response caching
+
+            def initialize
+                begin
+                    require 'aws'
+                    require 'timedcache'
+                    require 'json'
+                rescue LoadError
+                    require 'rubygems'
+                    require 'aws'
+                    require 'timedcache'
+                    require 'json'
+                end
+
+                # Class variables
+                # Data shared amongst all instance of this class.
+                @@aws_config = {} # AWS access credentials from yaml.
+                # Caches are shared, so that multiple AWS instances from seperate threads can share cache data
+                # e.g. When a scaling event occurs.
+                @@output_cache = TimedCache.new(:default_timeout => 60)    #Default timeout in 60 seconds.
+                @@resource_cache = TimedCache.new(:default_timeout => 60)
+
+                # Class instance variables
+                # We want don't want two instances trying to reuse the same connection objects, and possible
+                # the same connection, so give each class it's own connection objects.
+                @cf = Hash.new # Variable for hash of connection options, keyed by region.
+
+
+                # Check our config key is present in hiera.yaml
+                if not Config.include?(:cloudformation) || Config[:cloudformation].nil? then
+                    error_message = "[cloudformation_backend]: No configuration found."
+                    Hiera.warn(error_message)
+                    raise Exception, error_message 
+                end
+
+                # Check we have the AWS access_key_id
+                # TODO we should have a fallback for access credentials as per documentaion.
+                if not Config[:cloudformation].include?(:access_key_id) then 
+                    error_message = "[cloudformation_backend]: :access_key_id missing in configuration."
+                    Hiera.warn(error_message)
+                    raise Exception, error_message 
+                end
+
+                #Check we have the AWS secret_access_key
+                if not Config[:cloudformation].include?(:secret_access_key) then
+                    error_message = "[cloudformation_backend]: :secret_access_key missing in configuration."
+                    Hiera.warn(error_message)
+                    raise Exception, error_message 
+                end
+
+                # Check we have the AWS region.
+                # This can be a static region name of an interpolated fact.
+                # TODO: This should be improved so it can fallback to default environment or 
+                # dot file values. 
+                if not Config[:cloudformation].include?(:region) then
+                    error_message = "[cloudformation_backend]: :region missing in configuration."
+                    Hiera.warn(error_message)
+                    raise Exception, error_message 
+                end
+
+                if Config[:cloudformation].fetch(:parse_metadata, false) then
+                    debug("Will convert CloudFormation stringified metadata back to numbers or booleans.")
+                    @parse_metadata = true
+                else
+                    @parse_metadata = false
+                end
+                
+                debug("Using AWS access key #{Config[:cloudformation][:access_key_id]}")
+                @@aws_config['access_key_id'] = Config[:cloudformation][:access_key_id]
+                @@aws_config['secret_access_key'] = Config[:cloudformation][:secret_access_key]
+                debug("Using AWS region #{Config[:cloudformation][:region]}")
+                @@aws_config['region'] = Config[:cloudformation][:region]
+                debug("Hiera cloudformation backend loaded")
+            end
+
+
+            def lookup(key, scope, order_override, resolution_type)
+                answer = nil
+                
+                # Lookups can potentially come from different agents in different AWS regions.
+                # Interpolate the value from hiera.yaml for this agent's region.
+                 if @@aws_config.include?('region')
+                    agent_region = Backend.parse_answer(@@aws_config['region'], scope)
+                end
+
+                # Idempotent connection creation of AWS connections for reuse.
+                  create_connection(agent_region)
+
+                Backend.datasources(scope, order_override) do |elem|
+                    case elem
+                    when /cfstack\/([^\/]+)\/outputs/
+                        debug("Looking up #{agent_region} #{$1} #{key} as an output of stack.")
+                        raw_answer = stack_output_query($1, key, agent_region)
+                    when /cfstack\/([^\/]+)\/resources\/([^\/]+)/
+                        debug("Looking up #{agent_region} #{$1} #{$2} #{key} in metadata of stack resource")
+                        raw_answer = stack_resource_query($1, $2, key,agent_region)
+                    else
+                        #debug("#{elem} doesn't seem to be a CloudFormation hierarchy element")
+                        next
                     end
 
-		    		# Check this is a valid aws region.
-                    if not aws_region_names.include? @aws_config[:region]
-                    	# If we don't have a region specified then the cloudformation endpoint will be malformed
-                    	# resulting in networking errors.
-                    	# Fail now with a proper error mesage.
-                    	error_message = "[cloudformation_backend]: AWS Region #{@aws_config[:region]} is invalid."
-   					    Hiera.warn(error_message)
-                        raise Exception, error_message
+                    next if raw_answer.nil?
+
+                    if @parse_metadata then
+                        raw_answer = convert_metadata(raw_answer)
                     end
 
-	 				debug("Using lookups from region #{@aws_config[:region]} for this run.")
-	 			end
-
-				if @aws_config.length != 0 then
-					@cf = AWS::CloudFormation.new(@aws_config)
-				else
-					debug("No AWS configuration found, will fall back to env variables or IAM role")
-					@cf = AWS::CloudFormation.new
-				end
-			end
-
-
-			def lookup(key, scope, order_override, resolution_type)
-				answer = nil
-
-				# Idempotent connection creation.
-	  			create_connection(scope)
-
-				Backend.datasources(scope, order_override) do |elem|
-					case elem
-					when /cfstack\/([^\/]+)\/outputs/
-						debug("Looking up #{key} as an output of stack #{$1}")
-						raw_answer = stack_output_query($1, key)
-					when /cfstack\/([^\/]+)\/resources\/([^\/]+)/
-						debug("Looking up #{key} in metadata of stack #{$1} resource #{$2}")
-						raw_answer = stack_resource_query($1, $2, key)
-					else
-						debug("#{elem} doesn't seem to be a CloudFormation hierarchy element")
-						next
-					end
-
-					next if raw_answer.nil?
-
-					if @parse_metadata then
-						raw_answer = convert_metadata(raw_answer)
-					end
-
-					new_answer = Backend.parse_answer(raw_answer, scope)
-
-					case resolution_type
-					when :array
-						raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
-						answer ||= []
-						answer << new_answer
-					when :hash
-						raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
-						answer ||= {}
-						answer = Backend.merge_answer(new_answer, answer)
-					else
-						answer = new_answer
-						break
-					end
-				end
-
-				return answer
-			end
-
-			def stack_output_query(stack_name, key)
-				outputs = @output_cache.get(stack_name)
-
-				if outputs.nil? then
-					debug("#{stack_name} outputs not cached, fetching...")
-					begin
-						outputs = @cf.stacks[stack_name].outputs
-					rescue AWS::CloudFormation::Errors::ValidationError
-						debug("Stack #{stack_name} outputs can't be retrieved")
-						outputs = []  # this is just a non-nil value to serve as marker in cache
-					end
-					@output_cache.put(stack_name, outputs, TIMEOUT)
-				end
-
-				output = outputs.select { |item| item.key == key }
-
-				return output.empty? ? nil : output.shift.value
-			end
-
-			def stack_resource_query(stack_name, resource_id, key)
-				metadata = @resource_cache.get({:stack => stack_name, :resource => resource_id})
-
-				if metadata.nil? then
-					debug("#{stack_name} #{resource_id} metadata not cached, fetching")
-					begin
-						metadata = @cf.stacks[stack_name].resources[resource_id].metadata
-					rescue AWS::CloudFormation::Errors::ValidationError
-						# Stack or resource doesn't exist
-						debug("Stack #{stack_name} resource #{resource_id} can't be retrieved")
-						metadata = "{}" # This is just a non-nil value to serve as marker in cache
-					end
-					@resource_cache.put({:stack => stack_name, :resource => resource_id}, metadata, TIMEOUT)
-				end
-
-				if metadata.respond_to?(:to_str) then
-					data = JSON.parse(metadata)
-
-					if data.include?('hiera') then
-						return data['hiera'][key] if data['hiera'].include?(key)
-					end
-				end
-
-				return nil
-			end
-
-			def convert_metadata(json_object)
-				if json_object.is_a?(Hash) then
-					# convert each value of a Hash
-					converted_object = {}
-					json_object.each do |key, value|
-						converted_object[key] = convert_metadata(value)
-					end
-					return converted_object
-				elsif json_object.is_a?(Array) then
-					# convert each item in an Array
-					return json_object.map { |item| convert_metadata(item) }
-				elsif json_object == "true" then
-					# Boolean literals
-					return true
-				elsif json_object == "false" then
-					return false
-				elsif json_object == "null" then
-					return nil
-				elsif /^-?([1-9]\d*|0)(.\d+)?([eE][+-]?\d+)?$/.match(json_object) then
-					# Numeric literals
-					if json_object.include?('.') then
-						return json_object.to_f
-					else
-						return json_object.to_i
-					end
-				else
-					return json_object
-				end
-			end
+                    new_answer = Backend.parse_answer(raw_answer, scope)
+
+                    case resolution_type
+                    when :array
+                        raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+                        answer ||= []
+                        answer << new_answer
+                    when :hash
+                        raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+                        answer ||= {}
+                        answer = Backend.merge_answer(new_answer, answer)
+                    else
+                        answer = new_answer
+                        break
+                    end
+                end
+
+                return answer
+            end
+
+
+            # Ensures that connetion is created for this region in the class variable for connection.
+            def create_connection(region)
+              
+                # If we already have a connection object then return early.
+                if @cf.has_key?(region) then
+                    return
+                end
+
+                debug("Creating new persistent aws connection for region #{region}.")                       
+             
+                # Check this is a valid aws region.
+                if not is_aws_region_name?(region)
+                    # If we don't have a region specified then the cloudformation endpoint will be malformed
+                    # resulting in networking errors.
+                    # Fail now with a proper error mesage.
+                    error_message = "[cloudformation_backend]: AWS Region #{region} is invalid."
+                        Hiera.warn(error_message)
+                    raise Exception, error_message
+                end
+
+                # Create an AWS connecton object for this region.
+                @cf[region] = AWS::CloudFormation.new(
+                  :access_key_id => @@aws_config['access_key_id'],
+                    :secret_access_key => @@aws_config['secret_access_key'],
+                    :region => region
+                )
+            end
+
+
+            def stack_output_query(stack_name, key, region)
+                outputs = @@output_cache.get(region+stack_name)
+
+                if outputs.nil? then
+                    debug("#{stack_name} outputs not cached, fetching...")
+                    begin
+                         outputs = @cf[region].stacks[stack_name].outputs
+                    rescue AWS::CloudFormation::Errors::ValidationError
+                        debug("Stack #{stack_name} outputs can't be retrieved")
+                        outputs = []  # this is just a non-nil value to serve as marker in cache
+                    end
+                    @@output_cache.put(region+stack_name, outputs, TIMEOUT)
+                end
 
-            # Custom function to wrap our debug messages to make them easier to find in the output.
-		    def debug(message)
-		      Hiera.debug("[cloudformation_backend]: #{message}")
-		    end
+                output = outputs.select { |item| item.key == key }
+
+                return output.empty? ? nil : output.shift.value
+            end
+
+
+            def stack_resource_query(stack_name, resource_id, key, region)
+                metadata = @@resource_cache.get({:stack => region+stack_name, :resource => resource_id})
+
+                if metadata.nil? then
+                    debug("#{stack_name} #{resource_id} metadata not cached, fetching")
+                    begin
+                        metadata = @cf[region].stacks[stack_name].resources[resource_id].metadata
+                    rescue AWS::CloudFormation::Errors::ValidationError
+                        # Stack or resource doesn't exist
+                        debug("Stack #{stack_name} resource #{resource_id} can't be retrieved")
+                        metadata = "{}" # This is just a non-nil value to serve as marker in cache
+                    end
+                    @@resource_cache.put({:stack => region+stack_name, :resource => resource_id}, metadata, TIMEOUT)
+                end
+
+                if metadata.respond_to?(:to_str) then
+                    data = JSON.parse(metadata)
 
-		end
-	end
+                    if data.include?('hiera') then
+                        return data['hiera'][key] if data['hiera'].include?(key)
+                    end
+                end
+
+                return nil
+            end
+
+
+            def convert_metadata(json_object)
+                if json_object.is_a?(Hash) then
+                    # convert each value of a Hash
+                    converted_object = {}
+                    json_object.each do |key, value|
+                        converted_object[key] = convert_metadata(value)
+                    end
+                    return converted_object
+                elsif json_object.is_a?(Array) then
+                    # convert each item in an Array
+                    return json_object.map { |item| convert_metadata(item) }
+                elsif json_object == "true" then
+                    # Boolean literals
+                    return true
+                elsif json_object == "false" then
+                    return false
+                elsif json_object == "null" then
+                    return nil
+                elsif /^-?([1-9]\d*|0)(.\d+)?([eE][+-]?\d+)?$/.match(json_object) then
+                    # Numeric literals
+                    if json_object.include?('.') then
+                        return json_object.to_f
+                    else
+                        return json_object.to_i
+                    end
+                else
+                    return json_object
+                end
+            end
+
+
+            # Custom function to wrap our debug messages to make them easier to find in the output.
+            def debug(message)
+              Hiera.debug("[cloudformation_backend]: #{message}")
+            end
+
+
+            # Check region name is a valid AWS regoion
+            def is_aws_region_name?(name)
+                   # Make an array of valid AWS regions.
+                 aws_region_names = []
+                 AWS.regions.each do |aws_region|
+                    aws_region_names.push(aws_region.name)
+                end
+
+                # Check this is a valid aws region.
+                if aws_region_names.include?(name)
+                    return true
+                else
+                    return false
+                end
+            end
+
+        end
+    end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carinadigital-hiera-cloudformation
 version: !ruby/object:Gem::Version
-  version: 0.0.2.1
+  version: 0.0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-19 00:00:00.000000000 Z
+date: 2014-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -112,6 +112,6 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: CloudFormation backend for Hiera
+summary: Multiregion CloudFormation backend for Hiera
 test_files:
 - test/convert_metadata_test.rb