hiera-cloudformation 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA512: 
+  data.tar.gz: 528e5773c1de04e1eb38d19c73ae8629fac0b0ebe235694c8bdb724b666d4aee0badafa093bd34e1f7b840b1ed92b8788ce3f3b045d390cd11f0ccfc66e6766c
+  metadata.gz: 891a69aff5ed2455a88d4e3c008084bf7e964a4cd36227258977232881331e1f0a230ddf2d4504a5d43870779b4baf105ca5f0348d894ca55d2d8862e102d1b7
+SHA1: 
+  data.tar.gz: 048f0606a4b2313a1de2e20732a136b2fb071b33
+  metadata.gz: 1fb1aba71ac1d4d39bf325d21a0af7ab1ae484b3

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Hugh Cole-Baker
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,103 @@
+# Hiera::Cloudformation
+
+This backend for Hiera can retrieve information from:
+
+* the outputs of a CloudFormation stack
+* the metadata of a resource in a stack
+
+## Installation
+
+    gem install hiera-cloudformation
+
+## Usage
+
+Add the backend to the list of backends in hiera.yaml:
+
+    ---
+    :backends:
+      - yaml
+      - cloudformation
+
+To provide the backend with an AWS access key, you can add the following configuration to the
+`:cloudformation` section in hiera.yaml:
+
+    :cloudformation:
+      :access_key_id: Your_AWS_Access_Key_ID_Here
+      :secret_access_key: Your_AWS_Secret_Access_Key_Here
+
+If you do not add these keys to your configuration file, the access keys will be looked up from
+the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables, or from an IAM
+instance role (if you are running Hiera on an EC2 instance with an IAM role assigned).
+
+To use this backend you also need to add entries to your "hierarchy" in your hiera.yaml file.
+If you put an entry of this form in your hierarchy:
+
+    cfstack/<stack name>/outputs
+
+the backend will request the outputs of the stack named `<stack name>` and search for an output
+named the same as the requested key. If an output named the same as the key is found, the backend
+will return the value of that output (as a string).
+If you put an entry of this form in your hierarchy:
+
+    cfstack/<stack name>/resources/<logical resource ID>
+
+the backend will request the JSON metadata of the logical resource identified by the given ID,
+in the named stack, and look for a JSON object under the top-level key "hiera" in the metadata.
+If a JSON object is present under the top-level key "hiera", the backend will search for the
+requested key in that JSON object, and return the value of the key if present.
+
+The recommended way of constructing your hierarchy so that Puppet-managed nodes can retrieve
+data relating to the CloudFormation stack they're part of, is to create facts on the Puppet nodes
+describing what CloudFormation stack they're part of and what logical resource ID corresponds to
+their instance. Then, assuming you have two facts "cloudformation_stack" and "cloudformation_resource"
+reported on your node, you can add these entries to the "hierarchy" in hiera.yaml:
+
+    - cfstack/%{cloudformation_stack}/outputs
+    - cfstack/%{cloudformation_stack}/resources/%{cloudformation_resource}
+
+and Hiera will replace the `%{...}` placeholders with the value of the facts on the node, enabling
+the node to look up information about the stack it "belongs" to.
+
+For example, if this snippet is included in your CloudFormation template, and you include an entry
+
+    cfstack/<stack name>/outputs
+
+in your hierarchy, you can look up the key "example_key" in Hiera and get the value of the logical
+resource identified by "AWSEC2Instance" (for example, this might be an EC2 instance, so Hiera
+would return the instance ID)
+
+    "Outputs" : {
+      "example_key": {
+        "Description" : "AWSEC2Instance is the logical resource ID of the instance that was created",
+        "Value" : { "Ref": "AWSEC2Instance" }
+      }
+    }
+
+As another example, if you define an EC2 instance with the following snippet in your CloudFormation
+template, including some metadata:
+
+    "Resources" : {
+      "MyIAMKey" : {
+        "Type" : "AWS::IAM::AccessKey",
+        ...
+      },
+      "AWSEC2Instance" : {
+        "Type" : "AWS::EC2::Instance",
+        "Properties" : {
+          ...
+        },
+        "Metadata" : {
+          "hiera" : {
+            "class::access_key_id": { "Ref": "MyIAMKey" },
+            "class::secret_access_key": { "Fn::GetAtt" : [ "MyIAMKey" , "SecretAccessKey" ] }
+          }
+        }
+      }
+    }
+
+and you include an entry:
+
+    cfstack/<stack name>/resources/AWSEC2Instance
+
+in your hierarchy, you can query hiera for the key "class::access_key_id" or "class::secret_access_key"
+and retrieve the attributes of the "MyIAMKey" resource created by CloudFormation.

data/Rakefile

@@ -0,0 +1,43 @@
+=begin
+Copyright 2013 FanDuel Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=end
+
+require 'rubygems'
+require 'rubygems/package_task'
+
+spec = Gem::Specification.new do |gem|
+	gem.name          = "hiera-cloudformation"
+	gem.version       = '0.0.1'
+	gem.authors       = ["Hugh Cole-Baker"]
+	gem.email         = ["hugh@fanduel.com"]
+	gem.summary       = %q{CloudFormation backend for Hiera}
+	gem.description   = %q{Queries CloudFormation stack outputs or resource metadata for Hiera data}
+	gem.homepage      = "https://github.com/fanduel/hiera-cloudformation"
+	gem.license       = "Apache License (2.0)"
+
+	gem.files         = Dir['{bin,lib,man,test,spec}/**/*', 'Rakefile', 'README*', 'LICENSE*']
+	gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+	gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+	gem.require_paths = ["lib"]
+
+	gem.add_development_dependency "rake"
+	gem.add_runtime_dependency "aws-sdk", "~> 1.11.2"
+	gem.add_runtime_dependency "timedcache", "~> 0.4.0"
+	gem.add_runtime_dependency "json", "~> 1.8.0"
+end
+
+Gem::PackageTask.new(spec) do |pkg|
+	pkg.need_tar = true
+end

data/lib/hiera/backend/cloudformation_backend.rb

@@ -0,0 +1,136 @@
+=begin
+Copyright 2013 FanDuel Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=end
+
+class Hiera
+	module Backend
+		class Cloudformation_backend
+			TIMEOUT = 60  # 1 minute timeout for AWS API response caching
+
+			def initialize
+				begin
+					require 'aws'
+					require 'timedcache'
+					require 'json'
+				rescue LoadError
+					require 'rubygems'
+					require 'aws'
+					require 'timedcache'
+					require 'json'
+				end
+
+				if Config.include?(:cloudformation) && !Config[:cloudformation].nil? &&
+					Config[:cloudformation].include?(:access_key_id) && Config[:cloudformation].include?(:secret_access_key)
+					Hiera.debug("Found AWS access key from configuration")
+					AWS.config({
+						:access_key_id => Config[:cloudformation][:access_key_id],
+						:secret_access_key => Config[:cloudformation][:secret_access_key]
+					})
+				else
+					Hiera.debug("No configuration found, will fall back to env variables or IAM role")
+				end
+
+				@cf = AWS::CloudFormation.new
+				@output_cache = TimedCache.new
+				@resource_cache = TimedCache.new
+
+				Hiera.debug("Hiera cloudformation backend loaded")
+			end
+
+			def lookup(key, scope, order_override, resolution_type)
+				answer = nil
+
+				Backend.datasources(scope, order_override) do |elem|
+					case elem
+					when /cfstack\/([^\/]+)\/outputs/
+						Hiera.debug("Looking up #{key} as an output of stack #{$1}")
+						raw_answer = stack_output_query($1, key)
+					when /cfstack\/([^\/]+)\/resources\/([^\/]+)/
+						Hiera.debug("Looking up #{key} in metadata of stack #{$1} resource #{$2}")
+						raw_answer = stack_resource_query($1, $2, key)
+					else
+						Hiera.debug("#{elem} doesn't seem to be a CloudFormation hierarchy element")
+						next
+					end
+
+					next if raw_answer.nil?
+
+					new_answer = Backend.parse_answer(raw_answer, scope)
+
+					case resolution_type
+					when :array
+						raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+						answer ||= []
+						answer << new_answer
+					when :hash
+						raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+						answer ||= {}
+						answer = Backend.merge_answer(new_answer, answer)
+					else
+						answer = new_answer
+						break
+					end
+				end
+
+				return answer
+			end
+
+			def stack_output_query(stack_name, key)
+				outputs = @output_cache.get(stack_name)
+
+				if outputs.nil? then
+					Hiera.debug("#{stack_name} outputs not cached, fetching...")
+					begin
+						outputs = @cf.stacks[stack_name].outputs
+					rescue AWS::CloudFormation::Errors::ValidationError
+						Hiera.debug("Stack #{stack_name} outputs can't be retrieved")
+						outputs = []  # this is just a non-nil value to serve as marker in cache
+					end
+					@output_cache.put(stack_name, outputs, TIMEOUT)
+				end
+
+				output = outputs.select { |item| item.key == key }
+
+				return output.empty? ? nil : output.shift.value
+			end
+
+			def stack_resource_query(stack_name, resource_id, key)
+				metadata = @resource_cache.get({:stack => stack_name, :resource => resource_id})
+
+				if metadata.nil? then
+					Hiera.debug("#{stack_name} #{resource_id} metadata not cached, fetching")
+					begin
+						metadata = @cf.stacks[stack_name].resources[resource_id].metadata
+					rescue AWS::CloudFormation::Errors::ValidationError
+						# Stack or resource doesn't exist
+						Hiera.debug("Stack #{stack_name} resource #{resource_id} can't be retrieved")
+						metadata = "{}" # This is just a non-nil value to serve as marker in cache
+					end
+					@resource_cache.put({:stack => stack_name, :resource => resource_id}, metadata, TIMEOUT)
+				end
+
+				if metadata.respond_to?(:to_str) then
+					data = JSON.parse(metadata)
+
+					if data.include?('hiera') then
+						return data['hiera'][key] if data['hiera'].include?(key)
+					end
+				end
+
+				return nil
+			end
+		end
+	end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification 
+name: hiera-cloudformation
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Hugh Cole-Baker
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-07-19 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - &id005 
+      - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.11.2
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: timedcache
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.4.0
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.8.0
+  type: :runtime
+  version_requirements: *id004
+description: Queries CloudFormation stack outputs or resource metadata for Hiera data
+email: 
+- hugh@fanduel.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/hiera/backend/cloudformation_backend.rb
+- Rakefile
+- README.md
+- LICENSE.txt
+homepage: https://github.com/fanduel/hiera-cloudformation
+licenses: 
+- Apache License (2.0)
+metadata: {}
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id005
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id005
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: CloudFormation backend for Hiera
+test_files: []
+