sf-hiera-aws 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5158fadbf94eccfdfafaebf7b175d3c11f36a841
-  data.tar.gz: d69011bbc8cd91207d999bbcb1889b88171a29cf
+  metadata.gz: 743247150dec1c5548e31200315a1fdd765afd22
+  data.tar.gz: ad06bef85101602ce5b65565289a60a5b13d2920
 SHA512:
-  metadata.gz: bca35ff02edb74348a7e3d5a85e3160f6841b553ae8a09fbf350b741344f052eb7f92c21621069bbf2e58f4d2d9b3d5365b75b99d2d925bc6fe8c47364bdf378
-  data.tar.gz: e6dd2449724903a05aa7b013868cd4237e02d6dc816ae126ce221a4a8d4af04a264f03fb6a2eaf24932685fa7c070a7c06574de1c873b6d100a37959294cba6a
+  metadata.gz: 5ab644f595db9b4752aac825b44286dd181282ba06d24ae1e4228a7b539c9a93c7aa9afc4c208b77a2c923f7d2412fea1728261f5124a253dfeb86c3556dc2d5
+  data.tar.gz: 1e701867da2da23d371b5d43f2b8c2e08f3663a06c9afcd5f71fbcae3138c59c912a6779f6d6086894ed375b4fe363af0ac94faccefdbdc28443f37b44fff673

data/.gitignore

@@ -7,3 +7,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+.ruby-version
+*.swp

data/.rubocop.yml

@@ -0,0 +1,5 @@
+Style/IndentationWidth:
+    Width: 4
+
+Style/TrailingComma:
+    Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+## 0.0.3 (16 December 2015)
+
+ * Add rspec tests
+ * Support configuration under a directory
+ * Configurable changes to return values
+
+## 0.0.2 (3 September 2015)
+
+ * Calculate AWS region from link-local data source
+
+## 0.0.1 (2 September 2015)
+
+ * Initial release
+

data/Gemfile

@@ -2,3 +2,13 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in sf-hiera-aws.gemspec
 gemspec
+
+group :development do
+    gem 'rspec'
+    gem 'rspec-mocks'
+    gem 'webmock'
+    gem 'hiera'
+    gem 'mocha'
+    gem 'rubocop'
+    gem 'simplecov'
+end

data/README.md

@@ -42,6 +42,7 @@ The IAM role will need the following permissions:
 ## Configuration
 
 The plugin expects to find a configuration file under `/etc/puppet/sf_hiera_aws.yaml`, defining how we look up named keys.  The keys at the top level of this file determine the names of the hiera keys the plugin will provide; the configuration determines how these are looked up.
+Additional configuration can be given in files under `/etc/puppet/sf_hiera_aws.d`, which are evaluated in alphanumerical order. If a duplicate key is encountered in files evaluated later, this will override the earlier config.
 
 ### Example - EC2 nodes by tag
 
@@ -68,7 +69,9 @@ aws_am_bullseye_rds:
   db_instance_identifier: "%{::sf_location}-%{::sf_environment}-db"
 ```
 
-Calls to `:rds_db_instance` type keys return the instance identifier, endpoint address and endpoint port.
+Calls to `:rds_db_instance` type keys return the instance identifier, endpoint address and endpoint port in a hash.
+Pass a `return` key with value `:hostname` to have the hostname of the first matching instance returned.
+Pass a `return` key with value `:hostname_and_port` to have a `"<hostname>:<port>"` string of the first matching instance returned.
 
 ### Example - ElastiCache cluster by name
 
@@ -79,5 +82,6 @@ aws_am_bullseye_redis:
 ```
 
 Calls to `:elasticache_cache_cluster` type keys return a list of cache nodes, their IDs and endpoint address/ports.
-
+Pass a `return` key with value `:hostname` to have a list of hostnames of keys of all cache nodes matching the cache_cluster_id returned.
+Pass a `return` key with value `:hostname_and_port` to have a list of `"<hostname>:<port>"` strings returned.
 

data/Rakefile

@@ -1,2 +1,10 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rspec/core/rake_task'
 
+namespace :test do
+    RSpec::Core::RakeTask.new(:unit) do |t|
+        t.pattern = 'spec/unit/*_test.rb'
+        t.rspec_opts = '--color'
+    end
+end

data/lib/hiera/backend/sf_hiera_aws_backend.rb

@@ -1,14 +1,11 @@
 class Hiera
-
     module Backend
-
         class Sf_hiera_aws_backend
+            private
 
-            public
-
-            def initialize
+            @instance_identity = nil
 
-                require 'aws-sdk-resources'
+            def read_link_local_data
                 require 'net/http'
                 require 'json'
 
@@ -16,88 +13,140 @@ class Hiera
                     http = Net::HTTP.new('169.254.169.254', 80)
                     http.open_timeout = 1
                     http.read_timeout = 1
-                    instance_identity = JSON.parse(http.request(Net::HTTP::Get.new("/latest/dynamic/instance-identity/document")).body)
-                    Aws.config.update({ region: instance_identity['region'] })
-                rescue Exception
+                    @instance_identity = JSON.parse(http.request(Net::HTTP::Get.new('/latest/dynamic/instance-identity/document')).body)
+                rescue Errno::EHOSTUNREACH, Net::OpenTimeout, Timeout::Error
                     Hiera.warn('No link-local endpoint - can\'t calculate region')
                 end
-                Hiera.debug('Hiera AWS SDK backend started')
+            end
+
+            # Wrapped these in getters for dependency injection purposes
+            def get_rds_client
+                Aws::RDS::Client.new
+            end
 
+            def get_ec2_client
+                Aws::EC2::Client.new
             end
 
-            def lookup (key, scope, order_override, resolution_type)
+            def get_elasticache_client
+                Aws::ElastiCache::Client.new
+            end
 
+            public
+
+            def initialize
+                require 'aws-sdk-resources'
+
+                read_link_local_data
+                unless @instance_identity.nil?
+                    Aws.config.update(region: @instance_identity['region'])
+                end
+
+                Hiera.debug('Hiera AWS SDK backend started')
+            end
+
+            def lookup(key, scope, _order_override, _resolution_type)
                 config = recursive_interpolate_config(aws_config, scope)
 
                 Hiera.debug("Looking up '#{key} in AWS SDK backend")
 
-                if ! config.key? key
-                    return nil
-                end
+                return nil unless config.key? key
 
                 Hiera.debug("Config: #{config[key].inspect}")
                 type = config[key]['type']
 
-                if self.methods.include? "type_#{type}".to_sym
+                if methods.include? "type_#{type}".to_sym
 
                     begin
-                        answer = self.send("type_#{type}".to_sym, config[key])
-                        Hiera.debug( answer )
+                        answer = send("type_#{type}".to_sym, config[key])
+                        Hiera.debug(answer)
                         return answer
                     rescue Aws::Errors::MissingRegionError, Aws::Errors::MissingCredentialsError
-                        Hiera.warn("No IAM role or ENV based AWS config - skipping")
+                        Hiera.warn('No IAM role or ENV based AWS config - skipping')
                         return nil
                     end
 
                 end
 
                 Hiera.debug("Type of AWS SDK lookup '#{type}' invalid")
-                return nil
-
+                nil
             end
 
-            def aws_config
-
-                require 'yaml'
+            def config_file_name
 
-                default_config_path = "/etc/puppet/sf_hiera_aws.yaml"
+                default_config_path = '/etc/puppet/sf_hiera_aws.yaml'
 
-                if ! Config[:aws_sdk].nil?
+                if !Config[:aws_sdk].nil?
                     config_file = Config[:aws_sdk][:config_file] || default_config_path
                 else
                     config_file = default_config_path
                 end
 
+                return config_file
+
+            end
+
+            def config_directory_name
+
+                default_config_path = '/etc/puppet/sf_hiera_aws.d'
+
+                if !Config[:aws_sdk].nil?
+                    config_dir = Config[:aws_sdk][:config_directory] || default_config_path
+                else
+                    config_dir = default_config_path
+                end
+
+                return config_dir
+
+            end
+
+
+            def aws_config
+
+                require 'yaml'
+
+                config_file = config_file_name
+
                 if File.exist?(config_file)
-                    config = YAML::load_file(config_file)
+                    config = YAML.load_file(config_file)
                 else
                     Hiera.warn("No config file #{config_file} found")
                     config = {}
                 end
 
-                config
+                # Merge configs from the config directory too
 
+                config_directory = config_directory_name
+
+                if File.directory?(config_directory)
+                    Dir.entries(config_directory).sort.each do |p|
+                        next if p == '.' or p == '..'
+                        to_merge = YAML.load_file( File.join( config_directory, p ) )
+                        config.merge! to_merge
+                    end
+                end
+
+                config
             end
 
-            def recursive_interpolate_config(h,scope)
+            def recursive_interpolate_config(h, scope)
                 case h
                 when Hash
                     Hash[
                     h.map do |k, v|
-                        [ Backend.parse_answer(k, scope), recursive_interpolate_config(v,scope) ]
+                        [Backend.parse_answer(k, scope), recursive_interpolate_config(v, scope)]
                     end
                     ]
                 when Enumerable
-                    h.map { |v| recursive_interpolate_config(v,scope) }
+                    h.map { |v| recursive_interpolate_config(v, scope) }
                 when String
-                    Backend.parse_answer(h,scope)
+                    Backend.parse_answer(h, scope)
                 else
                     h
                 end
             end
 
             def type_ec2_instance(options)
-
                 options = {
                     'return'  => [
                         :instance_id,
@@ -106,25 +155,23 @@ class Hiera
                     ]
                 }.merge(options)
 
-                ec2 = Aws::EC2::Resource.new()
+                ec2 = get_ec2_client
 
                 if options.key? 'filters'
-                    instances = ec2.instances( filters: options['filters'] ) || []
+                    instances = ec2.describe_instances(filters: options['filters']).reservations.first.instances || []
                 else
-                    instances = ec2.instances() || []
+                    instances = ec2.describe_instances().reservations.first.instances || []
                 end
 
                 instances.collect do |i|
-                    Hash[ options['return'].map { |f|
-                        [f.to_s, i.methods.include?(f) ? i.send(f) : nil ]
-                    } ]
+                    Hash[options['return'].map do |f|
+                        [f.to_s, i.key?(f) ? i[f] : nil]
+                    end]
                 end
-
             end
 
             def type_rds_db_instance(options)
-
-                rds = Aws::RDS::Client.new()
+                rds = get_rds_client
 
                 if options.key? 'db_instance_identifier'
                     instances = rds.describe_db_instances(
@@ -134,19 +181,42 @@ class Hiera
                     instances = rds.describe_db_instances.db_instances
                 end
 
-                instances.collect do |i|
-                    {
-                        'db_instance_identifier' => i.db_instance_identifier,
-                        'endpoint_address'       => i.endpoint.address,
-                        'endpoint_port'          => i.endpoint.port,
-                    }
+                if !options.key? 'return'
+
+                    return instances.collect do |i|
+                        {
+                            'db_instance_identifier' => i.db_instance_identifier,
+                            'endpoint_address'       => i.endpoint.address,
+                            'endpoint_port'          => i.endpoint.port,
+                        }
+                    end
+
                 end
 
+                if options['return'] == :hostname
+
+                    return instances.first.endpoint.address
+
+                elsif options['return'] == :hostname_and_port
+
+                    return [
+                        instances.first.endpoint.address,
+                        instances.first.endpoint.port,
+                    ].join(':')
+
+                else
+
+                    Hiera.warn("No return handler for #{options['return']} in rds_db_instance")
+                    return nil
+
+                end
+
+
+
             end
 
             def type_elasticache_cache_cluster(options)
-
-                elasticache = Aws::ElastiCache::Client.new()
+                elasticache = get_elasticache_client
 
                 if options.key? 'cache_cluster_id'
                     clusters = elasticache.describe_cache_clusters(
@@ -159,23 +229,50 @@ class Hiera
                     ).cache_clusters
                 end
 
-                clusters.collect do |i|
-                    {
-                        'cache_cluster_id' => i.cache_cluster_id,
-                        'cache_nodes'      => i.cache_nodes.collect do |n|
-                            {
-                                'cache_node_id'    => n.cache_node_id,
-                                'endpoint_address' => n.endpoint.address,
-                                'endpoint_port'    => n.endpoint.port,
-                            }
+                if !options.key? 'return'
+
+                    return clusters.collect do |i|
+                        {
+                            'cache_cluster_id' => i.cache_cluster_id,
+                            'cache_nodes'      => i.cache_nodes.collect do |n|
+                                {
+                                    'cache_node_id'    => n.cache_node_id,
+                                    'endpoint_address' => n.endpoint.address,
+                                    'endpoint_port'    => n.endpoint.port,
+                                }
+                            end
+                        }
+                    end
+
+                end
+
+                if options['return'] == :hostname
+
+                    nodes = []
+
+                    clusters.each do |c|
+                        c.cache_nodes.each do |n|
+                            nodes.push(n.endpoint.address)
+                        end
+                    end
+
+                    return nodes
+
+                elsif options['return'] == :hostname_and_port
+
+                    nodes = []
+
+                    clusters.each do |c|
+                        c.cache_nodes.each do |n|
+                            nodes.push( [ n.endpoint.address, n.endpoint.port ].join(':') )
                         end
-                    }
+                    end
+
+                    return nodes
+
                 end
-            end
 
+            end
         end
-
     end
-
 end
-

data/sf-hiera-aws.gemspec

@@ -3,21 +3,23 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "sf-hiera-aws"
-  spec.version       = "0.0.2"
-  spec.authors       = ["Jon Topper"]
-  spec.email         = ["jon@scalefactory.com"]
+    spec.name          = 'sf-hiera-aws'
+    spec.version       = '0.0.3'
+    spec.authors       = ['Jon Topper']
+    spec.email         = ['jon@scalefactory.com']
 
-  spec.summary       = %q{Hiera backend for querying AWS resources}
-  spec.homepage      = "https://github.com/scalefactory/sf-hiera-aws"
-  spec.license       = "MIT"
+    spec.summary       = 'Hiera backend for querying AWS resources'
+    spec.homepage      = 'https://github.com/scalefactory/sf-hiera-aws'
+    spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+    spec.files         = `git ls-files -z`.split("\x0").reject { |f|
+                             f.match(%r{^(test|spec|features)/})
+                         }
+    spec.bindir        = 'exe'
+    spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+    spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.8"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_dependency "aws-sdk-resources", ">=2.1.18"
+    spec.add_development_dependency 'bundler', '~> 1.8'
+    spec.add_development_dependency 'rake', '~> 10.0'
+    spec.add_dependency 'aws-sdk-resources', '>=2.2.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sf-hiera-aws
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Jon Topper
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-03 00:00:00.000000000 Z
+date: 2015-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.18
+        version: 2.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.18
+        version: 2.2.6
 description: 
 email:
 - jon@scalefactory.com
@@ -60,6 +60,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md