hiera-aws 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db0800685a66b940ed05671f7b7e6ef30c44d561
-  data.tar.gz: 39bd81a8d5ec926b3388f4f115a13e18c7f499d9
+  metadata.gz: 56537b23031d0700addcb408d1d47fd3a1eecd2e
+  data.tar.gz: e672330d9bf693caa50207dab7e0ce371a8d70db
 SHA512:
-  metadata.gz: 2c15a81ed1e10cc95e8b10738707480bcf8814b9760cf3235bb35b5365cdef783742338f2b833ac5fd4f898e51ae9af9176fb65c756e4c66abb59302efe7972e
-  data.tar.gz: 567661b4d8c7973c9d9e3b995bb15d9f3b2f25c94cd26a652090182a580388469dd6bf77f3cd5cb2aacbacda07349957bf1cf22a94d32cca7f72ac52f42e519e
+  metadata.gz: 1121e5a75df9263b4654cd2ba82da332d078550bfb538b7819e588d5615cb19f85e9043bf7a84b2450d922000bfb1e4d958602e0dd1c94924ab74ad6b422b254
+  data.tar.gz: 4b6b9056a6173ab307fd55470a92793bd501b5f9c6b0ad70e87230093432fec1ec2c65cfcbd01ad386455e855bdca1d62c5d108ce0d73d9bd240f38f79d8e1ee

data/.rubocop.yml

@@ -0,0 +1,17 @@
+AllCops:
+  Includes:
+    - Gemfile
+    - Rakefile
+
+Encoding:
+  Enabled: false
+HashSyntax:
+  Enabled: false
+LineLength:
+  Enabled: false
+MethodLength:
+  Max: 30
+SignalException:
+  Enabled: false
+StringLiterals:
+  Enabled: false

data/.travis.yml

@@ -1,12 +1,10 @@
 language: ruby
 
-rvm:
-  - 1.8.7
-  - 1.9.3
+rvm: 1.9.3
 
 install: bundle install
 
-script: bundle exec rake spec
+script: bundle exec rake test
 
 branches:
   only:

data/README.md

@@ -25,18 +25,25 @@ First, add the backend to the list of backends in `hiera.yaml`:
   - aws
 ```
 
-Next, add `aws/elasticache` to the hierarchy. ElastiCache is the only AWS
-service currently supported by this backend.
+Next, add the AWS services supported by this backend to the hierarchy:
 
 ```yaml
 :hierarchy:
   - aws/elasticache
+  - aws/rds
 ```
 
-To grant sufficient privileges for Hiera to work, you either have to assign the
-EC2 instances an IAM role that allows the action `elasticache:Describe*`
-(preferred) or provide credentials for a user with the same privileges via the
-backend configuration in `hiera.yml`:
+The following AWS privileges are required for Hiera to work:
+
+- `AmazonEC2ReadOnlyAccess`
+- `AmazonElastiCacheReadOnlyAccess`
+- `AmazonRDSReadOnlyAccess`
+- `AWSCloudFormationReadOnlyAccess`
+- `IAMReadOnlyAccess`
+
+To grant those privileges, you either have to assign the EC2 instances an IAM
+role (preferred) or provide credentials for a user with the same privileges via
+the backend configuration in `hiera.yml`:
 
 ```yaml
 :aws:
@@ -44,7 +51,6 @@ backend configuration in `hiera.yml`:
   :secret_access_key: your_aws_secret_access_key_here
 ```
 
-
 ## Hiera Keys
 
 The backend currently supports the following keys that you can pass to the
@@ -89,6 +95,24 @@ Usage:
 cluster_nodes = hiera("memcached_cluster_nodes_for_cfn_stack")
 ```
 
+### rds tag=value...
+
+Returns an array of all RDS database instances that have one or more tags. The
+returned array has the format `["host1", "host2"]`.
+
+Usage:
+
+```
+# Get all database instances
+rds_instances = hiera("rds")
+
+# Get all database instances that have a tag named "environment" with the value "dev"
+rds_instances = hiera("rds environment=dev")
+
+# Get all database instances that have two specific tags
+rds_instances = hiera("rds environment=production role=mgmt-db")
+```
+
 ## License and Authors
 
 * Author:: Mathias Lafeldt (mathias.lafeldt@jimdo.com)

data/Rakefile

@@ -1,8 +1,13 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "rubocop/rake_task"
+
+desc "Run RuboCop style and lint checks"
+Rubocop::RakeTask.new(:rubocop)
 
 RSpec::Core::RakeTask.new(:spec) do |t|
   t.rspec_opts = "--color --format documentation"
 end
 
-task :default => :spec
+task :test => [:rubocop, :spec]
+task :default => :test

data/hiera-aws.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
 end

data/lib/hiera/backend/aws/base.rb

@@ -5,16 +5,26 @@ class Hiera
     module Aws
       class MissingFactError < StandardError; end
 
+      # Base class for all AWS service classes
       class Base
-        def initialize(scope={})
+        def initialize(scope = {})
           @scope = scope
         end
 
+        def aws_region
+          @scope["location"] || "eu-west-1"
+        end
+
+        def aws_account_number
+          @scope["aws_account_number"] ||
+            AWS::IAM.new.users.first.arn.split(":")[4]
+        end
+
         attr_reader :scope
 
         def lookup(key, scope)
+          @scope = scope
           if respond_to? key
-            @scope = scope
             send(key)
           else
             # Found no handler for key

data/lib/hiera/backend/aws/elasticache.rb

@@ -3,11 +3,10 @@ require "hiera/backend/aws/base"
 class Hiera
   module Backend
     module Aws
+      # Implementation of Hiera keys for aws/elasticache
       class ElastiCache < Base
         def cache_nodes_by_cache_cluster_id
-          region = scope["location"] || "eu-west-1"
-          client = AWS::ElastiCache::Client.new :region => region
-
+          client = AWS::ElastiCache::Client.new :region => aws_region
           cache_cluster_id = scope["cache_cluster_id"]
           raise MissingFactError, "cache_cluster_id not found" unless cache_cluster_id
           options = { :cache_cluster_id => cache_cluster_id, :show_cache_node_info => true }
@@ -19,25 +18,21 @@ class Hiera
         # XXX: Lots of spiked code ahead that MUST be refactored.
         #
         def cfn_stack_name(instance_id)
-          region = scope["location"] || "eu-west-1"
-          client = AWS::EC2.new :region => region
-
+          client = AWS::EC2.new :region => aws_region
           instances = client.instances[instance_id]
           instances.tags["aws:cloudformation:stack-name"]
         end
 
         def cache_cluster_info(cluster_id)
-          region = scope["location"] || "eu-west-1"
-          client = AWS::ElastiCache::Client.new :region => region
-
+          client = AWS::ElastiCache::Client.new :region => aws_region
           options = { :cache_cluster_id => cluster_id, :show_cache_node_info => true }
           info = client.describe_cache_clusters(options)
           info.fetch(:cache_clusters).first
         end
 
-        def cache_clusters_in_cfn_stack(stack_name, cluster_engine=nil)
-          region = scope["location"] || "eu-west-1"
-          client = AWS::CloudFormation.new :region => region
+        # rubocop:disable MultilineBlockChain
+        def cache_clusters_in_cfn_stack(stack_name, cluster_engine = nil)
+          client = AWS::CloudFormation.new :region => aws_region
 
           stack = client.stacks[stack_name]
           stack.resources.select do |r|
@@ -54,8 +49,9 @@ class Hiera
             end
           end
         end
+        # rubocop:enable MultilineBlockChain
 
-        def cluster_nodes_for_cfn_stack(cluster_engine=nil)
+        def cluster_nodes_for_cfn_stack(cluster_engine = nil)
           ec2_instance_id = scope["ec2_instance_id"]
           raise MissingFactError, "ec2_instance_id not found" unless ec2_instance_id
 

data/lib/hiera/backend/aws/rds.rb

@@ -0,0 +1,57 @@
+require "hiera/backend/aws/base"
+
+class Hiera
+  module Backend
+    module Aws
+      # Implementation of Hiera keys for aws/rds
+      class RDS < Base
+        def initialize(scope = {})
+          super(scope)
+          @client = AWS::RDS::Client.new :region => aws_region
+        end
+
+        # Override default key lookup to implement custom format. Examples:
+        #  - hiera("rds")
+        #  - hiera("rds environment=dev")
+        #  - hiera("rds role=mgmt-db")
+        #  - hiera("rds environment=production role=mgmt-db")
+        def lookup(key, scope)
+          r = super(key, scope)
+          return r if r
+
+          args = key.split
+          if args.shift == "rds"
+            if args.length > 0
+              tags = Hash[args.map { |t| t.split("=") }]
+              db_instances_with_tags(tags).map { |i| i[:endpoint][:address] }
+            else
+              db_instances.map { |i| i[:endpoint][:address] }
+            end
+          end
+        end
+
+        private
+
+        def db_instances
+          @db_instances ||= @client.describe_db_instances[:db_instances]
+        end
+
+        def db_instances_with_tags(tags)
+          db_instances.select do |i|
+            all_tags = db_instance_tags(i[:db_instance_identifier])
+            tags.all? { |k, v| tags[k] == all_tags[k] }
+          end
+        end
+
+        def db_resource_name(db_instance_id)
+          "arn:aws:rds:#{aws_region}:#{aws_account_number}:db:#{db_instance_id}"
+        end
+
+        def db_instance_tags(db_instance_id)
+          tags = @client.list_tags_for_resource(:resource_name => db_resource_name(db_instance_id))
+          Hash[tags[:tag_list].map { |t| [t[:key], t[:value]] }]
+        end
+      end
+    end
+  end
+end

data/lib/hiera/backend/aws/version.rb

@@ -1,7 +1,7 @@
 class Hiera
   module Backend
-    module Aws
-      VERSION = "0.0.5"
+    module Aws # rubocop:disable Documentation
+      VERSION = "0.0.6"
     end
   end
 end

data/lib/hiera/backend/aws_backend.rb

@@ -1,8 +1,10 @@
 require "hiera/backend/aws/elasticache"
+require "hiera/backend/aws/rds"
 
 class Hiera
   module Backend
-    class Aws_backend
+    # Hiera AWS backend
+    class Aws_backend # rubocop:disable ClassAndModuleCamelCase
       def initialize
         begin
           require "aws-sdk"
@@ -20,16 +22,16 @@ class Hiera
         if Config[:aws] && Config[:aws][:access_key_id] && Config[:aws][:secret_access_key]
           Hiera.debug("Using AWS credentials from backend configuration")
 
-          AWS.config({
+          AWS.config(
             :access_key_id     => Config[:aws][:access_key_id],
             :secret_access_key => Config[:aws][:secret_access_key]
-          })
+          )
         else
           Hiera.debug("Using AWS credentials from environment or IAM role")
         end
       end
 
-      def lookup(key, scope, order_override, resolution_type)
+      def lookup(key, scope, order_override, resolution_type) # rubocop:disable CyclomaticComplexity
         Backend.datasources(scope, order_override) do |elem|
           elements = elem.split "/"
           next unless elements[0] == "aws"
@@ -39,6 +41,8 @@ class Hiera
           service_class = case service
                           when "elasticache"
                             Hiera::Backend::Aws::ElastiCache.new
+                          when "rds"
+                            Hiera::Backend::Aws::RDS.new
                           end
           next if service_class.nil?
 
@@ -46,7 +50,7 @@ class Hiera
           next if value.nil?
 
           # this only supports resolution_type => :priority at the moment.
-          # TODO implement :array and :hash merging
+          # TODO: implement :array and :hash merging
           value = Backend.parse_answer(value, scope)
           return value unless value.nil?
         end

data/spec/aws_backend_spec.rb

@@ -1,7 +1,7 @@
 require "hiera/backend/aws_backend"
 
 class Hiera
-  module Backend
+  module Backend # rubocop:disable Documentation
     describe Aws_backend do
       before do
         Hiera.stub(:debug)
@@ -51,6 +51,12 @@ class Hiera
           expect_any_instance_of(Aws::ElastiCache).to receive(:lookup).with(key, scope)
           backend.lookup(*params)
         end
+
+        it "properly forwards lookup to RDS service" do
+          Backend.stub(:datasources).and_yield "aws/rds"
+          expect_any_instance_of(Aws::RDS).to receive(:lookup).with(key, scope)
+          backend.lookup(*params)
+        end
       end
     end
   end

data/spec/aws_base_spec.rb

@@ -1,11 +1,11 @@
 require "hiera/backend/aws/base"
 
 class Hiera
-  module Backend
+  module Backend # rubocop:disable Documentation
     describe Aws::Base do
-      let(:service) { Aws::Base.new  }
-
       describe "#lookup" do
+        let(:service) { Aws::Base.new  }
+
         it "returns nil if key is unknown since Hiera iterates over all configured backends" do
           value = service.lookup("key_with_no_matching_method", {})
           expect(value).to be_nil
@@ -23,6 +23,35 @@ class Hiera
           expect(scope).to eq scope
         end
       end
+
+      describe "#aws_region" do
+        it "defaults to eu-west-1" do
+          service = Aws::Base.new
+          expect(service.aws_region).to eq "eu-west-1"
+        end
+
+        it "can be set via Puppet fact" do
+          scope = { "location" => "some-aws-region" }
+          service = Aws::Base.new scope
+          expect(service.aws_region).to eq "some-aws-region"
+        end
+      end
+
+      describe "#aws_account_number" do
+        it "can be set via Puppet fact" do
+          scope = { "aws_account_number" => "12345678" }
+          service = Aws::Base.new scope
+          expect(service.aws_account_number).to eq "12345678"
+        end
+
+        it "is retrieved from AWS when Puppet fact is not set" do
+          AWS::IAM.any_instance.stub(
+            :users => [double(:arn => "arn:aws:iam::12345678:user/some-user")]
+          )
+          service = Aws::Base.new
+          expect(service.aws_account_number).to eq "12345678"
+        end
+      end
     end
   end
 end

data/spec/aws_elasticache_spec.rb

@@ -1,51 +1,59 @@
 require "hiera/backend/aws/elasticache"
 
 class Hiera
-  module Backend
+  module Backend # rubocop:disable Documentation
     describe Aws::ElastiCache do
-      let(:ec_redis_client) { double(
-        :describe_cache_clusters => {
-          :cache_clusters => [{
-            :cache_nodes => [
-              { :endpoint => { :address => "1.1.1.1", :port => 1234 } },
-              { :endpoint => { :address => "2.2.2.2", :port => 1234 } },
+      let(:ec_redis_client) do
+        double(
+          :describe_cache_clusters => {
+            :cache_clusters => [{
+              :cache_nodes => [
+                { :endpoint => { :address => "1.1.1.1", :port => 1234 } },
+                { :endpoint => { :address => "2.2.2.2", :port => 1234 } }
 
-            ],
-            :engine => "redis"
-          }]
-        }
-      )}
-      let(:ec_memcached_client) { double(
-        :describe_cache_clusters => {
-          :cache_clusters => [{
-            :cache_nodes => [
-              { :endpoint => { :address => "3.3.3.3", :port => 5678 } },
-              { :endpoint => { :address => "4.4.4.4", :port => 5678 } },
+              ],
+              :engine => "redis"
+            }]
+          }
+        )
+      end
+      let(:ec_memcached_client) do
+        double(
+          :describe_cache_clusters => {
+            :cache_clusters => [{
+              :cache_nodes => [
+                { :endpoint => { :address => "3.3.3.3", :port => 5678 } },
+                { :endpoint => { :address => "4.4.4.4", :port => 5678 } }
 
-            ],
-            :engine => "memcached"
-          }]
-        }
-      )}
-      let(:ec2_client) { double(
-        :instances => {
-          "some-ec2-instance-id" => double(
-            :tags => { "aws:cloudformation:stack-name" => "some-stack-name" }
-          )
-        }
-      )}
-      let(:cfn_client) { double(
-        :stacks => {
-          "some-stack-name" => double(
-            :resources => [
-              double(
-                :resource_type => "AWS::ElastiCache::CacheCluster",
-                :physical_resource_id => "some-cluster-id"
-              )
-            ]
-          )
-        }
-      )}
+              ],
+              :engine => "memcached"
+            }]
+          }
+        )
+      end
+      let(:ec2_client) do
+        double(
+          :instances => {
+            "some-ec2-instance-id" => double(
+              :tags => { "aws:cloudformation:stack-name" => "some-stack-name" }
+            )
+          }
+        )
+      end
+      let(:cfn_client) do
+        double(
+          :stacks => {
+            "some-stack-name" => double(
+              :resources => [
+                double(
+                  :resource_type => "AWS::ElastiCache::CacheCluster",
+                  :physical_resource_id => "some-cluster-id"
+                )
+              ]
+            )
+          }
+        )
+      end
 
       before do
         AWS::EC2.stub(:new => ec2_client)

data/spec/aws_rds_spec.rb

@@ -0,0 +1,83 @@
+require "hiera/backend/aws/rds"
+
+class Hiera
+  module Backend # rubocop:disable Documentation
+    describe Aws::RDS do
+      let(:rds) { Aws::RDS.new }
+      let(:rds_instances) do
+        {
+          :db_instances => [
+            {
+              :db_instance_identifier => "db1",
+              :endpoint => { :address => "db1.eu-west-1.rds.amazonaws.com" }
+            },
+            {
+              :db_instance_identifier => "db2",
+              :endpoint => { :address => "db2.eu-west-1.rds.amazonaws.com" }
+            },
+            {
+              :db_instance_identifier => "db3",
+              :endpoint => { :address => "db3.eu-west-1.rds.amazonaws.com" }
+            }
+          ]
+        }
+      end
+      let(:rds_tags) do
+        {
+          "arn:aws:rds:eu-west-1:12345678:db:db1" => {
+            :tag_list => [
+              { :key => "environment", :value => "dev" }
+            ]
+          },
+          "arn:aws:rds:eu-west-1:12345678:db:db2" => {
+            :tag_list => [
+              { :key => "environment", :value => "dev" },
+              { :key => "role", :value => "mgmt-db" }
+            ]
+          },
+          "arn:aws:rds:eu-west-1:12345678:db:db3" => {
+            :tag_list => [
+              { :key => "environment", :value => "production" },
+              { :key => "role", :value => "mgmt-db" }
+            ]
+          }
+        }
+      end
+
+      before do
+        rds_client = double
+        AWS::RDS::Client.stub(:new).and_return(rds_client)
+        allow(rds_client).to receive(:describe_db_instances).and_return(rds_instances)
+        allow(rds_client).to receive(:list_tags_for_resource) do |options|
+          rds_tags.fetch(options[:resource_name])
+        end
+      end
+
+      describe "#lookup" do
+        let(:scope) { { "aws_account_number" => "12345678" } }
+
+        it "returns nil if Hiera key is unknown" do
+          expect(rds.lookup("doge", scope)).to be_nil
+        end
+
+        it "returns all database instances if no tags are provided" do
+          expect(rds.lookup("rds", scope)).to eq ["db1.eu-west-1.rds.amazonaws.com",
+                                                  "db2.eu-west-1.rds.amazonaws.com",
+                                                  "db3.eu-west-1.rds.amazonaws.com"]
+        end
+
+        it "returns database instances with specific tags" do
+          expect(rds.lookup("rds role=mgmt-db", scope)).to eq ["db2.eu-west-1.rds.amazonaws.com",
+                                                               "db3.eu-west-1.rds.amazonaws.com"]
+          expect(rds.lookup("rds environment=dev", scope)).to eq ["db1.eu-west-1.rds.amazonaws.com",
+                                                                  "db2.eu-west-1.rds.amazonaws.com"]
+          expect(rds.lookup("rds environment=production role=mgmt-db", scope)).to eq ["db3.eu-west-1.rds.amazonaws.com"]
+        end
+
+        it "returns empty array if no database instances can be found" do
+          expect(rds.lookup("rds environment=staging", scope)).to eq []
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hiera-aws
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Mathias Lafeldt
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-12 00:00:00.000000000 Z
+date: 2014-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -67,6 +67,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Hiera AWS Backend
 email:
 - mathias.lafeldt@jimdo.com
@@ -76,6 +90,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rubocop.yml
 - .travis.yml
 - Gemfile
 - LICENSE
@@ -84,11 +99,13 @@ files:
 - hiera-aws.gemspec
 - lib/hiera/backend/aws/base.rb
 - lib/hiera/backend/aws/elasticache.rb
+- lib/hiera/backend/aws/rds.rb
 - lib/hiera/backend/aws/version.rb
 - lib/hiera/backend/aws_backend.rb
 - spec/aws_backend_spec.rb
 - spec/aws_base_spec.rb
 - spec/aws_elasticache_spec.rb
+- spec/aws_rds_spec.rb
 homepage: https://github.com/Jimdo/hiera-aws
 licenses:
 - Apache 2.0
@@ -117,3 +134,4 @@ test_files:
 - spec/aws_backend_spec.rb
 - spec/aws_base_spec.rb
 - spec/aws_elasticache_spec.rb
+- spec/aws_rds_spec.rb