settings_reader-vault_resolver 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1a6da72b08435eb9eae6bff284c09d0ab717de2e73e3bfea7e4bdc255b1bf2b
-  data.tar.gz: 2003918d4ad830d9dc4ab0127ec0e970eb2994e66e42ce8fb3f3936e1bb2e476
+  metadata.gz: 7a9aaa9ade4b4a390d408f4b29bed51bcf1c85c8345cb9434497bd565bd302a2
+  data.tar.gz: '0955dde971303ab6899e9beae5e9b276ef7209b9f9cf3e4769c875a2245619bf'
 SHA512:
-  metadata.gz: 4d4618260512414b329893c4651f1594398f34a5d54d94b0afe0989aa57f54208d85c16dae2f43fd0df3a0842f40c05e87202477a0c04dba9c6c0341f401f108
-  data.tar.gz: 6880bd9a5166bd7a01c2444fb4ce520a724c6df1ea6b4ec3ab1e844d4216602cd80b3d04b7d3b40dccbc3d9d3c7e808b54a818e2dadf78f9c9c400df4ea379dd
+  metadata.gz: 69a39ecbcf836c8116531c35b5ed9ea386595182aae4b7afb947d85709071a7dde75763759145e12c031eaff9ee8f85c13dcf55b1e08f2c763813bcb67dd9e2b
+  data.tar.gz: 6732daeeec4a32a63b218926d96974aec750c41abc3097f2ef9cfad58fa7c46626d2df434c147f28a7d7178682d2c3938324c53d5f0e30a5835a85f3fb3e7f8b

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 ## [Unreleased]
 
+## [0.4.2]
+### Fixes
+- Fix lost secret data after lease renewal
+- Fix exception when getting value from secret with nil data
+
+## [0.4.1]
+### Changes
+- Broader entry secret data access to allow retrieval of secret attributes
+
+### Fixes
+- Fix exception when retrieving authenticating via k8s endpoint
+
+### New features
+- Retry secret retrieval and renewal
+- Capture more vault exceptions including connectivity errors
+- Introduce vault engine adapter concept
+- Separate kv, database, and auth engine logic
+
+## [0.4.0]
+### Breaking changes
+- Reworked authentication helpers interface
+
+### New features
+- Retry secret retrieval and renewal
+- Capture more vault exceptions including connectivity errors
+- Introduce vault engine adapter concept
+- Separate kv, database, and auth engine logic 
+
 ## [0.3.0]
 ### Breaking changes
 - Require configuration before use
@@ -45,8 +73,11 @@
 - Secrets caching
 - Automatic secrets lease renewal
 
-[Unreleased]: https://github.com/matic-insurance/settings_reader-vault_resolver/compare/0.3.0...HEAD
-[0.2.4]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.3.0
+[Unreleased]: https://github.com/matic-insurance/settings_reader-vault_resolver/compare/0.4.2...HEAD
+[0.4.2]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.4.2
+[0.4.1]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.4.1
+[0.4.0]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.4.0
+[0.3.0]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.3.0
 [0.2.4]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.2.4
 [0.2.3]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.2.3
 [0.2.2]: https://github.com/matic-insurance/settings_reader-vault_resolver/commits/0.2.2

data/lib/settings_reader/vault_resolver/cache.rb

@@ -36,6 +36,10 @@ module SettingsReader
         @secrets.each_value(&block)
       end
 
+      def clear_all
+        @secrets = {}
+      end
+
       private
 
       def cache_key(address)

data/lib/settings_reader/vault_resolver/configuration.rb

@@ -6,6 +6,10 @@ module SettingsReader
       # Default: Logger.new(STDOUT, level: Logger::ERROR)
       attr_accessor :logger
 
+      # How many times to retry retrieval of the secret
+      # Default: 2
+      attr_accessor :retrieval_retries
+
       # How often do we check if secret lease is about to expire
       # Default: 60seconds
       attr_accessor :lease_refresh_interval
@@ -14,6 +18,10 @@ module SettingsReader
       # Default: 300seconds
       attr_accessor :lease_renew_delay
 
+      # How many times to retry renew of the secret
+      # Default: 4
+      attr_accessor :lease_renew_retries
+
       # Block to be executed when lease is refreshed
       # Default: empty proc
       attr_accessor :lease_renew_success_listener
@@ -24,8 +32,10 @@ module SettingsReader
 
       def initialize
         @logger = Logger.new($stdout, level: Logger::ERROR)
+        @retrieval_retries = 2
         @lease_refresh_interval = 60
         @lease_renew_delay = 300
+        @lease_renew_retries = 4
         @lease_renew_error_listener = proc {}
         @lease_renew_success_listener = proc {}
       end
@@ -40,6 +50,22 @@ module SettingsReader
         timer_task.execute
         timer_task
       end
+
+      def vault_engines
+        @vault_engines ||= [
+          SettingsReader::VaultResolver::Engines::KV2.new(self),
+          SettingsReader::VaultResolver::Engines::Database.new(self),
+          SettingsReader::VaultResolver::Engines::Auth.new(self)
+        ]
+      end
+
+      def vault_engine_for(address)
+        unless (engine = vault_engines.detect { |e| e.retrieves?(address) })
+          raise SettingsReader::VaultResolver::Error, "Unknown engine for #{address}"
+        end
+
+        engine
+      end
     end
   end
 end

data/lib/settings_reader/vault_resolver/engines/abstract.rb

@@ -0,0 +1,64 @@
+module SettingsReader
+  module VaultResolver
+    module Engines
+      # Abstract interface for Vault Backends
+      class Abstract
+        include Logging
+
+        attr_reader :config
+
+        def initialize(config)
+          @config = config
+        end
+
+        def retrieves?(_address)
+          raise NotImplementedError
+        end
+
+        def get(address)
+          return unless (vault_secret = get_secret_with_retries(address))
+
+          wrap_secret(address, vault_secret)
+        rescue Vault::VaultError => e
+          raise SettingsReader::VaultResolver::Error, e.message
+        end
+
+        def renew(entry)
+          return unless entry.leased?
+
+          new_secret = renew_lease_with_retries(entry)
+          entry.update_renewed(new_secret)
+          true
+        rescue Vault::VaultError => e
+          raise SettingsReader::VaultResolver::Error, e.message
+        end
+
+        protected
+
+        def get_secret_with_retries(address)
+          Vault.with_retries(Vault::HTTPConnectionError, attempts: config.retrieval_retries) do
+            get_secret(address)
+          end
+        end
+
+        def renew_lease_with_retries(address)
+          Vault.with_retries(Vault::HTTPConnectionError, attempts: config.lease_renew_retries) do
+            renew_lease(address)
+          end
+        end
+
+        def get_secret(address)
+          raise NotImplementedError
+        end
+
+        def renew_lease(entry)
+          raise NotImplementedError
+        end
+
+        def wrap_secret(address, secret)
+          SettingsReader::VaultResolver::Entry.new(address, secret)
+        end
+      end
+    end
+  end
+end

data/lib/settings_reader/vault_resolver/engines/auth.rb

@@ -0,0 +1,36 @@
+module SettingsReader
+  module VaultResolver
+    module Engines
+      # Adapter to retrieve / renew auth tokens
+      class Auth < Abstract
+        MOUNT = 'auth'.freeze
+        K8S_AUTH = 'kubernetes/login'.freeze
+
+        def retrieves?(address)
+          address.mount == MOUNT
+        end
+
+        protected
+
+        def get_secret(address)
+          return k8s_auth(address) if address.path == K8S_AUTH
+
+          raise SettingsReader::VaultResolver::Error, "Unsupported auth backed for #{address}"
+        end
+
+        def renew_lease(_entry)
+          secret = Vault.client.auth_token.renew_self
+          secret&.auth
+        end
+
+        private
+
+        def k8s_auth(address)
+          options = { route: address.options['route'], service_token_path: address.options['service_token_path'] }
+          secret = Vault.auth.kubernetes(address.options['role'], **options)
+          secret&.auth
+        end
+      end
+    end
+  end
+end

data/lib/settings_reader/vault_resolver/engines/database.rb

@@ -0,0 +1,29 @@
+module SettingsReader
+  module VaultResolver
+    module Engines
+      # Adapter to retrieve / renew secret from database engine
+      class Database < Abstract
+        MOUNT = 'database'.freeze
+
+        def retrieves?(address)
+          address.mount == MOUNT
+        end
+
+        private
+
+        def get_secret(address)
+          debug { "Fetching new database secret at: #{address}" }
+          Vault.logical.read(address.full_path)
+        rescue Vault::HTTPClientError => e
+          return nil if e.message.include?('* unknown role')
+
+          raise e
+        end
+
+        def renew_lease(entry)
+          Vault.sys.renew(entry.lease_id)
+        end
+      end
+    end
+  end
+end

data/lib/settings_reader/vault_resolver/engines/kv2.rb

@@ -0,0 +1,25 @@
+module SettingsReader
+  module VaultResolver
+    module Engines
+      # Adapter to retrieve / renew secret from kv2 engine
+      class KV2 < Abstract
+        MOUNT = 'secret'.freeze
+
+        def retrieves?(address)
+          address.mount == MOUNT
+        end
+
+        def renew(_entry)
+          # KV secrets are static. Nothing to do
+        end
+
+        private
+
+        def get_secret(address)
+          debug { "Fetching new kv secret at: #{address}" }
+          Vault.kv(address.mount).read(address.path)
+        end
+      end
+    end
+  end
+end

data/lib/settings_reader/vault_resolver/entry.rb

@@ -2,18 +2,19 @@ module SettingsReader
   module VaultResolver
     # Wrapper around vault secret object
     class Entry
-      attr_reader :address, :secret
+      attr_reader :address, :secret, :data
 
       MONTH = 30 * 60 * 60
 
       def initialize(address, secret)
         @address = address
         @secret = secret
+        @data = extract_data(secret)
         @lease_started = Time.now
       end
 
       def leased?
-        @secret.lease_id && lease_duration.positive?
+        @secret.renewable?
       end
 
       def expired?
@@ -28,29 +29,36 @@ module SettingsReader
         @lease_started + lease_duration - Time.now
       end
 
-      def renew
-        return unless leased?
+      def lease_id
+        @secret.lease_id
+      end
 
-        @secret = Vault.sys.renew(@secret.lease_id)
+      def update_renewed(new_secret)
+        @secret = new_secret
+        @data = @data.merge(extract_data(new_secret).compact)
         @lease_started = Time.now
-        true
-      rescue Vault::HTTPClientError => e
-        raise SettingsReader::VaultResolver::Error, e.message
       end
 
       def value_for(attribute)
-        secret.data[attribute.to_sym]
+        return data[attribute.to_sym] if data.key?(attribute.to_sym)
+        return secret.public_send(attribute) if secret.respond_to?(attribute)
+
+        nil
       end
 
       def to_s
         address.to_s
       end
 
-      private
-
       def lease_duration
         @secret.lease_duration.to_i
       end
+
+      private
+
+      def extract_data(secret)
+        secret.respond_to?(:data) && secret.data ? secret.data : {}
+      end
     end
   end
 end

data/lib/settings_reader/vault_resolver/helpers/vault_authentication.rb

@@ -0,0 +1,23 @@
+require_relative '../patches/authenticate'
+
+module SettingsReader
+  module VaultResolver
+    module Helpers
+      # Helps with authentication using different schemes
+      class VaultAuthentication
+        FAKE_RESOLVER_PATH = 'vault/authentication'.freeze
+
+        def authenticate_via_k8s(role, route: nil, service_token_path: nil)
+          params = URI.encode_www_form({ role: role, route: route, service_token_path: service_token_path }.compact)
+          resolver.resolve("vault://auth/kubernetes/login?#{params}#client_token", FAKE_RESOLVER_PATH)
+        end
+
+        private
+
+        def resolver
+          @resolver = SettingsReader::VaultResolver.resolver
+        end
+      end
+    end
+  end
+end

data/lib/settings_reader/vault_resolver/instance.rb

@@ -13,6 +13,7 @@ module SettingsReader
 
       def initialize(config)
         @config = config
+        @engines = config.vault_engines
       end
 
       def resolvable?(value, _path)
@@ -29,32 +30,12 @@ module SettingsReader
         entry&.value_for(address.attribute)
       end
 
-      # Resolve KV secret
-      def kv_secret(address)
-        debug { "Fetching new kv secret at: #{address}" }
-        Vault.kv(address.mount).read(address.path)
-      rescue Vault::HTTPClientError => e
-        raise SettingsReader::VaultResolver::Error, e.message
-      end
-
-      def database_secret(address)
-        debug { "Fetching new database secret at: #{address}" }
-        Vault.logical.read(address.full_path)
-      rescue Vault::HTTPClientError => e
-        return nil if e.message.include?('* unknown role')
-
-        raise SettingsReader::VaultResolver::Error, e.message
-      end
-
       private
 
       def fetch_entry(address)
         cache.fetch(address) do
           info { "Retrieving new secret at: #{address}" }
-          if (secret = address.mount == DATABASE_MOUNT ? database_secret(address) : kv_secret(address))
-            debug { "Retrieved secret at: #{address}" }
-            SettingsReader::VaultResolver::Entry.new(address, secret)
-          end
+          config.vault_engine_for(address).get(address)
         end
       rescue StandardError => e
         error { "Error retrieving secret: #{address}: #{e.message}" }

data/lib/settings_reader/vault_resolver/patches/authenticate.rb

@@ -1,7 +1,7 @@
 module Vault
   # Monkey patch to support k8s authenticaiton. Taken from https://github.com/hashicorp/vault-ruby/pull/202
   class Authenticate < Request
-    def kubernetes(role, route = nil, service_token_path = nil)
+    def kubernetes(role, route: nil, service_token_path: nil)
       route ||= '/v1/auth/kubernetes/login'
       service_token_path ||= '/var/run/secrets/kubernetes.io/serviceaccount/token'
 

data/lib/settings_reader/vault_resolver/refresher.rb

@@ -27,11 +27,11 @@ module SettingsReader
       end
 
       def refresh_entry(entry)
-        return unless entry.leased? && entry.expires_in < config.lease_renew_delay
+        return unless entry.expires_in < config.lease_renew_delay
 
         Concurrent::Promise.execute do
           debug { "Refreshing lease for #{entry}. Expires in: #{entry.expires_in}" }
-          entry.renew
+          config.vault_engine_for(entry.address).renew(entry)
           info { "Lease renewed for #{entry}. Expires in: #{entry.expires_in}" }
           entry
         rescue StandardError => e

data/lib/settings_reader/vault_resolver/version.rb

@@ -1,5 +1,5 @@
 module SettingsReader
   module VaultResolver
-    VERSION = '0.3.0'.freeze
+    VERSION = '0.4.2'.freeze
   end
 end

data/lib/settings_reader/vault_resolver.rb

@@ -2,15 +2,19 @@ require 'logger'
 require 'concurrent/timer_task'
 
 require 'settings_reader'
-require 'settings_reader/vault_resolver/version'
-require 'settings_reader/vault_resolver/logging'
-require 'settings_reader/vault_resolver/configuration'
-require 'settings_reader/vault_resolver/address'
-require 'settings_reader/vault_resolver/entry'
-require 'settings_reader/vault_resolver/cache'
-require 'settings_reader/vault_resolver/refresher'
-require 'settings_reader/vault_resolver/refresher_observer'
-require 'settings_reader/vault_resolver/instance'
+require_relative 'vault_resolver/version'
+require_relative 'vault_resolver/logging'
+require_relative 'vault_resolver/configuration'
+require_relative 'vault_resolver/address'
+require_relative 'vault_resolver/entry'
+require_relative 'vault_resolver/engines/abstract'
+require_relative 'vault_resolver/engines/auth'
+require_relative 'vault_resolver/engines/kv2'
+require_relative 'vault_resolver/engines/database'
+require_relative 'vault_resolver/cache'
+require_relative 'vault_resolver/refresher'
+require_relative 'vault_resolver/refresher_observer'
+require_relative 'vault_resolver/instance'
 
 module SettingsReader
   # Singleton for lease renewals and secrets cache

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: settings_reader-vault_resolver
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Volodymyr Mykhailyk
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-29 00:00:00.000000000 Z
+date: 2022-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -83,8 +83,12 @@ files:
 - lib/settings_reader/vault_resolver/address.rb
 - lib/settings_reader/vault_resolver/cache.rb
 - lib/settings_reader/vault_resolver/configuration.rb
+- lib/settings_reader/vault_resolver/engines/abstract.rb
+- lib/settings_reader/vault_resolver/engines/auth.rb
+- lib/settings_reader/vault_resolver/engines/database.rb
+- lib/settings_reader/vault_resolver/engines/kv2.rb
 - lib/settings_reader/vault_resolver/entry.rb
-- lib/settings_reader/vault_resolver/helpers/k8s_auth.rb
+- lib/settings_reader/vault_resolver/helpers/vault_authentication.rb
 - lib/settings_reader/vault_resolver/instance.rb
 - lib/settings_reader/vault_resolver/logging.rb
 - lib/settings_reader/vault_resolver/patches/authenticate.rb

data/lib/settings_reader/vault_resolver/helpers/k8s_auth.rb

@@ -1,43 +0,0 @@
-require_relative '../patches/authenticate'
-
-module SettingsReader
-  module VaultResolver
-    module Helpers
-      # Helps with Vault authentication using kubernetes login
-      class K8sAuth
-        def call(role)
-          secret = Vault.auth.kubernetes(role)
-
-          cache_token_for_renewal(secret)
-          secret
-        end
-
-        private
-
-        def cache_token_for_renewal(secret)
-          address = SettingsReader::VaultResolver::Address.new('vault://v1/auth/token/lookup-self')
-          entry = SettingsReader::VaultResolver::AuthEntry.new(address, secret)
-          SettingsReader::VaultResolver.cache.save(entry)
-          entry
-        end
-      end
-    end
-
-    # Helps with auth token lease renewal
-    class AuthEntry < SettingsReader::VaultResolver::Entry
-      def leased?
-        true
-      end
-
-      def lease_duration
-        @secret.auth.lease_duration
-      end
-
-      def renew
-        Vault.client.auth_token.renew_self
-        @lease_started = Time.now
-        true
-      end
-    end
-  end
-end