consul_application_settings 2.1.0 → 4.0.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afd23b9159a84c892c965172fd02452d93c77aeaa1391aa31e9690ed69dedd7d
-  data.tar.gz: ec62bb128a4aa6cc773f59010a56b59ec59961e76158bf45d2dbc2b606f98e81
+  metadata.gz: 709d93b5794a39ed2f78eb61462cc07ff0ed77dccd66343ad22fc13abc45c267
+  data.tar.gz: 6daa5f0e9266fad7baee4e44efe8ed49ac5451744bd3abb60667afb1afb1a132
 SHA512:
-  metadata.gz: 5762fe9054a80ea27e464f5778a98234540801165ff24244087e498bd7d8634a9bab3ecb69b3cfdaa8850196822ce9eb484e3eaa04d0929c5ba5810e1d619c8e
-  data.tar.gz: 97cb89da4c846ff2c2df46e71b493a28030f3266017ff37a657e7afb628555f3fbe0a5e74fa418a64f17c8a2a1e2ad9f62455cddc5ea725d4505a532e32992f7
+  metadata.gz: 399c74e1bed212dea80f9b5d3f0f202c702b2e7704824214703f8003841b35792b66e3c6707ec7594d4b6860acbdd1b341c65e85312448d9fef3e2aa4a1c6c16
+  data.tar.gz: 4b04c272e96bc975d0cae6984dc6c44e4d442b0dc1a78a51b9d6ca772b4d5b09bcc10ce6155113a955210e9abe4d91ccb446224fd881f0a79f29da55aebb6fc7

data/.github/workflows/main.yml

@@ -14,12 +14,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [ '2.5.x', '2.6.x' ]
+        ruby: [ '2.5.x', '2.6.x', '2.7.x', '3.0.x' ]
     services:
-      redis:
+      consul:
         image: consul:1.6
         ports:
           - '8500:8500'
+      vault:
+        image: vault
+        ports:
+          - "8200:8200"
+        env:
+          VAULT_DEV_ROOT_TOKEN_ID: root_token
+          SKIP_SETCAP: true
     steps:
       - name: Checkout
         uses: actions/checkout@v1
@@ -39,13 +46,15 @@ jobs:
       - name: Set up Dependencies
         run: bundle install --path vendor/bundle
 
-      - name: Run Tests
-        uses: paambaati/codeclimate-action@v2.3.0
+      - name: Run specs
         env:
           COVERAGE: true
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        with:
-          coverageCommand: ./bin/rspec
+        run: bundle exec rspec
+
+      - name: Upload coverage
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: bash <(curl -s https://codecov.io/bash)
 
   release:
     runs-on: ubuntu-latest

data/.rubocop.yml

@@ -2,6 +2,7 @@ AllCops:
   TargetRubyVersion: 2.5
   Exclude:
     - 'tmp/**/*'
+    - 'bin/**/*'
 
 Metrics/LineLength:
   Max: 120
@@ -10,4 +11,4 @@ Style/FrozenStringLiteralComment:
   Enabled: false
 
 Metrics/BlockLength:
-  ExcludedMethods: ['describe', 'context']
+  ExcludedMethods: ['describe', 'context', 'shared_examples']

data/.simplecov

@@ -1 +1,6 @@
-SimpleCov.start if ENV["COVERAGE"]
+if ENV["COVERAGE"]
+  require 'simplecov'
+  require 'codecov'
+  SimpleCov.start
+  SimpleCov.formatter = SimpleCov::Formatter::Codecov
+end

data/CHANGELOG.md

@@ -1,5 +1,51 @@
 ## [Unreleased]
 
+## [4.0.0-alpha]
+### Breaking Changes
+- When using `settings.get` to retrieve object - exception is raised 
+### New features
+- Introduce resolvers concept
+- Add environment resolver
+- Add erb resolver
+- Add vault alpha resolver
+
+## [3.0.1]
+### Fixes
+- Fix exception when preloading settings without consul 
+
+## [3.0.0]
+### Breaking Changes
+- Use Preloaded Consul Settings Provider by default
+### New features
+- Configurable setting providers
+- Preloaded Consul Settings Provider to prioritize performance over consistency
+- Performance tests in spec
+- Benchmarking script
+### Fixes
+- Return nil instead of empty hash when reading missing setting from file
+- Return nil instead of empty string when reading missing value from Consul
+- Add missing load method on Settings Reader to create object with narrow scope
+
+## [2.1.1]
+### Changes
+- Update Diplomat to latest version 
+
+## [2.1.0]
+### Fixes
+- Return nil for unknown keys 
+
+## [2.0.0]
+### Breaking Changes
+- Change default naming for setting files 
+
+## [1.0.0]
+### Features
+- Add support for second settings file (local settings for development) 
+
+## [0.1.4]
+### Fixes
+- Clone values before returning
+
 ## [0.1.3]
 ### Fixes
 - Add `Diplomat::PathNotFound` to the list of caught exceptions
@@ -22,7 +68,15 @@
 - Support deep settings search
 - Support nested configs
 
-[Unreleased]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.3...HEAD
+[Unreleased]: https://github.com/matic-insurance/consul_application_settings/compare/4.0.0-alpha...HEAD
+[4.0.0-alpha]: https://github.com/matic-insurance/consul_application_settings/compare/3.0.1...4.0.0-alpha
+[3.0.1]: https://github.com/matic-insurance/consul_application_settings/compare/3.0.0...3.0.1
+[3.0.0]: https://github.com/matic-insurance/consul_application_settings/compare/2.0.0...3.0.0
+[2.1.1]: https://github.com/matic-insurance/consul_application_settings/compare/2.1.0...2.1.1
+[2.1.0]: https://github.com/matic-insurance/consul_application_settings/compare/2.0.0...2.1.0
+[2.0.0]: https://github.com/matic-insurance/consul_application_settings/compare/1.0.0...2.0.0
+[1.0.0]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.4...1.0.0
+[0.1.4]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.3...0.1.4
 [0.1.3]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.2...0.1.3
 [0.1.2]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.1...0.1.2
 [0.1.1]: https://github.com/matic-insurance/consul_application_settings/compare/0.1.0...0.1.1

data/Gemfile.lock

@@ -2,22 +2,31 @@ PATH
   remote: .
   specs:
     consul_application_settings (0.0.0)
-      diplomat (~> 2.1.3)
+      deep_merge (~> 1.2)
+      diplomat (~> 2.5.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
+    aws-eventstream (1.2.0)
+    aws-sigv4 (1.4.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+    codecov (0.4.3)
+      simplecov (>= 0.15, < 0.22)
     deep_merge (1.2.1)
     diff-lcs (1.3)
-    diplomat (2.1.3)
-      deep_merge (~> 1.0, >= 1.0.1)
-      faraday (~> 0.9)
+    diplomat (2.5.1)
+      deep_merge (~> 1.2)
+      faraday (>= 0.9)
     docile (1.3.1)
-    faraday (0.17.3)
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
       multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    faraday-net_http (1.0.1)
     jaro_winkler (1.5.2)
-    json (2.2.0)
+    json (2.3.1)
     multipart-post (2.1.1)
     parallel (1.16.2)
     parser (2.6.2.0)
@@ -49,24 +58,29 @@ GEM
     rubocop-rspec (1.32.0)
       rubocop (>= 0.60.0)
     ruby-progressbar (1.10.0)
+    ruby2_keywords (0.0.4)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     unicode-display_width (1.5.0)
+    vault (0.16.0)
+      aws-sigv4
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 2.0)
+  codecov (~> 0.4)
   consul_application_settings!
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 0.66)
   rubocop-rspec (~> 1.32.0)
   simplecov (~> 0.16)
+  vault (~> 0.16)
 
 BUNDLED WITH
    2.1.2

data/README.md

@@ -1,29 +1,18 @@
 # ConsulApplicationSettings
 
 ![Build Status](https://github.com/matic-insurance/consul_application_settings/workflows/ci/badge.svg?branch=master)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/b0eaebcf83898535ea4e/test_coverage)](https://codeclimate.com/github/matic-insurance/consul_application_settings/test_coverage)
-[![Maintainability](https://api.codeclimate.com/v1/badges/b0eaebcf83898535ea4e/maintainability)](https://codeclimate.com/github/matic-insurance/consul_application_settings/maintainability)
+[![Test Coverage](https://codecov.io/gh/matic-insurance/consul_application_settings/branch/master/graph/badge.svg?token=5E8NA8EE8L)](https://codecov.io/gh/matic-insurance/consul_application_settings)
 
-Gem that simplifies usage of Consul (via [Diplomat gem](https://github.com/WeAreFarmGeek/diplomat)) 
-to host application settings. Gem provides defaults via yaml files and other utilities 
-to simplify storage and control of application with Consul KV storage.
+This gem that simplifies usage of Consul (via [Diplomat gem](https://github.com/WeAreFarmGeek/diplomat)) 
+to host application settings. 
 
-Gem is trying to solve a problem of distributing application settings for local development environment and provide defaults 
-in production before custom value is set inside of consul. 
+Except reading value from Consul the gem also:
+- Fallbacks to YAML if value is missing in consul
+- Resolve actual value from other sources to facilitate overriding via ENV, storing secret values in Vault,  
+  or executing small ERB snippets 
 
-Example use cases:
-
-- One engineer has created a new feature that depend on consul key/value. 
-  
-  How enginner can notify other engineers that they need to set this value in their consul environments?
-
-- DevOps team responsible to configure and maintain deployment. 
-
-  How do they learn (have reference) of what settings and structure application expect? 
-
-Gem reads any particular setting from consul and if it is missing tries to find value in YAML defaults file
-
-**NOTE** Consul is requested every time you query the settings. Defaults YAML file is loaded in memory and is not changing.
+Default values in YAML also can be considered as a way to communicate structure of settings to other engineers.
+Default values also support local settings to allow override on local environment or deployment in production.
 
 ## Installation
 
@@ -40,17 +29,27 @@ gem 'consul_application_settings'
 At the load of application: 
 ```ruby
 ConsulApplicationSettings.configure do |config|
-  # Specify path to the base settings YML. Default: 'config/application_settings.yml' 
-  config.base_file_path = Rails.root.join('config/my_settings.yml')
-  # Specify path to the local settings YML, which overrides the base file. Default: 'config/application_settings.local.yml'
-  config.local_file_path = Rails.root.join('config/my_settings.local.yml')
-  # Specify whether exceprion should be thrown on Consul connection errors. Default: false
+  # Specify path to the base settings YML. Default: 'config/app_settings.yml' 
+  config.base_file_path = Rails.root.join('config/app_settings.yml')
+  # Specify path to the local settings YML, which overrides the base file. Default: 'config/app_settings.local.yml'
+  config.local_file_path = Rails.root.join('config/app_settings.local.yml')
+  # Specify whether exception should be thrown on Consul connection errors. Default: false
   config.disable_consul_connection_errors = true
+  # Specify setting providers. Default: [ConsulApplicationSettings::Providers::ConsulPreloaded, ConsulApplicationSettings::Providers::LocalStorage]
+  config.settings_providers = [
+    ConsulApplicationSettings::Providers::Consul,          
+    ConsulApplicationSettings::Providers::LocalStorage          
+  ]
+  # Specify how values will be additionally resolved. Default: [ConsulApplicationSettings::Resolvers::Env] 
+  config.value_resolvers = [
+    ConsulApplicationSettings::Resolvers::Erb,
+    ConsulApplicationSettings::Resolvers::Env,
+  ]
 end
 
-APP_SETTINGS = ConsulApplicationSettings.load
 # Specify path to settings both in YML files and Consul
-AUTH_SETTIGNS = ConsulApplicationSettings.load('authentication')
+AUTH_SETTIGNS = ConsulApplicationSettings.load('my_cool_app')
+# Load at root without any prefix: APP_SETTINGS = ConsulApplicationSettings.load
 ```
 
 **NOTE** For rails you can add this code to custom initializer `console_application_settings.rb` in `app/config/initializers`
@@ -61,35 +60,32 @@ AUTH_SETTIGNS = ConsulApplicationSettings.load('authentication')
 
 Assuming your defaults file in repository `config/application_settings.yml` looks like:
 ```yaml
-staging:
-  my_cool_app:
-    app_name: 'MyCoolApp'
-    hostname: 'http://localhost:3001'
-    
-    integrations:
-      database:
-        domain: localhost
-        user: app
-        password: password1234
-      slack:
-        enabled: false
-        webhook_url: 'https://hooks.slack.com/services/XXXXXX/XXXXX/XXXXXXX'
+my_cool_app:
+  app_name: 'MyCoolApp'
+  hostname: 'http://localhost:3001'
+  
+  integrations:
+    database:
+      domain: localhost
+      user: app
+      password: password1234
+    slack:
+      enabled: false
+      webhook_url: 'https://hooks.slack.com/services/XXXXXX/XXXXX/XXXXXXX'
 ```
 
 And consul has following settings
 ```json
 {
-  "staging": {
-    "my_cool_app": {
-     "hostname": "https://mycoolapp.com",
-     "integrations": {
-        "database": {
-          "domain": "194.78.92.19",
-          "password": "*************"
-        },
-        "slack": {
-          "enabled": "true"
-        }
+  "my_cool_app": {
+   "hostname": "https://mycoolapp.com",
+   "integrations": {
+      "database": {
+        "domain": "194.78.92.19",
+        "password": "*************"
+      },
+      "slack": {
+        "enabled": "true"
       }
     }
   }
@@ -116,10 +112,60 @@ gem provides interface to avoid duplicating absolute path
 
 ```ruby
 # You can load subsettings from root object
-db_settings = APP_SETTINGS.load('integrations/database')
+db_settings = APP_SETTINGS.load('integrations/database') # ConsulApplicationSettings::Reader
 db_settings.get(:domain)                  # "194.78.92.19"
 db_settings['user']                       # "app"
-``` 
+
+#if you try to get subsettings via get - error is raised
+APP_SETTINGS.get('integrations/database') # raise ConsulApplicationSettings::Error
+```
+
+## Advanced Configurations
+
+### Setting Providers
+Providers controls how and in which order settings are retrieved. 
+When application asks for specific setting - gem retrieves them from every provider in order of configuration
+until one returns not nil value.
+
+Default order for providers is:
+1. `ConsulApplicationSettings::Providers::ConsulPreloaded`
+2. `ConsulApplicationSettings::Providers::LocalStorage`
+
+List of built in providers:
+- `ConsulApplicationSettings::Providers::ConsulPreloaded` - Retrieves all settings from consul on every `.load`
+- `ConsulApplicationSettings::Providers::Consul` - Retrieves setting every time `.get` method is called
+- `ConsulApplicationSettings::Providers::LocalStorage` - Retrieves all settings from local files on every `.load`
+
+Custom provider can be added as long as it support following interface:
+```ruby
+class CustomProvider
+  #constructor
+  def initialize(base_path, config)
+  end
+  
+  # get value by `base_path + '/' + path`
+  def get(path)
+  end
+end
+```
+
+### Resolvers
+Once value is retrieved - it will be additionally processed by resolvers. 
+This allows for additional flexibility like getting values from external sources. 
+While every resolver can be implemented in a form of a provider - one will be limited by the structure of settings, 
+while other system might not be compatible with this.
+
+When value is retrieved - gem finds **first** provider that can resolve value and resolves it. 
+Resolved value is returned to application.
+
+Default list of resolvers:
+- `ConsulApplicationSettings::Resolvers::Env`
+
+List of built in resolvers
+- `ConsulApplicationSettings::Resolvers::Env` - resolves any value by looking up environment variable. 
+  Matching any value that starts with `env://`. Value like `env://TEST_URL` will be resolved as `ENV['TEST_URL']`
+- `ConsulApplicationSettings::Resolvers::Erb` - resolves value by rendering it via ERB. 
+  Matching any value that contains `<%` and `%>` in it. Value like `<%= 2 + 2 %>` will be resolved as `4`
 
 ### Gem Configuration
 You can configure gem with block:
@@ -140,15 +186,18 @@ All Gem configurations
 | base_file_path                   | no       | 'config/application_settings.yml'       | String  | Path to the file with base settings                                          |
 | local_file_path                  | no       | 'config/application_settings.local.yml' | String  | Path to the file with local settings overriding the base settings            |
 | disable_consul_connection_errors | no       | true                                    | Boolean | Do not raise exception when consul is not available (useful for development) |
+| settings_providers               | no       | Array(Provider)                         | Array   | Specify custom setting provider lists                                        |
+| value_resolvers                  | no       | Array(Resolver)                         | Array   | Specify custom value resolvers lists                                         |
 
 ## Development
 
-1. [Install Consul](https://www.consul.io/docs/install/index.html)
 1. Run `bin/setup` to install dependencies
-1. Run tests `rspec`
-1. Add new test
-1. Add new code
-1. Go to step 3
+2. Run `docker-compose up` to spin up dependencies (Consul)
+3. Run tests `rspec`
+4. Add new test
+5. Add new code
+6. Go to step 3
+7. Create PR
 
 ## Contributing
 

data/bin/benchmark

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'consul_application_settings'
+require 'benchmark'
+
+ITERATIONS = 1000
+
+def kill_consul
+  `pgrep consul | xargs kill`
+end
+
+def start_consul
+  spawn('consul agent -dev -node machine > /dev/null 2>&1')
+end
+
+def application_settings(providers)
+  file = 'spec/fixtures/base_application_settings.yml'
+  ConsulApplicationSettings.configure do |config|
+    config.settings_providers = providers
+  end
+  ConsulApplicationSettings.load(file)
+end
+
+def benchmark_gem
+  Benchmark.bm(20) do |x|
+    x.report('Real Time Consul') do
+      ca = application_settings([
+                                  ConsulApplicationSettings::Providers::Consul,
+                                  ConsulApplicationSettings::Providers::LocalStorage
+                                ])
+      ITERATIONS.times { ca.get('application/name') }
+    end
+
+    x.report('Preloaded Consul') do
+      ca = application_settings([
+                                  ConsulApplicationSettings::Providers::ConsulPreloaded,
+                                  ConsulApplicationSettings::Providers::LocalStorage
+                                ])
+      ITERATIONS.times { ca.get('application/name') }
+    end
+
+    x.report('File Only') do
+      ca = application_settings([
+                                  ConsulApplicationSettings::Providers::ConsulPreloaded
+                                ])
+      ITERATIONS.times { ca.get('application/name') }
+    end
+  end
+end
+
+puts '-' * 80
+puts 'Benchmark without consul agent'
+kill_consul
+benchmark_gem
+
+puts '-' * 80
+puts 'Benchmark with consul agent running'
+start_consul
+benchmark_gem
+
+kill_consul

data/consul_application_settings.gemspec

@@ -29,12 +29,15 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'diplomat', '~> 2.1.3'
+  spec.add_dependency 'deep_merge', '~> 1.2'
+  spec.add_dependency 'diplomat', '~> 2.5.1'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'codecov', '~> 0.4'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 0.66'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.32.0'
   spec.add_development_dependency 'simplecov', '~> 0.16'
+  spec.add_development_dependency 'vault', '~> 0.16'
 end

data/docker-compose.yml

@@ -0,0 +1,17 @@
+# dependencies needed for development environment
+version: '3'
+services:
+  vault:
+    image: vault
+    ports:
+      - "8200:8200"
+    cap_add:
+      - IPC_LOCK
+    environment:
+      VAULT_DEV_ROOT_TOKEN_ID: root_token
+  consul:
+    image: consul
+    ports:
+      - "8500:8500"
+    environment:
+      CONSUL_BIND_INTERFACE: eth0

data/lib/consul_application_settings/configuration.rb

@@ -3,12 +3,22 @@ module ConsulApplicationSettings
   class Configuration
     DEFAULT_BASE_FILE_PATH = 'config/app_settings.yml'.freeze
     DEFAULT_LOCAL_FILE_PATH = 'config/app_settings.local.yml'.freeze
-    attr_accessor :base_file_path, :local_file_path, :disable_consul_connection_errors
+    DEFAULT_PROVIDERS = [
+      ConsulApplicationSettings::Providers::ConsulPreloaded,
+      ConsulApplicationSettings::Providers::LocalStorage
+    ].freeze
+    DEFAULT_RESOLVERS = [
+      ConsulApplicationSettings::Resolvers::Env
+    ].freeze
+    attr_accessor :base_file_path, :local_file_path, :disable_consul_connection_errors,
+                  :settings_providers, :value_resolvers
 
     def initialize
       @base_file_path = DEFAULT_BASE_FILE_PATH
       @local_file_path = DEFAULT_LOCAL_FILE_PATH
       @disable_consul_connection_errors = true
+      @settings_providers = DEFAULT_PROVIDERS
+      @value_resolvers = DEFAULT_RESOLVERS
     end
   end
 end

data/lib/consul_application_settings/providers/abstract.rb

@@ -0,0 +1,26 @@
+module ConsulApplicationSettings
+  module Providers
+    # Abstract class with basic functionality
+    class Abstract
+      def initialize(base_path, config)
+        @base_path = base_path
+        @config = config
+      end
+
+      def get(_path)
+        raise NotImplementedError
+      end
+
+      protected
+
+      def absolute_key_path(path)
+        ConsulApplicationSettings::Utils.generate_path(@base_path, path)
+      end
+
+      def get_value_from_hash(path, data)
+        parts = ConsulApplicationSettings::Utils.decompose_path(path)
+        data.dig(*parts).clone
+      end
+    end
+  end
+end

data/lib/consul_application_settings/providers/consul.rb

@@ -0,0 +1,32 @@
+require 'diplomat'
+
+module ConsulApplicationSettings
+  module Providers
+    # Provides access to settings stored in Consul
+    class Consul < Abstract
+      def get(path)
+        full_path = absolute_key_path(path)
+        value = get_from_consul(full_path)
+        value = resolve_tree_response(value, full_path)
+        ConsulApplicationSettings::Utils.cast_consul_value(value)
+      end
+
+      private
+
+      def get_from_consul(path)
+        Diplomat::Kv.get(path, recurse: true)
+      rescue Diplomat::KeyNotFound
+        nil
+      rescue SystemCallError, Faraday::ConnectionFailed, Diplomat::PathNotFound => e
+        raise e unless @config.disable_consul_connection_errors
+      end
+
+      def resolve_tree_response(value, full_path)
+        return value unless value.is_a?(Array)
+
+        value.each { |item| item[:key] = item[:key].delete_prefix("#{full_path}/") }
+        value
+      end
+    end
+  end
+end

data/lib/consul_application_settings/providers/consul_preloaded.rb

@@ -0,0 +1,45 @@
+module ConsulApplicationSettings
+  module Providers
+    # Provides access to settings stored in Consul. Loads them once
+    class ConsulPreloaded < Abstract
+      def initialize(base_path, config)
+        super
+        @data = read_all_from_consul
+      end
+
+      def get(path)
+        value = get_value_from_hash(absolute_key_path(path), @data)
+        value = resolve_tree_response(value)
+        ConsulApplicationSettings::Utils.cast_consul_value(value)
+      end
+
+      protected
+
+      def read_all_from_consul
+        Diplomat::Kv.get_all(@base_path, convert_to_hash: true)
+      rescue Diplomat::KeyNotFound
+        {}
+      rescue SystemCallError, Faraday::ConnectionFailed, Diplomat::PathNotFound => e
+        raise e unless @config.disable_consul_connection_errors
+
+        {}
+      end
+
+      def resolve_tree_response(value)
+        return value unless value.is_a?(Hash)
+
+        deep_cast(value)
+      end
+
+      def deep_cast(hash)
+        hash.each_pair do |k, v|
+          if v.is_a?(Hash)
+            deep_cast(v)
+          else
+            hash[k] = ConsulApplicationSettings::Utils.cast_consul_value(v)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/consul_application_settings/providers/local_storage.rb

@@ -0,0 +1,42 @@
+require 'yaml'
+
+module ConsulApplicationSettings
+  module Providers
+    # Provides access to settings stored in file system with support of base and local files
+    class LocalStorage < Abstract
+      def initialize(base_path, config)
+        super
+        @data = load
+      end
+
+      def get(path)
+        get_value_from_hash(absolute_key_path(path), @data)
+      end
+
+      private
+
+      def load
+        base_yml = read_yml(base_file_path)
+        local_yml = read_yml(local_file_path)
+        DeepMerge.deep_merge!(local_yml, base_yml, preserve_unmergeables: false, overwrite_arrays: true,
+                                                   merge_nil_values: true)
+      end
+
+      def base_file_path
+        @config.base_file_path
+      end
+
+      def local_file_path
+        @config.local_file_path
+      end
+
+      def read_yml(path)
+        return {} unless File.exist?(path)
+
+        YAML.safe_load(IO.read(path))
+      rescue Psych::SyntaxError, Errno::ENOENT => e
+        raise ConsulApplicationSettings::Error, "Cannot read settings file at #{path}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/consul_application_settings/reader.rb

@@ -0,0 +1,50 @@
+module ConsulApplicationSettings
+  # Orchestrates fetching values from provider and resolving them
+  class Reader
+    def initialize(base_path, config)
+      @base_path = base_path
+      @config = config
+      @providers = config.settings_providers.map { |provider| provider.new(base_path, config) }
+      @resolvers = config.value_resolvers.map(&:new)
+    end
+
+    def get(path)
+      value = fetch_value(path)
+      resolve_value(value, path)
+    end
+
+    alias [] get
+
+    def load(sub_path)
+      new_path = ConsulApplicationSettings::Utils.generate_path(@base_path, sub_path)
+      self.class.new(new_path, @config)
+    end
+
+    protected
+
+    def check_deep_structure(value, path)
+      return unless value.is_a?(Hash)
+
+      message = "Getting value of complex object at path: '#{path}'. Use #load method to get new scoped instance"
+      raise ConsulApplicationSettings::Error, message if value.is_a?(Hash)
+    end
+
+    def fetch_value(path)
+      @providers.each do |provider|
+        value = provider.get(path)
+        check_deep_structure(value, path)
+        return value unless value.nil?
+      end
+      nil
+    end
+
+    def resolve_value(value, path)
+      resolver = @resolvers.detect { |r| r.resolvable?(value, path) }
+      if resolver
+        resolver.resolve(value, path)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/consul_application_settings/resolvers/abstract.rb

@@ -0,0 +1,14 @@
+module ConsulApplicationSettings
+  module Resolvers
+    # Abstract resolver with basic functionality
+    class Abstract
+      def resolvable?(_value, _path)
+        false
+      end
+
+      def resolve(value, _path)
+        value
+      end
+    end
+  end
+end

data/lib/consul_application_settings/resolvers/env.rb

@@ -0,0 +1,19 @@
+module ConsulApplicationSettings
+  module Resolvers
+    # Resolve values in environment
+    class Env
+      IDENTIFIER = 'env://'.freeze
+
+      def resolvable?(value, _path)
+        return unless value.respond_to?(:start_with?)
+
+        value.start_with?(IDENTIFIER)
+      end
+
+      def resolve(value, _path)
+        env_path = value.to_s.delete_prefix(IDENTIFIER)
+        ENV[env_path]
+      end
+    end
+  end
+end

data/lib/consul_application_settings/resolvers/erb.rb

@@ -0,0 +1,18 @@
+module ConsulApplicationSettings
+  module Resolvers
+    # Run values through ERB
+    class Erb
+      IDENTIFIER = /(<%).*(%>)/.freeze
+
+      def resolvable?(value, _path)
+        return unless value.is_a?(String)
+
+        IDENTIFIER.match?(value)
+      end
+
+      def resolve(value, _path)
+        ERB.new(value.to_s).result
+      end
+    end
+  end
+end

data/lib/consul_application_settings/resolvers/vault.rb

@@ -0,0 +1,24 @@
+module ConsulApplicationSettings
+  module Resolvers
+    # Resolve values using vault
+    class Vault
+      IDENTIFIER = 'vault://'.freeze
+
+      def resolvable?(value, _path)
+        return unless value.respond_to?(:start_with?)
+
+        value.start_with?(IDENTIFIER)
+      end
+
+      # Expect value in format `vault://mount/path/to/secret?attribute_name`
+      def resolve(value, _path)
+        value = value.delete_prefix(IDENTIFIER)
+        mount, secret = value.split('/', 2)
+        secret, attribute = secret.split('?')
+        attribute ||= 'value'
+        secret = ::Vault.kv(mount).read(secret)
+        secret && secret.data[attribute.to_sym]
+      end
+    end
+  end
+end

data/lib/consul_application_settings/utils.rb

@@ -6,8 +6,11 @@ module ConsulApplicationSettings
 
     class << self
       def cast_consul_value(value)
+        return nil if value.nil?
         return false if value == 'false'
         return true if value == 'true'
+        return value if value.is_a?(Hash)
+        return convert_to_hash(value) if value.is_a?(Array)
 
         cast_complex_value(value)
       end
@@ -33,6 +36,16 @@ module ConsulApplicationSettings
         end
         value.to_s
       end
+
+      def convert_to_hash(data)
+        data_h = data.map do |item|
+          value = cast_consul_value(item[:value])
+          item[:key].split('/').reverse.reduce(value) { |h, v| { v => h } }
+        end
+        data_h.reduce({}) do |dest, source|
+          DeepMerge.deep_merge!(source, dest, preserve_unmergeables: true)
+        end
+      end
     end
   end
 end

data/lib/consul_application_settings/version.rb

@@ -1,3 +1,3 @@
 module ConsulApplicationSettings
-  VERSION = '2.1.0'.freeze
+  VERSION = '4.0.0-alpha'.freeze
 end

data/lib/consul_application_settings.rb

@@ -1,8 +1,13 @@
 require 'consul_application_settings/version'
+require 'consul_application_settings/providers/abstract'
+require 'consul_application_settings/providers/consul'
+require 'consul_application_settings/providers/consul_preloaded'
+require 'consul_application_settings/providers/local_storage'
+require 'consul_application_settings/resolvers/abstract'
+require 'consul_application_settings/resolvers/env'
+require 'consul_application_settings/resolvers/erb'
 require 'consul_application_settings/configuration'
-require 'consul_application_settings/consul_provider'
-require 'consul_application_settings/file_provider'
-require 'consul_application_settings/settings_provider'
+require 'consul_application_settings/reader'
 require 'consul_application_settings/utils'
 
 # The gem provides possibility to load settings from Consul and automatically fall back to data stored in file system
@@ -11,7 +16,6 @@ module ConsulApplicationSettings
 
   class << self
     attr_accessor :config
-    attr_accessor :defaults
   end
 
   self.config ||= ConsulApplicationSettings::Configuration.new
@@ -21,6 +25,6 @@ module ConsulApplicationSettings
   end
 
   def self.load(path = '')
-    SettingsProvider.new(path, config)
+    Reader.new(path, config)
   end
 end

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: consul_application_settings
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 4.0.0.pre.alpha
 platform: ruby
 authors:
 - Volodymyr Mykhailyk
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-25 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: diplomat
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.3
+        version: 2.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.3
+        version: 2.5.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: vault
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
 description: |-
   Gem that simplifies usage of Consul (via Diplomat gem) to host application settings.
                         Gem provides defaults and utilities
@@ -117,7 +159,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".codeclimate.yml"
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rspec"
@@ -130,15 +171,23 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/benchmark
 - bin/console
 - bin/rspec
 - bin/setup
 - consul_application_settings.gemspec
+- docker-compose.yml
 - lib/consul_application_settings.rb
 - lib/consul_application_settings/configuration.rb
-- lib/consul_application_settings/consul_provider.rb
-- lib/consul_application_settings/file_provider.rb
-- lib/consul_application_settings/settings_provider.rb
+- lib/consul_application_settings/providers/abstract.rb
+- lib/consul_application_settings/providers/consul.rb
+- lib/consul_application_settings/providers/consul_preloaded.rb
+- lib/consul_application_settings/providers/local_storage.rb
+- lib/consul_application_settings/reader.rb
+- lib/consul_application_settings/resolvers/abstract.rb
+- lib/consul_application_settings/resolvers/env.rb
+- lib/consul_application_settings/resolvers/erb.rb
+- lib/consul_application_settings/resolvers/vault.rb
 - lib/consul_application_settings/utils.rb
 - lib/consul_application_settings/version.rb
 homepage: https://github.com/matic-insurance/consul_application_settings
@@ -159,11 +208,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Application settings via Consul with yaml defaults

data/.codeclimate.yml

@@ -1,15 +0,0 @@
-version: 2
-plugins:
-#  disabled before git-legal supports bundler 2 https://github.com/kmewhort/git.legal-codeclimate/issues/6
-#  git-legal:
-#    enabled: true
-  rubocop:
-    enabled: true
-    channel: rubocop-0-73
-    config:
-      file: ".rubocop.yml"
-  bundler-audit:
-    enabled: true
-exclude_patterns:
-  - "spec/"
-  - "*.gemspec"

data/lib/consul_application_settings/consul_provider.rb

@@ -1,33 +0,0 @@
-require 'diplomat'
-
-module ConsulApplicationSettings
-  # Provides access to settings stored in Consul
-  class ConsulProvider
-    def initialize(base_path, config)
-      @base_path = base_path
-      @config = config
-    end
-
-    def get(path)
-      value = fetch_value(path)
-      ConsulApplicationSettings::Utils.cast_consul_value(value)
-    end
-
-    private
-
-    def fetch_value(path)
-      full_path = generate_full_path(path)
-      Diplomat::Kv.get(full_path, {}, :return)
-    rescue SystemCallError, Faraday::ConnectionFailed, Diplomat::PathNotFound => e
-      raise e unless disable_consul_connection_errors?
-    end
-
-    def generate_full_path(path)
-      ConsulApplicationSettings::Utils.generate_path(@base_path, path)
-    end
-
-    def disable_consul_connection_errors?
-      @config.disable_consul_connection_errors
-    end
-  end
-end

data/lib/consul_application_settings/file_provider.rb

@@ -1,54 +0,0 @@
-require 'yaml'
-
-module ConsulApplicationSettings
-  # Provides access to settings stored in file system with support of base and local files
-  class FileProvider
-    def initialize(base_path, config)
-      @base_path = base_path
-      @config = config
-      load
-    end
-
-    def get(path)
-      read_path(path).clone
-    end
-
-    private
-
-    def load
-      base_yml = read_yml(base_file_path)
-      local_yml = read_yml(local_file_path)
-      @data = DeepMerge.deep_merge!(local_yml, base_yml, preserve_unmergeables: false, overwrite_arrays: true,
-                                                         merge_nil_values: true)
-    end
-
-    def base_file_path
-      @config.base_file_path
-    end
-
-    def local_file_path
-      @config.local_file_path
-    end
-
-    def read_yml(path)
-      return {} unless File.exist?(path)
-
-      YAML.safe_load(IO.read(path))
-    rescue Psych::SyntaxError, Errno::ENOENT => e
-      raise ConsulApplicationSettings::Error, "Cannot read settings file at #{path}: #{e.message}"
-    end
-
-    def read_path(path)
-      full_path = ConsulApplicationSettings::Utils.generate_path(@base_path, path)
-      parts = ConsulApplicationSettings::Utils.decompose_path(full_path)
-      parts.reduce(@data, &method(:read_value))
-    end
-
-    def read_value(hash, key)
-      raise ConsulApplicationSettings::Error, 'reading arrays not implemented' if hash.is_a? Array
-      return {} if hash.nil?
-
-      hash.fetch(key.to_s, nil)
-    end
-  end
-end

data/lib/consul_application_settings/settings_provider.rb

@@ -1,16 +0,0 @@
-module ConsulApplicationSettings
-  # Provides access to settings stored in Consul or in file system
-  class SettingsProvider
-    def initialize(base_path, config)
-      @consul_provider = ConsulProvider.new(base_path, config)
-      @file_provider = FileProvider.new(base_path, config)
-    end
-
-    def get(path)
-      consul_value = @consul_provider.get(path)
-      !consul_value.nil? && consul_value != '' ? consul_value : @file_provider.get(path)
-    end
-
-    alias [] get
-  end
-end