secret_config 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a86408c0a69f9e1c3e67a06d004c659abe13f7b482a3ff11e8de6b99c650d05
-  data.tar.gz: cd16668968bf171809b47b2c3b749de2ef63a7fb91245cdf68bac253aeb5639b
+  metadata.gz: 24792842f7d81af2c5d70eb431eae87ec96f190a52500c7e68267b6e875a4e09
+  data.tar.gz: 29f1683c2be73597476f976860dd021476559d97ed94e04195789fadaf4fb7f8
 SHA512:
-  metadata.gz: b71279e1a7855b8e081239f3eea1bf05c3c6edc5e4226e8e31e347667480156c493f87443d09bc060a2829a7cd87ab1acdcba9e0cad9811c634355b4944ecea7
-  data.tar.gz: ce127ef03fa642011deb394e3f3cd3de94e8d4ace3de712dbc218dd080ed4ea03ce2ff06c8ee35d32e2b06765e08c93a92cf2066ca072e4d16380e52d2d58504
+  metadata.gz: 89a0ecfab4bfb8263768ccf51ec56b987866cac08d23c2c9e34a69b9ba3210d9ed40198cfd525d72f54dcf4e6aa0f2c8965d30204ec96c4b825f1c61313c35b5
+  data.tar.gz: c0047de74541f0589ebe2ad66a67133b73d6fa5635610d81b685394aca395d46dfadc4171ca655975296289292203dc004a977fe265c6b0d704ad4a894cc6983

data/README.md

@@ -5,6 +5,13 @@ Centralized Configuration and Secrets Management for Ruby and Rails applications
 
 Securely store configuration information centrally, supporting multiple tenants of the same application.
 
+## Overview
+
+Securely store centralized configuration information such as:
+* Settings
+* Passwords
+* Encryption keys and certificates
+
 ## Features
 
 Supports storing configuration information in:
@@ -15,6 +22,17 @@ Supports storing configuration information in:
 * AWS System Manager Parameter Store
     * Encrypt and securely store secrets such as passwords centrally. 
 
+Supported data types:
+* integer
+* float
+* string
+* boolean
+* symbol
+* json
+
+Supported conversions:
+* base64
+
 ## Benefits
 
 Benefits of moving sensitive configuration information into AWS System Manager Parameter Store:
@@ -23,7 +41,7 @@ Benefits of moving sensitive configuration information into AWS System Manager P
     * Environment variables force all config into a single level.
   * Reduces the number of environment variables.
     * In a large application the number of secrets can grow dramatically.
-  * Removes the need to encrypt sensitive data config files.
+  * Replaces sensitive data stored in local yaml or configuration files.
     * Including securing and managing encryption keys.
   * When encryption keys change, such as during a key rotation, config files don't have to be changed.
   * Removes security concerns with placing passwords in the clear into environment variables.
@@ -222,7 +240,7 @@ Note: Do not put any production credentials into this file.
 
 ### Environment Variables
 
-Any of the above values can be overridden with an environment variable.
+Any of the above values can be overridden with an environment variable, unless explicitly configured `SecretConfig.check_env_var = false`.
 
 To overwrite any of these settings with an environment variable:
 
@@ -471,6 +489,49 @@ to view and modify parameters:
 - `ssm:GetParameters`
 - `ssm:GetParameter`
 
+## String Interpolation
+
+Values supplied for config settings can be replaced inline with date, time, hostname, pid and random values.
+
+For example to include the `hostname` in the log file name setting:
+
+~~~yaml
+development:
+  logger:
+    level:     info
+    file_name: /var/log/my_application_%{hostname}.log
+~~~
+
+Available interpolations:
+ 
+* %{date}
+    * Current date in the format of "%Y%m%d" (CCYYMMDD)
+* %{date:format}
+    * Current date in the supplied format. See strftime
+* %{time}
+    * Current date and time down to ms in the format of "%Y%m%d%Y%H%M%S%L" (CCYYMMDDHHMMSSmmm)
+* %{time:format}
+    * Current date and time in the supplied format. See strftime
+* %{env:name}
+    * Extract value from the named environment variable.
+* %{hostname}
+    * Full name of this host.
+* %{hostname:short}
+    * Short name of this host. Everything up to the first period.
+* %{pid}
+    * Process Id for this process.
+* %{random}
+    * URL safe Random 32 byte value.
+* %{random:size}
+    * URL safe Random value of `size` bytes.
+
+#### Notes:
+
+* To prevent interpolation use %%{...}
+* %% is not touched, only %{...} is searched for.
+* Since these interpolations are only evaluated at load time and
+  every time the registry is refreshed there is no runtime overhead when keys are fetched.
+
 ## Command Line Interface
 
 Secret Config has a command line interface for exporting, importing and copying between paths in the registry.

data/lib/secret_config/errors.rb

@@ -10,4 +10,7 @@ module SecretConfig
 
   class ConfigurationError < Error
   end
+
+  class InvalidInterpolation < Error
+  end
 end

data/lib/secret_config/providers/ssm.rb

@@ -62,7 +62,8 @@ module SecretConfig
           value:     value.to_s,
           type:      "SecureString",
           key_id:    key_id,
-          overwrite: true
+          overwrite: true,
+          tier:      "Intelligent-Tiering"
         )
       end
 

data/lib/secret_config/registry.rb

@@ -32,7 +32,13 @@ module SecretConfig
 
     # Returns [String] configuration value for the supplied key, or nil when missing.
     def [](key)
-      cache[expand_key(key)]
+      full_key = expand_key(key)
+      value    = cache[full_key]
+      if value.nil? && SecretConfig.check_env_var?
+        value           = env_var_override(key, value)
+        cache[full_key] = value unless value.nil?
+      end
+      value
     end
 
     # Returns [String] configuration value for the supplied key, or nil when missing.
@@ -45,8 +51,7 @@ module SecretConfig
       value = self[key]
       if value.nil?
         raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
-
-        value = default.respond_to?(:call) ? default.call : default
+        value = block_given? ? yield : default
       end
 
       value = convert_encoding(encoding, value) if encoding
@@ -78,7 +83,8 @@ module SecretConfig
       existing_keys = cache.keys
       updated_keys  = []
       provider.each(path) do |key, value|
-        cache[key] = env_var_override(key, value)
+        value      = interpolator.parse(value) if value.is_a?(String) && value.include?('%{')
+        cache[key] = env_var_override(relative_key(key), value)
         updated_keys << key
       end
 
@@ -92,10 +98,16 @@ module SecretConfig
 
     attr_reader :cache
 
+    def interpolator
+      @interpolator ||= SettingInterpolator.new
+    end
+
     # Returns the value from an env var if it is present,
     # Otherwise the value is returned unchanged.
     def env_var_override(key, value)
-      env_var_name = relative_key(key).upcase.gsub("/", "_")
+      return value unless SecretConfig.check_env_var?
+
+      env_var_name = key.upcase.gsub("/", "_")
       ENV[env_var_name] || value
     end
 

data/lib/secret_config/setting_interpolator.rb

@@ -0,0 +1,58 @@
+require 'date'
+require 'socket'
+require 'securerandom'
+# * SecretConfig Interpolations
+#
+# Expanding values inline for date, time, hostname, pid and random values.
+#   %{date}           # Current date in the format of "%Y%m%d" (CCYYMMDD)
+#   %{date:format}    # Current date in the supplied format. See strftime
+#   %{time}           # Current date and time down to ms in the format of "%Y%m%d%Y%H%M%S%L" (CCYYMMDDHHMMSSmmm)
+#   %{time:format}    # Current date and time in the supplied format. See strftime
+#   %{env:name}       # Extract value from the named environment value.
+#   %{hostname}       # Full name of this host.
+#   %{hostname:short} # Short name of this host. Everything up to the first period.
+#   %{pid}            # Process Id for this process.
+#   %{random}         # URL safe Random 32 byte value.
+#   %{random:size}    # URL safe Random value of `size` bytes.
+#
+# Retrieve values elsewhere in the registry.
+# Paths can be relative to the current root, or absolute paths outside the current root.
+#   %{fetch:key}      # Fetches a single value from a relative or absolute path
+#   %{include:path}   # Fetches a path of keys and values
+module SecretConfig
+  class SettingInterpolator < StringInterpolator
+    def date(format = "%Y%m%d")
+      Date.today.strftime(format)
+    end
+
+    def time(format = "%Y%m%d%H%M%S%L")
+      Time.now.strftime(format)
+    end
+
+    def env(name)
+      ENV[name]
+    end
+
+    def hostname(format = nil)
+      name = Socket.gethostname
+      name = name.split('.')[0] if format == "short"
+      name
+    end
+
+    def pid
+      $$
+    end
+
+    def random(size = 32)
+      SecureRandom.urlsafe_base64(size)
+    end
+
+    #def fetch(key)
+    #  SecretConfig[key]
+    #end
+    #
+    #def include(path)
+    #
+    #end
+  end
+end

data/lib/secret_config/string_interpolator.rb

@@ -0,0 +1,32 @@
+# Parse strings containing %{key:value1,value2,value3}
+# Where `key` is a method implemented by a class inheriting from this class
+#
+# The following `key`s are reserved:
+# * parse
+# * initialize
+#
+# Notes:
+# * To prevent interpolation use %%{...}
+# * %% is not touched, only %{...} is identified.
+module SecretConfig
+  class StringInterpolator
+    def initialize(pattern = nil)
+      @pattern = pattern || /%{1,2}\{([^}]+)\}/
+    end
+
+    def parse(string)
+      string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
+        if match.start_with?('%%')
+          match[1..-1]
+        else
+          expr          = $1 || $2 || match.tr("%{}", "")
+          key, args_str = expr.split(':')
+          key           = key.strip.to_sym
+          arguments     = args_str&.split(',')&.map { |v| v.strip == '' ? nil : v.strip } || []
+          raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
+          public_send(key, *arguments)
+        end
+      end
+    end
+  end
+end

data/lib/secret_config/version.rb

@@ -1,3 +1,3 @@
 module SecretConfig
-  VERSION = "0.6.4".freeze
+  VERSION = "0.7.0".freeze
 end

data/lib/secret_config.rb

@@ -18,6 +18,8 @@ module SecretConfig
   end
 
   autoload :CLI, "secret_config/cli"
+  autoload :SettingInterpolator, "secret_config/setting_interpolator"
+  autoload :StringInterpolator, "secret_config/string_interpolator"
   autoload :Utils, "secret_config/utils"
 
   class << self
@@ -65,8 +67,6 @@ module SecretConfig
     @check_env_var = check_env_var
   end
 
-  private
-
   @check_env_var = true
   @filters       = [/password/i, /key\Z/i, /passphrase/i]
 end

data/test/config/application.yml

@@ -41,7 +41,7 @@ test:
     mongo:
       database:   secret_config_test
       primary:    127.0.0.1:27017
-      secondary:  127.0.0.1:27018
+      secondary:  "%{hostname}:27018"
 
     secrets:
       secret_key_base: somereallylongteststring

data/test/providers/file_test.rb

@@ -15,7 +15,7 @@ module Providers
         {
           "/test/my_application/mongo/database"               => "secret_config_test",
           "/test/my_application/mongo/primary"                => "127.0.0.1:27017",
-          "/test/my_application/mongo/secondary"              => "127.0.0.1:27018",
+          "/test/my_application/mongo/secondary"              => "%{hostname}:27018",
           "/test/my_application/mysql/database"               => "secret_config_test",
           "/test/my_application/mysql/password"               => "secret_configrules",
           "/test/my_application/mysql/username"               => "secret_config",

data/test/registry_test.rb

@@ -1,4 +1,5 @@
 require_relative 'test_helper'
+require 'socket'
 
 class RegistryTest < Minitest::Test
   describe SecretConfig::Registry do
@@ -22,7 +23,7 @@ class RegistryTest < Minitest::Test
       {
         "/test/my_application/mongo/database"               => "secret_config_test",
         "/test/my_application/mongo/primary"                => "127.0.0.1:27017",
-        "/test/my_application/mongo/secondary"              => "127.0.0.1:27018",
+        "/test/my_application/mongo/secondary"              => "#{Socket.gethostname}:27018",
         "/test/my_application/mysql/database"               => "secret_config_test",
         "/test/my_application/mysql/password"               => "secret_configrules",
         "/test/my_application/mysql/username"               => "secret_config",

data/test/secret_config_test.rb

@@ -1,9 +1,10 @@
-require_relative 'test_helper'
+require_relative "test_helper"
+require "socket"
 
 class SecretConfigTest < Minitest::Test
   describe SecretConfig::Providers::File do
     let :file_name do
-      File.join(File.dirname(__FILE__), 'config', 'application.yml')
+      File.join(File.dirname(__FILE__), "config", "application.yml")
     end
 
     let :path do
@@ -14,39 +15,56 @@ class SecretConfigTest < Minitest::Test
       SecretConfig.use :file, path: path, file_name: file_name
     end
 
-    describe '#configuration' do
-      it 'returns a copy of the config' do
+    describe "#configuration" do
+      it "returns a copy of the config" do
         assert_equal "127.0.0.1", SecretConfig.configuration.dig("mysql", "host")
       end
     end
 
-    describe '#key?' do
-      it 'has key' do
+    describe "#key?" do
+      it "has key" do
         assert SecretConfig.key?("mysql/database")
       end
     end
 
-    describe '#[]' do
-      it 'returns values' do
+    describe "#[]" do
+      it "returns values" do
         assert_equal "secret_config_test", SecretConfig["mysql/database"]
       end
+
+      it "returns values with interpolation" do
+        assert_equal "#{Socket.gethostname}:27018", SecretConfig["mongo/secondary"]
+      end
     end
 
-    describe '#fetch' do
+    describe "#fetch" do
       after do
-        ENV['MYSQL_DATABASE'] = nil
+        ENV["MYSQL_DATABASE"]      = nil
+        SecretConfig.check_env_var = true
       end
 
-      it 'fetches values' do
+      it "fetches values" do
         assert_equal "secret_config_test", SecretConfig.fetch("mysql/database")
       end
 
-      it 'can be overridden by an environment variable' do
-        ENV['MYSQL_DATABASE'] = 'other'
+      it "can be overridden by an environment variable" do
+        ENV["MYSQL_DATABASE"] = "other"
 
         SecretConfig.use :file, path: path, file_name: file_name
         assert_equal "other", SecretConfig.fetch("mysql/database")
       end
+
+      it "returns values with interpolation" do
+        assert_equal "#{Socket.gethostname}:27018", SecretConfig.fetch("mongo/secondary")
+      end
+
+      it "can be omitted an environment variable override with #check_env_var configuration" do
+        ENV["MYSQL_DATABASE"] = "other"
+
+        SecretConfig.check_env_var = false
+        SecretConfig.use :file, path: path, file_name: file_name
+        assert_equal "secret_config_test", SecretConfig.fetch("mysql/database")
+      end
     end
   end
 end

data/test/setting_interpolator_test.rb

@@ -0,0 +1,152 @@
+require_relative 'test_helper'
+module SecretConfig
+  class SettingInterpolatorTest < Minitest::Test
+    describe SettingInterpolator do
+      let(:interpolator) { SettingInterpolator.new }
+
+      describe '#parse' do
+        it "handles good key" do
+          string   = "Set a date of %{date} here."
+          expected = string.gsub("%{date}", Date.today.strftime("%Y%m%d"))
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+
+        it "handles multiple keys" do
+          string   = "%{pid}: Set a date of %{date} here and a %{time:%H%M} here and for luck %{pid}"
+          expected = string.gsub("%{date}", Date.today.strftime("%Y%m%d"))
+          expected = expected.gsub("%{time:%H%M}", Time.now.strftime("%H%M"))
+          expected = expected.gsub("%{pid}", $$.to_s)
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+
+        it "handles bad key" do
+          string = "Set a date of %{blah} here."
+          assert_raises InvalidInterpolation do
+            interpolator.parse(string)
+          end
+        end
+      end
+
+      describe "#date" do
+        it 'interpolates date only' do
+          string   = "%{date}"
+          expected = Date.today.strftime("%Y%m%d")
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+
+        it 'interpolates date' do
+          string   = "Set a date of %{date} here."
+          expected = string.gsub("%{date}", Date.today.strftime("%Y%m%d"))
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+
+        it 'interpolates date with custom format' do
+          string   = "Set a custom %{date:%m%d%Y} here."
+          expected = string.gsub("%{date:%m%d%Y}", Date.today.strftime("%m%d%Y"))
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+      end
+
+      describe "#time" do
+        it 'interpolates time only' do
+          string = "%{time}"
+          time   = Time.now
+          Time.stub(:now, time) do
+            expected = Time.now.strftime("%Y%m%d%H%M%S%L")
+            actual   = interpolator.parse(string)
+            assert_equal expected, actual, string
+          end
+        end
+
+        it 'interpolates time' do
+          string = "Set a time of %{time} here."
+          time   = Time.now
+          Time.stub(:now, time) do
+            expected = string.gsub("%{time}", Time.now.strftime("%Y%m%d%H%M%S%L"))
+            actual   = interpolator.parse(string)
+            assert_equal expected, actual, string
+          end
+        end
+
+        it 'interpolates time with custom format' do
+          string   = "Set a custom time of %{time:%H%M} here."
+          expected = string.gsub("%{time:%H%M}", Time.now.strftime("%H%M"))
+          actual   = interpolator.parse(string)
+          assert_equal expected, actual, string
+        end
+      end
+
+      describe "#env" do
+        before do
+          ENV["TEST_SETTING"] = "Secret"
+        end
+
+        it 'fetches existing ENV var' do
+          string = "%{env:TEST_SETTING}"
+          actual = interpolator.parse(string)
+          assert_equal "Secret", actual, string
+        end
+
+        it 'fetches existing ENV var into a larger string' do
+          string   = "Hello %{env:TEST_SETTING}. How are you?"
+          actual   = interpolator.parse(string)
+          expected = string.gsub("%{env:TEST_SETTING}", "Secret")
+          assert_equal expected, actual, string
+        end
+
+        it 'handles missing ENV var' do
+          string = "%{env:OTHER_TEST_SETTING}"
+          actual = interpolator.parse(string)
+          assert_equal "", actual, string
+        end
+      end
+
+      describe "#hostname" do
+        it 'returns hostname' do
+          string = "%{hostname}"
+          actual = interpolator.parse(string)
+          assert_equal Socket.gethostname, actual, string
+        end
+
+        it 'returns short hostname' do
+          string = "%{hostname:short}"
+          actual = interpolator.parse(string)
+          assert_equal Socket.gethostname.split('.')[0], actual, string
+        end
+      end
+
+      describe "#pid" do
+        it 'returns process id' do
+          string = "%{pid}"
+          actual = interpolator.parse(string)
+          assert_equal $$.to_s, actual, string
+        end
+      end
+
+      describe "#random" do
+        it 'interpolates random 32 byte string' do
+          string = "%{random}"
+          random = SecureRandom.urlsafe_base64(32)
+          SecureRandom.stub(:urlsafe_base64, random) do
+            actual = interpolator.parse(string)
+            assert_equal random, actual, string
+          end
+        end
+
+        it 'interpolates custom length random string' do
+          string = "%{random:64}"
+          random = SecureRandom.urlsafe_base64(64)
+          SecureRandom.stub(:urlsafe_base64, random) do
+            actual = interpolator.parse(string)
+            assert_equal random, actual, string
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: secret_config
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-21 00:00:00.000000000 Z
+date: 2020-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -44,6 +44,8 @@ files:
 - lib/secret_config/providers/ssm.rb
 - lib/secret_config/railtie.rb
 - lib/secret_config/registry.rb
+- lib/secret_config/setting_interpolator.rb
+- lib/secret_config/string_interpolator.rb
 - lib/secret_config/utils.rb
 - lib/secret_config/version.rb
 - test/config/application.yml
@@ -51,6 +53,7 @@ files:
 - test/providers/ssm_test.rb
 - test/registry_test.rb
 - test/secret_config_test.rb
+- test/setting_interpolator_test.rb
 - test/test_helper.rb
 - test/utils_test.rb
 homepage: https://github.com/rocketjob/secret_config
@@ -81,6 +84,7 @@ test_files:
 - test/providers/ssm_test.rb
 - test/providers/file_test.rb
 - test/registry_test.rb
+- test/setting_interpolator_test.rb
 - test/test_helper.rb
 - test/utils_test.rb
 - test/secret_config_test.rb