secret_config 0.7.1 → 0.8.0

data/lib/secret_config/parser.rb

@@ -0,0 +1,75 @@
+module SecretConfig
+  class Parser
+    attr_reader :tree, :path, :registry, :interpolator
+
+    def initialize(path, registry)
+      @path         = path
+      @registry     = registry
+      @fetch_list   = {}
+      @import_list  = {}
+      @tree         = {}
+      @interpolator = SettingInterpolator.new
+    end
+
+    # Returns a flat path of keys and values from the provider without looking in the local path.
+    # Keys are returned with path names relative to the supplied path.
+    def parse(key, value)
+      relative_key       = relative_key?(key) ? key : key.sub("#{path}/", "")
+      tree[relative_key] = value.is_a?(String) && value.include?("%{") ? interpolator.parse(value) : value
+    end
+
+    # Returns a flat Hash of the rendered paths.
+    def render
+      # apply_fetches
+      apply_imports
+      tree
+    end
+
+    private
+
+    # def apply_fetches
+    #   tree[key] = relative_key?(fetch_key) ? registry[fetch_key] : registry.provider.fetch(fetch_key)
+    # end
+
+    # Import from the current registry as well as new fetches.
+    #
+    # Notes:
+    # - A lot of absolute key lookups can be expensive since each one is a separate call.
+    # - Imports cannot reference other imports at this time.
+    def apply_imports
+      tree.keys.each do |key|
+        next unless key =~ /\/__import__\Z/
+
+        import_key = tree.delete(key)
+        key, _     = ::File.split(key)
+
+        # binding.irb
+
+        # With a relative key, look for the values in the current registry.
+        # With an absolute key call the provider and fetch the value directly.
+
+        if relative_key?(import_key)
+          tree.keys.each do |current_key|
+            match = current_key.match(/\A#{import_key}\/(.*)/)
+            next unless match
+
+            imported_key       = ::File.join(key, match[1])
+            tree[imported_key] = tree[current_key] unless tree.key?(imported_key)
+          end
+        else
+          relative_paths = registry.send(:fetch_path, import_key)
+          relative_paths.each_pair do |relative_key, value|
+            imported_key       = ::File.join(key, relative_key)
+            tree[imported_key] = value unless tree.key?(imported_key)
+          end
+        end
+      end
+    end
+
+    # Returns [true|false] whether the supplied key is considered a relative key.
+    def relative_key?(key)
+      !key.start_with?("/")
+    end
+
+  end
+end

data/lib/secret_config/providers/file.rb

@@ -12,17 +12,30 @@ module SecretConfig
         raise(ConfigurationError, "Cannot find config file: #{file_name}") unless ::File.exist?(file_name)
       end
 
+      # Yields the key with its absolute path and corresponding string value
       def each(path, &block)
-        config = YAML.load(ERB.new(::File.new(file_name).read).result)
-
-        paths    = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
-        settings = config.dig(*paths)
+        settings = fetch_path(path)
 
         raise(ConfigurationError, "Path #{paths.join('/')} not found in file: #{file_name}") unless settings
 
         Utils.flatten_each(settings, path, &block)
         nil
       end
+
+      # Returns the value or `nil` if not found
+      def fetch(key)
+        values = fetch_path(path)
+        value.is_a?(Hash) ? nil : value
+      end
+
+      private
+
+      def fetch_path(path)
+        config = YAML.load(ERB.new(::File.new(file_name).read).result)
+
+        paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
+        config.dig(*paths)
+      end
     end
   end
 end

data/lib/secret_config/providers/ssm.rb

@@ -10,7 +10,13 @@ module SecretConfig
     class Ssm < Provider
       attr_reader :client, :key_id, :retry_count, :retry_max_ms, :logger
 
-      def initialize(key_id: ENV["AWS_ACCESS_KEY_ID"], key_alias: ENV["AWS_ACCESS_KEY_ALIAS"], retry_count: 10, retry_max_ms: 3_000)
+      def initialize(
+        key_id: ENV["SECRET_CONFIG_KEY_ID"],
+        key_alias: ENV["SECRET_CONFIG_KEY_ALIAS"],
+        retry_count: 25,
+        retry_max_ms: 10_000,
+        **args
+      )
         @key_id       =
           if key_alias
             key_alias =~ %r{^alias/} ? key_alias : "alias/#{key_alias}"
@@ -20,9 +26,10 @@ module SecretConfig
         @retry_count  = retry_count
         @retry_max_ms = retry_max_ms
         @logger       = SemanticLogger["Aws::SSM"] if defined?(SemanticLogger)
-        @client       = Aws::SSM::Client.new(logger: logger)
+        @client       = Aws::SSM::Client.new({logger: logger}.merge!(args))
       end
 
+      # Yields the key with its absolute path and corresponding string value
       def each(path)
         retries = 0
         token   = nil

data/lib/secret_config/registry.rb

@@ -20,10 +20,14 @@ module SecretConfig
     end
 
     # Returns [Hash] a copy of the in memory configuration data.
-    def configuration(relative: true, filters: SecretConfig.filters)
+    #
+    # Supply the relative path to start from so that only keys and values in that
+    # path will be returned.
+    def configuration(path: nil, filters: SecretConfig.filters)
       h = {}
       cache.each_pair do |key, value|
-        key   = relative_key(key) if relative
+        next if path && !key.start_with?(path)
+
         value = filter_value(key, value, filters)
         Utils.decompose(key, value, h)
       end
@@ -32,30 +36,36 @@ module SecretConfig
 
     # Returns [String] configuration value for the supplied key, or nil when missing.
     def [](key)
-      full_key = expand_key(key)
-      value    = cache[full_key]
+      value = cache[key]
       if value.nil? && SecretConfig.check_env_var?
-        value           = env_var_override(key, value)
-        cache[full_key] = value unless value.nil?
+        value      = env_var_override(key, value)
+        cache[key] = value unless value.nil?
       end
-      value
+      value.nil? ? nil : value.to_s
     end
 
     # Returns [String] configuration value for the supplied key, or nil when missing.
     def key?(key)
-      cache.key?(expand_key(key))
+      cache.key?(key)
     end
 
     # Returns [String] configuration value for the supplied key
-    def fetch(key, default: :no_default_supplied, type: :string, encoding: nil)
+    # Convert the string value into an array of values by supplying a `separator`.
+    def fetch(key, default: :no_default_supplied, type: :string, encoding: nil, separator: nil)
       value = self[key]
       if value.nil?
         raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
+
         value = block_given? ? yield : default
       end
 
       value = convert_encoding(encoding, value) if encoding
-      type == :string ? value : convert_type(type, value)
+
+      if separator
+        value.to_s.split(separator).collect { |element| convert_type(type, element.strip) }
+      else
+        convert_type(type, value)
+      end
     end
 
     # Set the value for a key in the centralized configuration store.
@@ -65,26 +75,25 @@ module SecretConfig
 
     # Set the value for a key in the centralized configuration store.
     def set(key, value)
-      key = expand_key(key)
-      provider.set(key, value)
+      full_key = expand_key(key)
+      provider.set(full_key, value)
       cache[key] = value
     end
 
     # Delete a key from the centralized configuration store.
     def delete(key)
-      key = expand_key(key)
-      provider.delete(key)
+      full_key = expand_key(key)
+      provider.delete(full_key)
       cache.delete(key)
     end
 
     # Refresh the in-memory cached copy of the centralized configuration information.
-    # Environment variable values will take precendence over the central store values.
+    # Environment variable values will take precedence over the central store values.
     def refresh!
       existing_keys = cache.keys
       updated_keys  = []
-      provider.each(path) do |key, value|
-        value      = interpolator.parse(value) if value.is_a?(String) && value.include?('%{')
-        cache[key] = env_var_override(relative_key(key), value)
+      fetch_path(path).each_pair do |key, value|
+        cache[key] = env_var_override(key, value)
         updated_keys << key
       end
 
@@ -98,8 +107,17 @@ module SecretConfig
 
     attr_reader :cache
 
-    def interpolator
-      @interpolator ||= SettingInterpolator.new
+    # Returns [true|false] whether the supplied key is considered a relative key.
+    def relative_key?(key)
+      !key.start_with?("/")
+    end
+
+    # Returns a flat path of keys and values from the provider without looking in the local path.
+    # Keys are returned with path names relative to the supplied path.
+    def fetch_path(path)
+      parser = Parser.new(path, self)
+      provider.each(path) { |key, value| parser.parse(key, value) }
+      parser.render
     end
 
     # Returns the value from an env var if it is present,
@@ -113,12 +131,7 @@ module SecretConfig
 
     # Add the path to the path if it is a relative path.
     def expand_key(key)
-      key.start_with?("/") ? key : "#{path}/#{key}"
-    end
-
-    # Convert the key to a relative path by removing the path.
-    def relative_key(key)
-      key.start_with?("/") ? key.sub("#{path}/", "") : key
+      relative_key?(key) ? "#{path}/#{key}" : key
     end
 
     def filter_value(key, value, filters)
@@ -140,12 +153,12 @@ module SecretConfig
 
     def convert_type(type, value)
       case type
+      when :string
+        value.to_s
       when :integer
         value.to_i
       when :float
         value.to_f
-      when :string
-        value
       when :boolean
         %w[true 1 t].include?(value.to_s.downcase)
       when :symbol
@@ -171,7 +184,7 @@ module SecretConfig
 
       raise(UndefinedRootError, "Either set env var 'SECRET_CONFIG_PATH' or call SecretConfig.use") unless path
 
-      path.start_with?("/") ? path : "/#{path}"
+      relative_key?(path) ? "/#{path}" : path
     end
 
     def default_provider(provider)

data/lib/secret_config/setting_interpolator.rb

@@ -1,6 +1,6 @@
-require 'date'
-require 'socket'
-require 'securerandom'
+require "date"
+require "socket"
+require "securerandom"
 # * SecretConfig Interpolations
 #
 # Expanding values inline for date, time, hostname, pid and random values.
@@ -14,11 +14,6 @@ require 'securerandom'
 #   %{pid}            # Process Id for this process.
 #   %{random}         # URL safe Random 32 byte value.
 #   %{random:size}    # URL safe Random value of `size` bytes.
-#
-# Retrieve values elsewhere in the registry.
-# Paths can be relative to the current root, or absolute paths outside the current root.
-#   %{fetch:key}      # Fetches a single value from a relative or absolute path
-#   %{include:path}   # Fetches a path of keys and values
 module SecretConfig
   class SettingInterpolator < StringInterpolator
     def date(format = "%Y%m%d")
@@ -35,7 +30,7 @@ module SecretConfig
 
     def hostname(format = nil)
       name = Socket.gethostname
-      name = name.split('.')[0] if format == "short"
+      name = name.split(".")[0] if format == "short"
       name
     end
 
@@ -46,13 +41,5 @@ module SecretConfig
     def random(size = 32)
       SecureRandom.urlsafe_base64(size)
     end
-
-    #def fetch(key)
-    #  SecretConfig[key]
-    #end
-    #
-    #def include(path)
-    #
-    #end
   end
 end

data/lib/secret_config/string_interpolator.rb

@@ -16,14 +16,15 @@ module SecretConfig
 
     def parse(string)
       string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
-        if match.start_with?('%%')
+        if match.start_with?("%%")
           match[1..-1]
         else
-          expr          = $1 || $2 || match.tr("%{}", "")
-          key, args_str = expr.split(':')
+          expr          = Regexp.last_match(1) || Regexp.last_match(2) || match.tr("%{}", "")
+          key, args_str = expr.split(":")
           key           = key.strip.to_sym
-          arguments     = args_str&.split(',')&.map { |v| v.strip == '' ? nil : v.strip } || []
+          arguments     = args_str&.split(",")&.map { |v| v.strip == "" ? nil : v.strip } || []
           raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
+
           public_send(key, *arguments)
         end
       end

data/lib/secret_config/utils.rb

@@ -34,7 +34,7 @@ module SecretConfig
         h[key] = value
         return h
       end
-      last       = full_path.split("/").reduce(h) do |target, path|
+      last = full_path.split("/").reduce(h) do |target, path|
         if path == ""
           target
         elsif target.key?(path)

data/lib/secret_config/version.rb

@@ -1,3 +1,3 @@
 module SecretConfig
-  VERSION = "0.7.1".freeze
+  VERSION = "0.8.0".freeze
 end

data/test/config/application.yml

@@ -1,9 +1,5 @@
 # Local application config goes here.  Do not check in production secrets.
 # These are for development and test only.
-
-#
-# Development - Local - Root: '/test/my_application'
-#
 development:
   my_application:
     symmetric_encryption:
@@ -31,12 +27,20 @@ test:
       key:     QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=
       iv:      QUJDREVGMTIzNDU2Nzg5MA==
       version: 2
+      previous_key:
+        key:     key1
+        iv:      iv1
+        version: 1
 
     mysql:
       database:   secret_config_test
       username:   secret_config
       password:   secret_configrules
       host:       127.0.0.1
+      ports:      "12345,5343,26815"
+      ports2:     "    12345, 5343 ,  26815"
+      hostnames:  "primary.example.net,secondary.example.net,backup.example.net"
+      hostnames2: "   primary.example.net,  secondary.example.net ,  backup.example.net"
 
     mongo:
       database:   secret_config_test
@@ -45,3 +49,29 @@ test:
 
     secrets:
       secret_key_base: somereallylongteststring
+
+  other_application:
+    symmetric_encryption:
+      version:    3
+      __import__: /test/my_application/symmetric_encryption
+      iv:         MTIzNDU2Nzg5MEFCQ0RFRg==
+      previous_key:
+        key:     key0
+
+    mysql:
+#      database:   "%{fetch: /test/my_application/mysql/database }"
+      username:   other
+      password:   otherrules
+      host:       "%{hostname}"
+
+    mongo:
+      database:   secret_config_test
+      primary:    localhost:27017
+      secondary:  "%{hostname}:27018"
+
+    mongo2:
+      __import__: mongo
+      database:   secret_config_test2
+
+    mongo3:
+      __import__: mongo

data/test/parser_test.rb

@@ -0,0 +1,82 @@
+require_relative "test_helper"
+require "socket"
+
+class ParserTest < Minitest::Test
+  describe SecretConfig::Registry do
+    let :file_name do
+      File.join(File.dirname(__FILE__), "config", "application.yml")
+    end
+
+    let :path do
+      "/test/other_application"
+    end
+
+    let :provider do
+      SecretConfig::Providers::File.new(file_name: file_name)
+    end
+
+    let :registry do
+      SecretConfig::Registry.new(path: path, provider: provider)
+    end
+
+    # let :parser do
+    #   SecretConfig::Parser.new(path, registry)
+    # end
+
+    #
+    # Retrieve values elsewhere in the registry.
+    # Paths can be relative to the current root, or absolute paths outside the current root.
+    #   %{fetch:key}      # Fetches a single value from a relative or absolute path
+    # Return the value of the supplied key.
+    #
+    # With a relative key, look for the value in the current registry.
+    # With an absolute key call the provider and fetch the value directly.
+    #
+    # Notes:
+    # - A lot of absolute key lookups can be expensive since each one is a separate call.
+    # def fetch(key)
+    #   fetch_list[key] = key
+    # end
+    # describe "#fetch" do
+    #   it "inside current path" do
+    #
+    #   end
+    #
+    #   it "outside current path" do
+    #
+    #   end
+    # end
+
+    #   %{import:path}    # Imports a path of keys and values into the current path
+    # Replace the current value with a tree of values with the supplied path.
+    #
+    describe "#import" do
+      it "removes import key" do
+        refute registry.key?("symmetric_encryption/__import__"), -> { registry.configuration(filters: nil).ai }
+      end
+
+      it "retains overrides" do
+        assert_equal "3", registry["symmetric_encryption/version"], -> { registry.configuration(filters: nil).ai }
+        assert_equal "MTIzNDU2Nzg5MEFCQ0RFRg==", registry["symmetric_encryption/iv"]
+      end
+
+      it "retains child overrides" do
+        assert_equal "key0", registry["symmetric_encryption/previous_key/key"], -> { registry.configuration(filters: nil).ai }
+      end
+
+      it "imports new fields" do
+        assert_equal "QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=", registry["symmetric_encryption/key"]
+      end
+
+      it "relative import empty" do
+        assert_equal "secret_config_test", registry["mongo3/database"]
+        assert_equal "localhost:27017", registry["mongo3/primary"]
+      end
+
+      it "relative import with overrides" do
+        assert_equal "secret_config_test2", registry["mongo2/database"]
+        assert_equal "localhost:27017", registry["mongo3/primary"]
+      end
+    end
+  end
+end