secret_config 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,75 @@
1
+ module SecretConfig
2
+ class Parser
3
+ attr_reader :tree, :path, :registry, :interpolator
4
+
5
+ def initialize(path, registry)
6
+ @path = path
7
+ @registry = registry
8
+ @fetch_list = {}
9
+ @import_list = {}
10
+ @tree = {}
11
+ @interpolator = SettingInterpolator.new
12
+ end
13
+
14
+ # Returns a flat path of keys and values from the provider without looking in the local path.
15
+ # Keys are returned with path names relative to the supplied path.
16
+ def parse(key, value)
17
+ relative_key = relative_key?(key) ? key : key.sub("#{path}/", "")
18
+ tree[relative_key] = value.is_a?(String) && value.include?("%{") ? interpolator.parse(value) : value
19
+ end
20
+
21
+ # Returns a flat Hash of the rendered paths.
22
+ def render
23
+ # apply_fetches
24
+ apply_imports
25
+ tree
26
+ end
27
+
28
+ private
29
+
30
+ # def apply_fetches
31
+ # tree[key] = relative_key?(fetch_key) ? registry[fetch_key] : registry.provider.fetch(fetch_key)
32
+ # end
33
+
34
+ # Import from the current registry as well as new fetches.
35
+ #
36
+ # Notes:
37
+ # - A lot of absolute key lookups can be expensive since each one is a separate call.
38
+ # - Imports cannot reference other imports at this time.
39
+ def apply_imports
40
+ tree.keys.each do |key|
41
+ next unless key =~ /\/__import__\Z/
42
+
43
+ import_key = tree.delete(key)
44
+ key, _ = ::File.split(key)
45
+
46
+ # binding.irb
47
+
48
+ # With a relative key, look for the values in the current registry.
49
+ # With an absolute key call the provider and fetch the value directly.
50
+
51
+ if relative_key?(import_key)
52
+ tree.keys.each do |current_key|
53
+ match = current_key.match(/\A#{import_key}\/(.*)/)
54
+ next unless match
55
+
56
+ imported_key = ::File.join(key, match[1])
57
+ tree[imported_key] = tree[current_key] unless tree.key?(imported_key)
58
+ end
59
+ else
60
+ relative_paths = registry.send(:fetch_path, import_key)
61
+ relative_paths.each_pair do |relative_key, value|
62
+ imported_key = ::File.join(key, relative_key)
63
+ tree[imported_key] = value unless tree.key?(imported_key)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ # Returns [true|false] whether the supplied key is considered a relative key.
70
+ def relative_key?(key)
71
+ !key.start_with?("/")
72
+ end
73
+
74
+ end
75
+ end
@@ -12,17 +12,30 @@ module SecretConfig
12
12
  raise(ConfigurationError, "Cannot find config file: #{file_name}") unless ::File.exist?(file_name)
13
13
  end
14
14
 
15
+ # Yields the key with its absolute path and corresponding string value
15
16
  def each(path, &block)
16
- config = YAML.load(ERB.new(::File.new(file_name).read).result)
17
-
18
- paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
19
- settings = config.dig(*paths)
17
+ settings = fetch_path(path)
20
18
 
21
19
  raise(ConfigurationError, "Path #{paths.join('/')} not found in file: #{file_name}") unless settings
22
20
 
23
21
  Utils.flatten_each(settings, path, &block)
24
22
  nil
25
23
  end
24
+
25
+ # Returns the value or `nil` if not found
26
+ def fetch(key)
27
+ values = fetch_path(path)
28
+ value.is_a?(Hash) ? nil : value
29
+ end
30
+
31
+ private
32
+
33
+ def fetch_path(path)
34
+ config = YAML.load(ERB.new(::File.new(file_name).read).result)
35
+
36
+ paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
37
+ config.dig(*paths)
38
+ end
26
39
  end
27
40
  end
28
41
  end
@@ -10,7 +10,13 @@ module SecretConfig
10
10
  class Ssm < Provider
11
11
  attr_reader :client, :key_id, :retry_count, :retry_max_ms, :logger
12
12
 
13
- def initialize(key_id: ENV["AWS_ACCESS_KEY_ID"], key_alias: ENV["AWS_ACCESS_KEY_ALIAS"], retry_count: 10, retry_max_ms: 3_000)
13
+ def initialize(
14
+ key_id: ENV["SECRET_CONFIG_KEY_ID"],
15
+ key_alias: ENV["SECRET_CONFIG_KEY_ALIAS"],
16
+ retry_count: 25,
17
+ retry_max_ms: 10_000,
18
+ **args
19
+ )
14
20
  @key_id =
15
21
  if key_alias
16
22
  key_alias =~ %r{^alias/} ? key_alias : "alias/#{key_alias}"
@@ -20,9 +26,10 @@ module SecretConfig
20
26
  @retry_count = retry_count
21
27
  @retry_max_ms = retry_max_ms
22
28
  @logger = SemanticLogger["Aws::SSM"] if defined?(SemanticLogger)
23
- @client = Aws::SSM::Client.new(logger: logger)
29
+ @client = Aws::SSM::Client.new({logger: logger}.merge!(args))
24
30
  end
25
31
 
32
+ # Yields the key with its absolute path and corresponding string value
26
33
  def each(path)
27
34
  retries = 0
28
35
  token = nil
@@ -20,10 +20,14 @@ module SecretConfig
20
20
  end
21
21
 
22
22
  # Returns [Hash] a copy of the in memory configuration data.
23
- def configuration(relative: true, filters: SecretConfig.filters)
23
+ #
24
+ # Supply the relative path to start from so that only keys and values in that
25
+ # path will be returned.
26
+ def configuration(path: nil, filters: SecretConfig.filters)
24
27
  h = {}
25
28
  cache.each_pair do |key, value|
26
- key = relative_key(key) if relative
29
+ next if path && !key.start_with?(path)
30
+
27
31
  value = filter_value(key, value, filters)
28
32
  Utils.decompose(key, value, h)
29
33
  end
@@ -32,30 +36,36 @@ module SecretConfig
32
36
 
33
37
  # Returns [String] configuration value for the supplied key, or nil when missing.
34
38
  def [](key)
35
- full_key = expand_key(key)
36
- value = cache[full_key]
39
+ value = cache[key]
37
40
  if value.nil? && SecretConfig.check_env_var?
38
- value = env_var_override(key, value)
39
- cache[full_key] = value unless value.nil?
41
+ value = env_var_override(key, value)
42
+ cache[key] = value unless value.nil?
40
43
  end
41
- value
44
+ value.nil? ? nil : value.to_s
42
45
  end
43
46
 
44
47
  # Returns [String] configuration value for the supplied key, or nil when missing.
45
48
  def key?(key)
46
- cache.key?(expand_key(key))
49
+ cache.key?(key)
47
50
  end
48
51
 
49
52
  # Returns [String] configuration value for the supplied key
50
- def fetch(key, default: :no_default_supplied, type: :string, encoding: nil)
53
+ # Convert the string value into an array of values by supplying a `separator`.
54
+ def fetch(key, default: :no_default_supplied, type: :string, encoding: nil, separator: nil)
51
55
  value = self[key]
52
56
  if value.nil?
53
57
  raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
58
+
54
59
  value = block_given? ? yield : default
55
60
  end
56
61
 
57
62
  value = convert_encoding(encoding, value) if encoding
58
- type == :string ? value : convert_type(type, value)
63
+
64
+ if separator
65
+ value.to_s.split(separator).collect { |element| convert_type(type, element.strip) }
66
+ else
67
+ convert_type(type, value)
68
+ end
59
69
  end
60
70
 
61
71
  # Set the value for a key in the centralized configuration store.
@@ -65,26 +75,25 @@ module SecretConfig
65
75
 
66
76
  # Set the value for a key in the centralized configuration store.
67
77
  def set(key, value)
68
- key = expand_key(key)
69
- provider.set(key, value)
78
+ full_key = expand_key(key)
79
+ provider.set(full_key, value)
70
80
  cache[key] = value
71
81
  end
72
82
 
73
83
  # Delete a key from the centralized configuration store.
74
84
  def delete(key)
75
- key = expand_key(key)
76
- provider.delete(key)
85
+ full_key = expand_key(key)
86
+ provider.delete(full_key)
77
87
  cache.delete(key)
78
88
  end
79
89
 
80
90
  # Refresh the in-memory cached copy of the centralized configuration information.
81
- # Environment variable values will take precendence over the central store values.
91
+ # Environment variable values will take precedence over the central store values.
82
92
  def refresh!
83
93
  existing_keys = cache.keys
84
94
  updated_keys = []
85
- provider.each(path) do |key, value|
86
- value = interpolator.parse(value) if value.is_a?(String) && value.include?('%{')
87
- cache[key] = env_var_override(relative_key(key), value)
95
+ fetch_path(path).each_pair do |key, value|
96
+ cache[key] = env_var_override(key, value)
88
97
  updated_keys << key
89
98
  end
90
99
 
@@ -98,8 +107,17 @@ module SecretConfig
98
107
 
99
108
  attr_reader :cache
100
109
 
101
- def interpolator
102
- @interpolator ||= SettingInterpolator.new
110
+ # Returns [true|false] whether the supplied key is considered a relative key.
111
+ def relative_key?(key)
112
+ !key.start_with?("/")
113
+ end
114
+
115
+ # Returns a flat path of keys and values from the provider without looking in the local path.
116
+ # Keys are returned with path names relative to the supplied path.
117
+ def fetch_path(path)
118
+ parser = Parser.new(path, self)
119
+ provider.each(path) { |key, value| parser.parse(key, value) }
120
+ parser.render
103
121
  end
104
122
 
105
123
  # Returns the value from an env var if it is present,
@@ -113,12 +131,7 @@ module SecretConfig
113
131
 
114
132
  # Add the path to the path if it is a relative path.
115
133
  def expand_key(key)
116
- key.start_with?("/") ? key : "#{path}/#{key}"
117
- end
118
-
119
- # Convert the key to a relative path by removing the path.
120
- def relative_key(key)
121
- key.start_with?("/") ? key.sub("#{path}/", "") : key
134
+ relative_key?(key) ? "#{path}/#{key}" : key
122
135
  end
123
136
 
124
137
  def filter_value(key, value, filters)
@@ -140,12 +153,12 @@ module SecretConfig
140
153
 
141
154
  def convert_type(type, value)
142
155
  case type
156
+ when :string
157
+ value.to_s
143
158
  when :integer
144
159
  value.to_i
145
160
  when :float
146
161
  value.to_f
147
- when :string
148
- value
149
162
  when :boolean
150
163
  %w[true 1 t].include?(value.to_s.downcase)
151
164
  when :symbol
@@ -171,7 +184,7 @@ module SecretConfig
171
184
 
172
185
  raise(UndefinedRootError, "Either set env var 'SECRET_CONFIG_PATH' or call SecretConfig.use") unless path
173
186
 
174
- path.start_with?("/") ? path : "/#{path}"
187
+ relative_key?(path) ? "/#{path}" : path
175
188
  end
176
189
 
177
190
  def default_provider(provider)
@@ -1,6 +1,6 @@
1
- require 'date'
2
- require 'socket'
3
- require 'securerandom'
1
+ require "date"
2
+ require "socket"
3
+ require "securerandom"
4
4
  # * SecretConfig Interpolations
5
5
  #
6
6
  # Expanding values inline for date, time, hostname, pid and random values.
@@ -14,11 +14,6 @@ require 'securerandom'
14
14
  # %{pid} # Process Id for this process.
15
15
  # %{random} # URL safe Random 32 byte value.
16
16
  # %{random:size} # URL safe Random value of `size` bytes.
17
- #
18
- # Retrieve values elsewhere in the registry.
19
- # Paths can be relative to the current root, or absolute paths outside the current root.
20
- # %{fetch:key} # Fetches a single value from a relative or absolute path
21
- # %{include:path} # Fetches a path of keys and values
22
17
  module SecretConfig
23
18
  class SettingInterpolator < StringInterpolator
24
19
  def date(format = "%Y%m%d")
@@ -35,7 +30,7 @@ module SecretConfig
35
30
 
36
31
  def hostname(format = nil)
37
32
  name = Socket.gethostname
38
- name = name.split('.')[0] if format == "short"
33
+ name = name.split(".")[0] if format == "short"
39
34
  name
40
35
  end
41
36
 
@@ -46,13 +41,5 @@ module SecretConfig
46
41
  def random(size = 32)
47
42
  SecureRandom.urlsafe_base64(size)
48
43
  end
49
-
50
- #def fetch(key)
51
- # SecretConfig[key]
52
- #end
53
- #
54
- #def include(path)
55
- #
56
- #end
57
44
  end
58
45
  end
@@ -16,14 +16,15 @@ module SecretConfig
16
16
 
17
17
  def parse(string)
18
18
  string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
19
- if match.start_with?('%%')
19
+ if match.start_with?("%%")
20
20
  match[1..-1]
21
21
  else
22
- expr = $1 || $2 || match.tr("%{}", "")
23
- key, args_str = expr.split(':')
22
+ expr = Regexp.last_match(1) || Regexp.last_match(2) || match.tr("%{}", "")
23
+ key, args_str = expr.split(":")
24
24
  key = key.strip.to_sym
25
- arguments = args_str&.split(',')&.map { |v| v.strip == '' ? nil : v.strip } || []
25
+ arguments = args_str&.split(",")&.map { |v| v.strip == "" ? nil : v.strip } || []
26
26
  raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
27
+
27
28
  public_send(key, *arguments)
28
29
  end
29
30
  end
@@ -34,7 +34,7 @@ module SecretConfig
34
34
  h[key] = value
35
35
  return h
36
36
  end
37
- last = full_path.split("/").reduce(h) do |target, path|
37
+ last = full_path.split("/").reduce(h) do |target, path|
38
38
  if path == ""
39
39
  target
40
40
  elsif target.key?(path)
@@ -1,3 +1,3 @@
1
1
  module SecretConfig
2
- VERSION = "0.7.1".freeze
2
+ VERSION = "0.8.0".freeze
3
3
  end
@@ -1,9 +1,5 @@
1
1
  # Local application config goes here. Do not check in production secrets.
2
2
  # These are for development and test only.
3
-
4
- #
5
- # Development - Local - Root: '/test/my_application'
6
- #
7
3
  development:
8
4
  my_application:
9
5
  symmetric_encryption:
@@ -31,12 +27,20 @@ test:
31
27
  key: QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=
32
28
  iv: QUJDREVGMTIzNDU2Nzg5MA==
33
29
  version: 2
30
+ previous_key:
31
+ key: key1
32
+ iv: iv1
33
+ version: 1
34
34
 
35
35
  mysql:
36
36
  database: secret_config_test
37
37
  username: secret_config
38
38
  password: secret_configrules
39
39
  host: 127.0.0.1
40
+ ports: "12345,5343,26815"
41
+ ports2: " 12345, 5343 , 26815"
42
+ hostnames: "primary.example.net,secondary.example.net,backup.example.net"
43
+ hostnames2: " primary.example.net, secondary.example.net , backup.example.net"
40
44
 
41
45
  mongo:
42
46
  database: secret_config_test
@@ -45,3 +49,29 @@ test:
45
49
 
46
50
  secrets:
47
51
  secret_key_base: somereallylongteststring
52
+
53
+ other_application:
54
+ symmetric_encryption:
55
+ version: 3
56
+ __import__: /test/my_application/symmetric_encryption
57
+ iv: MTIzNDU2Nzg5MEFCQ0RFRg==
58
+ previous_key:
59
+ key: key0
60
+
61
+ mysql:
62
+ # database: "%{fetch: /test/my_application/mysql/database }"
63
+ username: other
64
+ password: otherrules
65
+ host: "%{hostname}"
66
+
67
+ mongo:
68
+ database: secret_config_test
69
+ primary: localhost:27017
70
+ secondary: "%{hostname}:27018"
71
+
72
+ mongo2:
73
+ __import__: mongo
74
+ database: secret_config_test2
75
+
76
+ mongo3:
77
+ __import__: mongo
@@ -0,0 +1,82 @@
1
+ require_relative "test_helper"
2
+ require "socket"
3
+
4
+ class ParserTest < Minitest::Test
5
+ describe SecretConfig::Registry do
6
+ let :file_name do
7
+ File.join(File.dirname(__FILE__), "config", "application.yml")
8
+ end
9
+
10
+ let :path do
11
+ "/test/other_application"
12
+ end
13
+
14
+ let :provider do
15
+ SecretConfig::Providers::File.new(file_name: file_name)
16
+ end
17
+
18
+ let :registry do
19
+ SecretConfig::Registry.new(path: path, provider: provider)
20
+ end
21
+
22
+ # let :parser do
23
+ # SecretConfig::Parser.new(path, registry)
24
+ # end
25
+
26
+ #
27
+ # Retrieve values elsewhere in the registry.
28
+ # Paths can be relative to the current root, or absolute paths outside the current root.
29
+ # %{fetch:key} # Fetches a single value from a relative or absolute path
30
+ # Return the value of the supplied key.
31
+ #
32
+ # With a relative key, look for the value in the current registry.
33
+ # With an absolute key call the provider and fetch the value directly.
34
+ #
35
+ # Notes:
36
+ # - A lot of absolute key lookups can be expensive since each one is a separate call.
37
+ # def fetch(key)
38
+ # fetch_list[key] = key
39
+ # end
40
+ # describe "#fetch" do
41
+ # it "inside current path" do
42
+ #
43
+ # end
44
+ #
45
+ # it "outside current path" do
46
+ #
47
+ # end
48
+ # end
49
+
50
+ # %{import:path} # Imports a path of keys and values into the current path
51
+ # Replace the current value with a tree of values with the supplied path.
52
+ #
53
+ describe "#import" do
54
+ it "removes import key" do
55
+ refute registry.key?("symmetric_encryption/__import__"), -> { registry.configuration(filters: nil).ai }
56
+ end
57
+
58
+ it "retains overrides" do
59
+ assert_equal "3", registry["symmetric_encryption/version"], -> { registry.configuration(filters: nil).ai }
60
+ assert_equal "MTIzNDU2Nzg5MEFCQ0RFRg==", registry["symmetric_encryption/iv"]
61
+ end
62
+
63
+ it "retains child overrides" do
64
+ assert_equal "key0", registry["symmetric_encryption/previous_key/key"], -> { registry.configuration(filters: nil).ai }
65
+ end
66
+
67
+ it "imports new fields" do
68
+ assert_equal "QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=", registry["symmetric_encryption/key"]
69
+ end
70
+
71
+ it "relative import empty" do
72
+ assert_equal "secret_config_test", registry["mongo3/database"]
73
+ assert_equal "localhost:27017", registry["mongo3/primary"]
74
+ end
75
+
76
+ it "relative import with overrides" do
77
+ assert_equal "secret_config_test2", registry["mongo2/database"]
78
+ assert_equal "localhost:27017", registry["mongo3/primary"]
79
+ end
80
+ end
81
+ end
82
+ end