secret_config 0.7.1 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,75 @@
1
+ module SecretConfig
2
+ class Parser
3
+ attr_reader :tree, :path, :registry, :interpolator
4
+
5
+ def initialize(path, registry)
6
+ @path = path
7
+ @registry = registry
8
+ @fetch_list = {}
9
+ @import_list = {}
10
+ @tree = {}
11
+ @interpolator = SettingInterpolator.new
12
+ end
13
+
14
+ # Returns a flat path of keys and values from the provider without looking in the local path.
15
+ # Keys are returned with path names relative to the supplied path.
16
+ def parse(key, value)
17
+ relative_key = relative_key?(key) ? key : key.sub("#{path}/", "")
18
+ tree[relative_key] = value.is_a?(String) && value.include?("%{") ? interpolator.parse(value) : value
19
+ end
20
+
21
+ # Returns a flat Hash of the rendered paths.
22
+ def render
23
+ # apply_fetches
24
+ apply_imports
25
+ tree
26
+ end
27
+
28
+ private
29
+
30
+ # def apply_fetches
31
+ # tree[key] = relative_key?(fetch_key) ? registry[fetch_key] : registry.provider.fetch(fetch_key)
32
+ # end
33
+
34
+ # Import from the current registry as well as new fetches.
35
+ #
36
+ # Notes:
37
+ # - A lot of absolute key lookups can be expensive since each one is a separate call.
38
+ # - Imports cannot reference other imports at this time.
39
+ def apply_imports
40
+ tree.keys.each do |key|
41
+ next unless key =~ /\/__import__\Z/
42
+
43
+ import_key = tree.delete(key)
44
+ key, _ = ::File.split(key)
45
+
46
+ # binding.irb
47
+
48
+ # With a relative key, look for the values in the current registry.
49
+ # With an absolute key call the provider and fetch the value directly.
50
+
51
+ if relative_key?(import_key)
52
+ tree.keys.each do |current_key|
53
+ match = current_key.match(/\A#{import_key}\/(.*)/)
54
+ next unless match
55
+
56
+ imported_key = ::File.join(key, match[1])
57
+ tree[imported_key] = tree[current_key] unless tree.key?(imported_key)
58
+ end
59
+ else
60
+ relative_paths = registry.send(:fetch_path, import_key)
61
+ relative_paths.each_pair do |relative_key, value|
62
+ imported_key = ::File.join(key, relative_key)
63
+ tree[imported_key] = value unless tree.key?(imported_key)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ # Returns [true|false] whether the supplied key is considered a relative key.
70
+ def relative_key?(key)
71
+ !key.start_with?("/")
72
+ end
73
+
74
+ end
75
+ end
@@ -12,17 +12,30 @@ module SecretConfig
12
12
  raise(ConfigurationError, "Cannot find config file: #{file_name}") unless ::File.exist?(file_name)
13
13
  end
14
14
 
15
+ # Yields the key with its absolute path and corresponding string value
15
16
  def each(path, &block)
16
- config = YAML.load(ERB.new(::File.new(file_name).read).result)
17
-
18
- paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
19
- settings = config.dig(*paths)
17
+ settings = fetch_path(path)
20
18
 
21
19
  raise(ConfigurationError, "Path #{paths.join('/')} not found in file: #{file_name}") unless settings
22
20
 
23
21
  Utils.flatten_each(settings, path, &block)
24
22
  nil
25
23
  end
24
+
25
+ # Returns the value or `nil` if not found
26
+ def fetch(key)
27
+ values = fetch_path(path)
28
+ value.is_a?(Hash) ? nil : value
29
+ end
30
+
31
+ private
32
+
33
+ def fetch_path(path)
34
+ config = YAML.load(ERB.new(::File.new(file_name).read).result)
35
+
36
+ paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
37
+ config.dig(*paths)
38
+ end
26
39
  end
27
40
  end
28
41
  end
@@ -10,7 +10,13 @@ module SecretConfig
10
10
  class Ssm < Provider
11
11
  attr_reader :client, :key_id, :retry_count, :retry_max_ms, :logger
12
12
 
13
- def initialize(key_id: ENV["AWS_ACCESS_KEY_ID"], key_alias: ENV["AWS_ACCESS_KEY_ALIAS"], retry_count: 10, retry_max_ms: 3_000)
13
+ def initialize(
14
+ key_id: ENV["SECRET_CONFIG_KEY_ID"],
15
+ key_alias: ENV["SECRET_CONFIG_KEY_ALIAS"],
16
+ retry_count: 25,
17
+ retry_max_ms: 10_000,
18
+ **args
19
+ )
14
20
  @key_id =
15
21
  if key_alias
16
22
  key_alias =~ %r{^alias/} ? key_alias : "alias/#{key_alias}"
@@ -20,9 +26,10 @@ module SecretConfig
20
26
  @retry_count = retry_count
21
27
  @retry_max_ms = retry_max_ms
22
28
  @logger = SemanticLogger["Aws::SSM"] if defined?(SemanticLogger)
23
- @client = Aws::SSM::Client.new(logger: logger)
29
+ @client = Aws::SSM::Client.new({logger: logger}.merge!(args))
24
30
  end
25
31
 
32
+ # Yields the key with its absolute path and corresponding string value
26
33
  def each(path)
27
34
  retries = 0
28
35
  token = nil
@@ -20,10 +20,14 @@ module SecretConfig
20
20
  end
21
21
 
22
22
  # Returns [Hash] a copy of the in memory configuration data.
23
- def configuration(relative: true, filters: SecretConfig.filters)
23
+ #
24
+ # Supply the relative path to start from so that only keys and values in that
25
+ # path will be returned.
26
+ def configuration(path: nil, filters: SecretConfig.filters)
24
27
  h = {}
25
28
  cache.each_pair do |key, value|
26
- key = relative_key(key) if relative
29
+ next if path && !key.start_with?(path)
30
+
27
31
  value = filter_value(key, value, filters)
28
32
  Utils.decompose(key, value, h)
29
33
  end
@@ -32,30 +36,36 @@ module SecretConfig
32
36
 
33
37
  # Returns [String] configuration value for the supplied key, or nil when missing.
34
38
  def [](key)
35
- full_key = expand_key(key)
36
- value = cache[full_key]
39
+ value = cache[key]
37
40
  if value.nil? && SecretConfig.check_env_var?
38
- value = env_var_override(key, value)
39
- cache[full_key] = value unless value.nil?
41
+ value = env_var_override(key, value)
42
+ cache[key] = value unless value.nil?
40
43
  end
41
- value
44
+ value.nil? ? nil : value.to_s
42
45
  end
43
46
 
44
47
  # Returns [String] configuration value for the supplied key, or nil when missing.
45
48
  def key?(key)
46
- cache.key?(expand_key(key))
49
+ cache.key?(key)
47
50
  end
48
51
 
49
52
  # Returns [String] configuration value for the supplied key
50
- def fetch(key, default: :no_default_supplied, type: :string, encoding: nil)
53
+ # Convert the string value into an array of values by supplying a `separator`.
54
+ def fetch(key, default: :no_default_supplied, type: :string, encoding: nil, separator: nil)
51
55
  value = self[key]
52
56
  if value.nil?
53
57
  raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
58
+
54
59
  value = block_given? ? yield : default
55
60
  end
56
61
 
57
62
  value = convert_encoding(encoding, value) if encoding
58
- type == :string ? value : convert_type(type, value)
63
+
64
+ if separator
65
+ value.to_s.split(separator).collect { |element| convert_type(type, element.strip) }
66
+ else
67
+ convert_type(type, value)
68
+ end
59
69
  end
60
70
 
61
71
  # Set the value for a key in the centralized configuration store.
@@ -65,26 +75,25 @@ module SecretConfig
65
75
 
66
76
  # Set the value for a key in the centralized configuration store.
67
77
  def set(key, value)
68
- key = expand_key(key)
69
- provider.set(key, value)
78
+ full_key = expand_key(key)
79
+ provider.set(full_key, value)
70
80
  cache[key] = value
71
81
  end
72
82
 
73
83
  # Delete a key from the centralized configuration store.
74
84
  def delete(key)
75
- key = expand_key(key)
76
- provider.delete(key)
85
+ full_key = expand_key(key)
86
+ provider.delete(full_key)
77
87
  cache.delete(key)
78
88
  end
79
89
 
80
90
  # Refresh the in-memory cached copy of the centralized configuration information.
81
- # Environment variable values will take precendence over the central store values.
91
+ # Environment variable values will take precedence over the central store values.
82
92
  def refresh!
83
93
  existing_keys = cache.keys
84
94
  updated_keys = []
85
- provider.each(path) do |key, value|
86
- value = interpolator.parse(value) if value.is_a?(String) && value.include?('%{')
87
- cache[key] = env_var_override(relative_key(key), value)
95
+ fetch_path(path).each_pair do |key, value|
96
+ cache[key] = env_var_override(key, value)
88
97
  updated_keys << key
89
98
  end
90
99
 
@@ -98,8 +107,17 @@ module SecretConfig
98
107
 
99
108
  attr_reader :cache
100
109
 
101
- def interpolator
102
- @interpolator ||= SettingInterpolator.new
110
+ # Returns [true|false] whether the supplied key is considered a relative key.
111
+ def relative_key?(key)
112
+ !key.start_with?("/")
113
+ end
114
+
115
+ # Returns a flat path of keys and values from the provider without looking in the local path.
116
+ # Keys are returned with path names relative to the supplied path.
117
+ def fetch_path(path)
118
+ parser = Parser.new(path, self)
119
+ provider.each(path) { |key, value| parser.parse(key, value) }
120
+ parser.render
103
121
  end
104
122
 
105
123
  # Returns the value from an env var if it is present,
@@ -113,12 +131,7 @@ module SecretConfig
113
131
 
114
132
  # Add the path to the path if it is a relative path.
115
133
  def expand_key(key)
116
- key.start_with?("/") ? key : "#{path}/#{key}"
117
- end
118
-
119
- # Convert the key to a relative path by removing the path.
120
- def relative_key(key)
121
- key.start_with?("/") ? key.sub("#{path}/", "") : key
134
+ relative_key?(key) ? "#{path}/#{key}" : key
122
135
  end
123
136
 
124
137
  def filter_value(key, value, filters)
@@ -140,12 +153,12 @@ module SecretConfig
140
153
 
141
154
  def convert_type(type, value)
142
155
  case type
156
+ when :string
157
+ value.to_s
143
158
  when :integer
144
159
  value.to_i
145
160
  when :float
146
161
  value.to_f
147
- when :string
148
- value
149
162
  when :boolean
150
163
  %w[true 1 t].include?(value.to_s.downcase)
151
164
  when :symbol
@@ -171,7 +184,7 @@ module SecretConfig
171
184
 
172
185
  raise(UndefinedRootError, "Either set env var 'SECRET_CONFIG_PATH' or call SecretConfig.use") unless path
173
186
 
174
- path.start_with?("/") ? path : "/#{path}"
187
+ relative_key?(path) ? "/#{path}" : path
175
188
  end
176
189
 
177
190
  def default_provider(provider)
@@ -1,6 +1,6 @@
1
- require 'date'
2
- require 'socket'
3
- require 'securerandom'
1
+ require "date"
2
+ require "socket"
3
+ require "securerandom"
4
4
  # * SecretConfig Interpolations
5
5
  #
6
6
  # Expanding values inline for date, time, hostname, pid and random values.
@@ -14,11 +14,6 @@ require 'securerandom'
14
14
  # %{pid} # Process Id for this process.
15
15
  # %{random} # URL safe Random 32 byte value.
16
16
  # %{random:size} # URL safe Random value of `size` bytes.
17
- #
18
- # Retrieve values elsewhere in the registry.
19
- # Paths can be relative to the current root, or absolute paths outside the current root.
20
- # %{fetch:key} # Fetches a single value from a relative or absolute path
21
- # %{include:path} # Fetches a path of keys and values
22
17
  module SecretConfig
23
18
  class SettingInterpolator < StringInterpolator
24
19
  def date(format = "%Y%m%d")
@@ -35,7 +30,7 @@ module SecretConfig
35
30
 
36
31
  def hostname(format = nil)
37
32
  name = Socket.gethostname
38
- name = name.split('.')[0] if format == "short"
33
+ name = name.split(".")[0] if format == "short"
39
34
  name
40
35
  end
41
36
 
@@ -46,13 +41,5 @@ module SecretConfig
46
41
  def random(size = 32)
47
42
  SecureRandom.urlsafe_base64(size)
48
43
  end
49
-
50
- #def fetch(key)
51
- # SecretConfig[key]
52
- #end
53
- #
54
- #def include(path)
55
- #
56
- #end
57
44
  end
58
45
  end
@@ -16,14 +16,15 @@ module SecretConfig
16
16
 
17
17
  def parse(string)
18
18
  string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
19
- if match.start_with?('%%')
19
+ if match.start_with?("%%")
20
20
  match[1..-1]
21
21
  else
22
- expr = $1 || $2 || match.tr("%{}", "")
23
- key, args_str = expr.split(':')
22
+ expr = Regexp.last_match(1) || Regexp.last_match(2) || match.tr("%{}", "")
23
+ key, args_str = expr.split(":")
24
24
  key = key.strip.to_sym
25
- arguments = args_str&.split(',')&.map { |v| v.strip == '' ? nil : v.strip } || []
25
+ arguments = args_str&.split(",")&.map { |v| v.strip == "" ? nil : v.strip } || []
26
26
  raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
27
+
27
28
  public_send(key, *arguments)
28
29
  end
29
30
  end
@@ -34,7 +34,7 @@ module SecretConfig
34
34
  h[key] = value
35
35
  return h
36
36
  end
37
- last = full_path.split("/").reduce(h) do |target, path|
37
+ last = full_path.split("/").reduce(h) do |target, path|
38
38
  if path == ""
39
39
  target
40
40
  elsif target.key?(path)
@@ -1,3 +1,3 @@
1
1
  module SecretConfig
2
- VERSION = "0.7.1".freeze
2
+ VERSION = "0.8.0".freeze
3
3
  end
@@ -1,9 +1,5 @@
1
1
  # Local application config goes here. Do not check in production secrets.
2
2
  # These are for development and test only.
3
-
4
- #
5
- # Development - Local - Root: '/test/my_application'
6
- #
7
3
  development:
8
4
  my_application:
9
5
  symmetric_encryption:
@@ -31,12 +27,20 @@ test:
31
27
  key: QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=
32
28
  iv: QUJDREVGMTIzNDU2Nzg5MA==
33
29
  version: 2
30
+ previous_key:
31
+ key: key1
32
+ iv: iv1
33
+ version: 1
34
34
 
35
35
  mysql:
36
36
  database: secret_config_test
37
37
  username: secret_config
38
38
  password: secret_configrules
39
39
  host: 127.0.0.1
40
+ ports: "12345,5343,26815"
41
+ ports2: " 12345, 5343 , 26815"
42
+ hostnames: "primary.example.net,secondary.example.net,backup.example.net"
43
+ hostnames2: " primary.example.net, secondary.example.net , backup.example.net"
40
44
 
41
45
  mongo:
42
46
  database: secret_config_test
@@ -45,3 +49,29 @@ test:
45
49
 
46
50
  secrets:
47
51
  secret_key_base: somereallylongteststring
52
+
53
+ other_application:
54
+ symmetric_encryption:
55
+ version: 3
56
+ __import__: /test/my_application/symmetric_encryption
57
+ iv: MTIzNDU2Nzg5MEFCQ0RFRg==
58
+ previous_key:
59
+ key: key0
60
+
61
+ mysql:
62
+ # database: "%{fetch: /test/my_application/mysql/database }"
63
+ username: other
64
+ password: otherrules
65
+ host: "%{hostname}"
66
+
67
+ mongo:
68
+ database: secret_config_test
69
+ primary: localhost:27017
70
+ secondary: "%{hostname}:27018"
71
+
72
+ mongo2:
73
+ __import__: mongo
74
+ database: secret_config_test2
75
+
76
+ mongo3:
77
+ __import__: mongo
@@ -0,0 +1,82 @@
1
+ require_relative "test_helper"
2
+ require "socket"
3
+
4
+ class ParserTest < Minitest::Test
5
+ describe SecretConfig::Registry do
6
+ let :file_name do
7
+ File.join(File.dirname(__FILE__), "config", "application.yml")
8
+ end
9
+
10
+ let :path do
11
+ "/test/other_application"
12
+ end
13
+
14
+ let :provider do
15
+ SecretConfig::Providers::File.new(file_name: file_name)
16
+ end
17
+
18
+ let :registry do
19
+ SecretConfig::Registry.new(path: path, provider: provider)
20
+ end
21
+
22
+ # let :parser do
23
+ # SecretConfig::Parser.new(path, registry)
24
+ # end
25
+
26
+ #
27
+ # Retrieve values elsewhere in the registry.
28
+ # Paths can be relative to the current root, or absolute paths outside the current root.
29
+ # %{fetch:key} # Fetches a single value from a relative or absolute path
30
+ # Return the value of the supplied key.
31
+ #
32
+ # With a relative key, look for the value in the current registry.
33
+ # With an absolute key call the provider and fetch the value directly.
34
+ #
35
+ # Notes:
36
+ # - A lot of absolute key lookups can be expensive since each one is a separate call.
37
+ # def fetch(key)
38
+ # fetch_list[key] = key
39
+ # end
40
+ # describe "#fetch" do
41
+ # it "inside current path" do
42
+ #
43
+ # end
44
+ #
45
+ # it "outside current path" do
46
+ #
47
+ # end
48
+ # end
49
+
50
+ # %{import:path} # Imports a path of keys and values into the current path
51
+ # Replace the current value with a tree of values with the supplied path.
52
+ #
53
+ describe "#import" do
54
+ it "removes import key" do
55
+ refute registry.key?("symmetric_encryption/__import__"), -> { registry.configuration(filters: nil).ai }
56
+ end
57
+
58
+ it "retains overrides" do
59
+ assert_equal "3", registry["symmetric_encryption/version"], -> { registry.configuration(filters: nil).ai }
60
+ assert_equal "MTIzNDU2Nzg5MEFCQ0RFRg==", registry["symmetric_encryption/iv"]
61
+ end
62
+
63
+ it "retains child overrides" do
64
+ assert_equal "key0", registry["symmetric_encryption/previous_key/key"], -> { registry.configuration(filters: nil).ai }
65
+ end
66
+
67
+ it "imports new fields" do
68
+ assert_equal "QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=", registry["symmetric_encryption/key"]
69
+ end
70
+
71
+ it "relative import empty" do
72
+ assert_equal "secret_config_test", registry["mongo3/database"]
73
+ assert_equal "localhost:27017", registry["mongo3/primary"]
74
+ end
75
+
76
+ it "relative import with overrides" do
77
+ assert_equal "secret_config_test2", registry["mongo2/database"]
78
+ assert_equal "localhost:27017", registry["mongo3/primary"]
79
+ end
80
+ end
81
+ end
82
+ end