secret_config 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +84 -42
- data/Rakefile +7 -7
- data/bin/secret_config +1 -1
- data/lib/secret_config.rb +30 -1
- data/lib/secret_config/config.rb +44 -0
- data/lib/secret_config/parser.rb +75 -0
- data/lib/secret_config/providers/file.rb +17 -4
- data/lib/secret_config/providers/ssm.rb +9 -2
- data/lib/secret_config/registry.rb +42 -29
- data/lib/secret_config/setting_interpolator.rb +4 -17
- data/lib/secret_config/string_interpolator.rb +5 -4
- data/lib/secret_config/utils.rb +1 -1
- data/lib/secret_config/version.rb +1 -1
- data/test/config/application.yml +34 -4
- data/test/parser_test.rb +82 -0
- data/test/providers/file_test.rb +4 -4
- data/test/providers/ssm_test.rb +37 -12
- data/test/registry_test.rb +51 -26
- data/test/secret_config_test.rb +35 -4
- data/test/setting_interpolator_test.rb +17 -17
- data/test/test_helper.rb +6 -6
- data/test/utils_test.rb +4 -4
- metadata +7 -3
@@ -0,0 +1,75 @@
|
|
1
|
+
module SecretConfig
|
2
|
+
class Parser
|
3
|
+
attr_reader :tree, :path, :registry, :interpolator
|
4
|
+
|
5
|
+
def initialize(path, registry)
|
6
|
+
@path = path
|
7
|
+
@registry = registry
|
8
|
+
@fetch_list = {}
|
9
|
+
@import_list = {}
|
10
|
+
@tree = {}
|
11
|
+
@interpolator = SettingInterpolator.new
|
12
|
+
end
|
13
|
+
|
14
|
+
# Returns a flat path of keys and values from the provider without looking in the local path.
|
15
|
+
# Keys are returned with path names relative to the supplied path.
|
16
|
+
def parse(key, value)
|
17
|
+
relative_key = relative_key?(key) ? key : key.sub("#{path}/", "")
|
18
|
+
tree[relative_key] = value.is_a?(String) && value.include?("%{") ? interpolator.parse(value) : value
|
19
|
+
end
|
20
|
+
|
21
|
+
# Returns a flat Hash of the rendered paths.
|
22
|
+
def render
|
23
|
+
# apply_fetches
|
24
|
+
apply_imports
|
25
|
+
tree
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# def apply_fetches
|
31
|
+
# tree[key] = relative_key?(fetch_key) ? registry[fetch_key] : registry.provider.fetch(fetch_key)
|
32
|
+
# end
|
33
|
+
|
34
|
+
# Import from the current registry as well as new fetches.
|
35
|
+
#
|
36
|
+
# Notes:
|
37
|
+
# - A lot of absolute key lookups can be expensive since each one is a separate call.
|
38
|
+
# - Imports cannot reference other imports at this time.
|
39
|
+
def apply_imports
|
40
|
+
tree.keys.each do |key|
|
41
|
+
next unless key =~ /\/__import__\Z/
|
42
|
+
|
43
|
+
import_key = tree.delete(key)
|
44
|
+
key, _ = ::File.split(key)
|
45
|
+
|
46
|
+
# binding.irb
|
47
|
+
|
48
|
+
# With a relative key, look for the values in the current registry.
|
49
|
+
# With an absolute key call the provider and fetch the value directly.
|
50
|
+
|
51
|
+
if relative_key?(import_key)
|
52
|
+
tree.keys.each do |current_key|
|
53
|
+
match = current_key.match(/\A#{import_key}\/(.*)/)
|
54
|
+
next unless match
|
55
|
+
|
56
|
+
imported_key = ::File.join(key, match[1])
|
57
|
+
tree[imported_key] = tree[current_key] unless tree.key?(imported_key)
|
58
|
+
end
|
59
|
+
else
|
60
|
+
relative_paths = registry.send(:fetch_path, import_key)
|
61
|
+
relative_paths.each_pair do |relative_key, value|
|
62
|
+
imported_key = ::File.join(key, relative_key)
|
63
|
+
tree[imported_key] = value unless tree.key?(imported_key)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Returns [true|false] whether the supplied key is considered a relative key.
|
70
|
+
def relative_key?(key)
|
71
|
+
!key.start_with?("/")
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
@@ -12,17 +12,30 @@ module SecretConfig
|
|
12
12
|
raise(ConfigurationError, "Cannot find config file: #{file_name}") unless ::File.exist?(file_name)
|
13
13
|
end
|
14
14
|
|
15
|
+
# Yields the key with its absolute path and corresponding string value
|
15
16
|
def each(path, &block)
|
16
|
-
|
17
|
-
|
18
|
-
paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
|
19
|
-
settings = config.dig(*paths)
|
17
|
+
settings = fetch_path(path)
|
20
18
|
|
21
19
|
raise(ConfigurationError, "Path #{paths.join('/')} not found in file: #{file_name}") unless settings
|
22
20
|
|
23
21
|
Utils.flatten_each(settings, path, &block)
|
24
22
|
nil
|
25
23
|
end
|
24
|
+
|
25
|
+
# Returns the value or `nil` if not found
|
26
|
+
def fetch(key)
|
27
|
+
values = fetch_path(path)
|
28
|
+
value.is_a?(Hash) ? nil : value
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def fetch_path(path)
|
34
|
+
config = YAML.load(ERB.new(::File.new(file_name).read).result)
|
35
|
+
|
36
|
+
paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
|
37
|
+
config.dig(*paths)
|
38
|
+
end
|
26
39
|
end
|
27
40
|
end
|
28
41
|
end
|
@@ -10,7 +10,13 @@ module SecretConfig
|
|
10
10
|
class Ssm < Provider
|
11
11
|
attr_reader :client, :key_id, :retry_count, :retry_max_ms, :logger
|
12
12
|
|
13
|
-
def initialize(
|
13
|
+
def initialize(
|
14
|
+
key_id: ENV["SECRET_CONFIG_KEY_ID"],
|
15
|
+
key_alias: ENV["SECRET_CONFIG_KEY_ALIAS"],
|
16
|
+
retry_count: 25,
|
17
|
+
retry_max_ms: 10_000,
|
18
|
+
**args
|
19
|
+
)
|
14
20
|
@key_id =
|
15
21
|
if key_alias
|
16
22
|
key_alias =~ %r{^alias/} ? key_alias : "alias/#{key_alias}"
|
@@ -20,9 +26,10 @@ module SecretConfig
|
|
20
26
|
@retry_count = retry_count
|
21
27
|
@retry_max_ms = retry_max_ms
|
22
28
|
@logger = SemanticLogger["Aws::SSM"] if defined?(SemanticLogger)
|
23
|
-
@client = Aws::SSM::Client.new(logger: logger)
|
29
|
+
@client = Aws::SSM::Client.new({logger: logger}.merge!(args))
|
24
30
|
end
|
25
31
|
|
32
|
+
# Yields the key with its absolute path and corresponding string value
|
26
33
|
def each(path)
|
27
34
|
retries = 0
|
28
35
|
token = nil
|
@@ -20,10 +20,14 @@ module SecretConfig
|
|
20
20
|
end
|
21
21
|
|
22
22
|
# Returns [Hash] a copy of the in memory configuration data.
|
23
|
-
|
23
|
+
#
|
24
|
+
# Supply the relative path to start from so that only keys and values in that
|
25
|
+
# path will be returned.
|
26
|
+
def configuration(path: nil, filters: SecretConfig.filters)
|
24
27
|
h = {}
|
25
28
|
cache.each_pair do |key, value|
|
26
|
-
key
|
29
|
+
next if path && !key.start_with?(path)
|
30
|
+
|
27
31
|
value = filter_value(key, value, filters)
|
28
32
|
Utils.decompose(key, value, h)
|
29
33
|
end
|
@@ -32,30 +36,36 @@ module SecretConfig
|
|
32
36
|
|
33
37
|
# Returns [String] configuration value for the supplied key, or nil when missing.
|
34
38
|
def [](key)
|
35
|
-
|
36
|
-
value = cache[full_key]
|
39
|
+
value = cache[key]
|
37
40
|
if value.nil? && SecretConfig.check_env_var?
|
38
|
-
value
|
39
|
-
cache[
|
41
|
+
value = env_var_override(key, value)
|
42
|
+
cache[key] = value unless value.nil?
|
40
43
|
end
|
41
|
-
value
|
44
|
+
value.nil? ? nil : value.to_s
|
42
45
|
end
|
43
46
|
|
44
47
|
# Returns [String] configuration value for the supplied key, or nil when missing.
|
45
48
|
def key?(key)
|
46
|
-
cache.key?(
|
49
|
+
cache.key?(key)
|
47
50
|
end
|
48
51
|
|
49
52
|
# Returns [String] configuration value for the supplied key
|
50
|
-
|
53
|
+
# Convert the string value into an array of values by supplying a `separator`.
|
54
|
+
def fetch(key, default: :no_default_supplied, type: :string, encoding: nil, separator: nil)
|
51
55
|
value = self[key]
|
52
56
|
if value.nil?
|
53
57
|
raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
|
58
|
+
|
54
59
|
value = block_given? ? yield : default
|
55
60
|
end
|
56
61
|
|
57
62
|
value = convert_encoding(encoding, value) if encoding
|
58
|
-
|
63
|
+
|
64
|
+
if separator
|
65
|
+
value.to_s.split(separator).collect { |element| convert_type(type, element.strip) }
|
66
|
+
else
|
67
|
+
convert_type(type, value)
|
68
|
+
end
|
59
69
|
end
|
60
70
|
|
61
71
|
# Set the value for a key in the centralized configuration store.
|
@@ -65,26 +75,25 @@ module SecretConfig
|
|
65
75
|
|
66
76
|
# Set the value for a key in the centralized configuration store.
|
67
77
|
def set(key, value)
|
68
|
-
|
69
|
-
provider.set(
|
78
|
+
full_key = expand_key(key)
|
79
|
+
provider.set(full_key, value)
|
70
80
|
cache[key] = value
|
71
81
|
end
|
72
82
|
|
73
83
|
# Delete a key from the centralized configuration store.
|
74
84
|
def delete(key)
|
75
|
-
|
76
|
-
provider.delete(
|
85
|
+
full_key = expand_key(key)
|
86
|
+
provider.delete(full_key)
|
77
87
|
cache.delete(key)
|
78
88
|
end
|
79
89
|
|
80
90
|
# Refresh the in-memory cached copy of the centralized configuration information.
|
81
|
-
# Environment variable values will take
|
91
|
+
# Environment variable values will take precedence over the central store values.
|
82
92
|
def refresh!
|
83
93
|
existing_keys = cache.keys
|
84
94
|
updated_keys = []
|
85
|
-
|
86
|
-
|
87
|
-
cache[key] = env_var_override(relative_key(key), value)
|
95
|
+
fetch_path(path).each_pair do |key, value|
|
96
|
+
cache[key] = env_var_override(key, value)
|
88
97
|
updated_keys << key
|
89
98
|
end
|
90
99
|
|
@@ -98,8 +107,17 @@ module SecretConfig
|
|
98
107
|
|
99
108
|
attr_reader :cache
|
100
109
|
|
101
|
-
|
102
|
-
|
110
|
+
# Returns [true|false] whether the supplied key is considered a relative key.
|
111
|
+
def relative_key?(key)
|
112
|
+
!key.start_with?("/")
|
113
|
+
end
|
114
|
+
|
115
|
+
# Returns a flat path of keys and values from the provider without looking in the local path.
|
116
|
+
# Keys are returned with path names relative to the supplied path.
|
117
|
+
def fetch_path(path)
|
118
|
+
parser = Parser.new(path, self)
|
119
|
+
provider.each(path) { |key, value| parser.parse(key, value) }
|
120
|
+
parser.render
|
103
121
|
end
|
104
122
|
|
105
123
|
# Returns the value from an env var if it is present,
|
@@ -113,12 +131,7 @@ module SecretConfig
|
|
113
131
|
|
114
132
|
# Add the path to the path if it is a relative path.
|
115
133
|
def expand_key(key)
|
116
|
-
|
117
|
-
end
|
118
|
-
|
119
|
-
# Convert the key to a relative path by removing the path.
|
120
|
-
def relative_key(key)
|
121
|
-
key.start_with?("/") ? key.sub("#{path}/", "") : key
|
134
|
+
relative_key?(key) ? "#{path}/#{key}" : key
|
122
135
|
end
|
123
136
|
|
124
137
|
def filter_value(key, value, filters)
|
@@ -140,12 +153,12 @@ module SecretConfig
|
|
140
153
|
|
141
154
|
def convert_type(type, value)
|
142
155
|
case type
|
156
|
+
when :string
|
157
|
+
value.to_s
|
143
158
|
when :integer
|
144
159
|
value.to_i
|
145
160
|
when :float
|
146
161
|
value.to_f
|
147
|
-
when :string
|
148
|
-
value
|
149
162
|
when :boolean
|
150
163
|
%w[true 1 t].include?(value.to_s.downcase)
|
151
164
|
when :symbol
|
@@ -171,7 +184,7 @@ module SecretConfig
|
|
171
184
|
|
172
185
|
raise(UndefinedRootError, "Either set env var 'SECRET_CONFIG_PATH' or call SecretConfig.use") unless path
|
173
186
|
|
174
|
-
|
187
|
+
relative_key?(path) ? "/#{path}" : path
|
175
188
|
end
|
176
189
|
|
177
190
|
def default_provider(provider)
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "date"
|
2
|
+
require "socket"
|
3
|
+
require "securerandom"
|
4
4
|
# * SecretConfig Interpolations
|
5
5
|
#
|
6
6
|
# Expanding values inline for date, time, hostname, pid and random values.
|
@@ -14,11 +14,6 @@ require 'securerandom'
|
|
14
14
|
# %{pid} # Process Id for this process.
|
15
15
|
# %{random} # URL safe Random 32 byte value.
|
16
16
|
# %{random:size} # URL safe Random value of `size` bytes.
|
17
|
-
#
|
18
|
-
# Retrieve values elsewhere in the registry.
|
19
|
-
# Paths can be relative to the current root, or absolute paths outside the current root.
|
20
|
-
# %{fetch:key} # Fetches a single value from a relative or absolute path
|
21
|
-
# %{include:path} # Fetches a path of keys and values
|
22
17
|
module SecretConfig
|
23
18
|
class SettingInterpolator < StringInterpolator
|
24
19
|
def date(format = "%Y%m%d")
|
@@ -35,7 +30,7 @@ module SecretConfig
|
|
35
30
|
|
36
31
|
def hostname(format = nil)
|
37
32
|
name = Socket.gethostname
|
38
|
-
name = name.split(
|
33
|
+
name = name.split(".")[0] if format == "short"
|
39
34
|
name
|
40
35
|
end
|
41
36
|
|
@@ -46,13 +41,5 @@ module SecretConfig
|
|
46
41
|
def random(size = 32)
|
47
42
|
SecureRandom.urlsafe_base64(size)
|
48
43
|
end
|
49
|
-
|
50
|
-
#def fetch(key)
|
51
|
-
# SecretConfig[key]
|
52
|
-
#end
|
53
|
-
#
|
54
|
-
#def include(path)
|
55
|
-
#
|
56
|
-
#end
|
57
44
|
end
|
58
45
|
end
|
@@ -16,14 +16,15 @@ module SecretConfig
|
|
16
16
|
|
17
17
|
def parse(string)
|
18
18
|
string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
|
19
|
-
if match.start_with?(
|
19
|
+
if match.start_with?("%%")
|
20
20
|
match[1..-1]
|
21
21
|
else
|
22
|
-
expr =
|
23
|
-
key, args_str = expr.split(
|
22
|
+
expr = Regexp.last_match(1) || Regexp.last_match(2) || match.tr("%{}", "")
|
23
|
+
key, args_str = expr.split(":")
|
24
24
|
key = key.strip.to_sym
|
25
|
-
arguments = args_str&.split(
|
25
|
+
arguments = args_str&.split(",")&.map { |v| v.strip == "" ? nil : v.strip } || []
|
26
26
|
raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
|
27
|
+
|
27
28
|
public_send(key, *arguments)
|
28
29
|
end
|
29
30
|
end
|
data/lib/secret_config/utils.rb
CHANGED
data/test/config/application.yml
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
# Local application config goes here. Do not check in production secrets.
|
2
2
|
# These are for development and test only.
|
3
|
-
|
4
|
-
#
|
5
|
-
# Development - Local - Root: '/test/my_application'
|
6
|
-
#
|
7
3
|
development:
|
8
4
|
my_application:
|
9
5
|
symmetric_encryption:
|
@@ -31,12 +27,20 @@ test:
|
|
31
27
|
key: QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=
|
32
28
|
iv: QUJDREVGMTIzNDU2Nzg5MA==
|
33
29
|
version: 2
|
30
|
+
previous_key:
|
31
|
+
key: key1
|
32
|
+
iv: iv1
|
33
|
+
version: 1
|
34
34
|
|
35
35
|
mysql:
|
36
36
|
database: secret_config_test
|
37
37
|
username: secret_config
|
38
38
|
password: secret_configrules
|
39
39
|
host: 127.0.0.1
|
40
|
+
ports: "12345,5343,26815"
|
41
|
+
ports2: " 12345, 5343 , 26815"
|
42
|
+
hostnames: "primary.example.net,secondary.example.net,backup.example.net"
|
43
|
+
hostnames2: " primary.example.net, secondary.example.net , backup.example.net"
|
40
44
|
|
41
45
|
mongo:
|
42
46
|
database: secret_config_test
|
@@ -45,3 +49,29 @@ test:
|
|
45
49
|
|
46
50
|
secrets:
|
47
51
|
secret_key_base: somereallylongteststring
|
52
|
+
|
53
|
+
other_application:
|
54
|
+
symmetric_encryption:
|
55
|
+
version: 3
|
56
|
+
__import__: /test/my_application/symmetric_encryption
|
57
|
+
iv: MTIzNDU2Nzg5MEFCQ0RFRg==
|
58
|
+
previous_key:
|
59
|
+
key: key0
|
60
|
+
|
61
|
+
mysql:
|
62
|
+
# database: "%{fetch: /test/my_application/mysql/database }"
|
63
|
+
username: other
|
64
|
+
password: otherrules
|
65
|
+
host: "%{hostname}"
|
66
|
+
|
67
|
+
mongo:
|
68
|
+
database: secret_config_test
|
69
|
+
primary: localhost:27017
|
70
|
+
secondary: "%{hostname}:27018"
|
71
|
+
|
72
|
+
mongo2:
|
73
|
+
__import__: mongo
|
74
|
+
database: secret_config_test2
|
75
|
+
|
76
|
+
mongo3:
|
77
|
+
__import__: mongo
|
data/test/parser_test.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
require "socket"
|
3
|
+
|
4
|
+
class ParserTest < Minitest::Test
|
5
|
+
describe SecretConfig::Registry do
|
6
|
+
let :file_name do
|
7
|
+
File.join(File.dirname(__FILE__), "config", "application.yml")
|
8
|
+
end
|
9
|
+
|
10
|
+
let :path do
|
11
|
+
"/test/other_application"
|
12
|
+
end
|
13
|
+
|
14
|
+
let :provider do
|
15
|
+
SecretConfig::Providers::File.new(file_name: file_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
let :registry do
|
19
|
+
SecretConfig::Registry.new(path: path, provider: provider)
|
20
|
+
end
|
21
|
+
|
22
|
+
# let :parser do
|
23
|
+
# SecretConfig::Parser.new(path, registry)
|
24
|
+
# end
|
25
|
+
|
26
|
+
#
|
27
|
+
# Retrieve values elsewhere in the registry.
|
28
|
+
# Paths can be relative to the current root, or absolute paths outside the current root.
|
29
|
+
# %{fetch:key} # Fetches a single value from a relative or absolute path
|
30
|
+
# Return the value of the supplied key.
|
31
|
+
#
|
32
|
+
# With a relative key, look for the value in the current registry.
|
33
|
+
# With an absolute key call the provider and fetch the value directly.
|
34
|
+
#
|
35
|
+
# Notes:
|
36
|
+
# - A lot of absolute key lookups can be expensive since each one is a separate call.
|
37
|
+
# def fetch(key)
|
38
|
+
# fetch_list[key] = key
|
39
|
+
# end
|
40
|
+
# describe "#fetch" do
|
41
|
+
# it "inside current path" do
|
42
|
+
#
|
43
|
+
# end
|
44
|
+
#
|
45
|
+
# it "outside current path" do
|
46
|
+
#
|
47
|
+
# end
|
48
|
+
# end
|
49
|
+
|
50
|
+
# %{import:path} # Imports a path of keys and values into the current path
|
51
|
+
# Replace the current value with a tree of values with the supplied path.
|
52
|
+
#
|
53
|
+
describe "#import" do
|
54
|
+
it "removes import key" do
|
55
|
+
refute registry.key?("symmetric_encryption/__import__"), -> { registry.configuration(filters: nil).ai }
|
56
|
+
end
|
57
|
+
|
58
|
+
it "retains overrides" do
|
59
|
+
assert_equal "3", registry["symmetric_encryption/version"], -> { registry.configuration(filters: nil).ai }
|
60
|
+
assert_equal "MTIzNDU2Nzg5MEFCQ0RFRg==", registry["symmetric_encryption/iv"]
|
61
|
+
end
|
62
|
+
|
63
|
+
it "retains child overrides" do
|
64
|
+
assert_equal "key0", registry["symmetric_encryption/previous_key/key"], -> { registry.configuration(filters: nil).ai }
|
65
|
+
end
|
66
|
+
|
67
|
+
it "imports new fields" do
|
68
|
+
assert_equal "QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=", registry["symmetric_encryption/key"]
|
69
|
+
end
|
70
|
+
|
71
|
+
it "relative import empty" do
|
72
|
+
assert_equal "secret_config_test", registry["mongo3/database"]
|
73
|
+
assert_equal "localhost:27017", registry["mongo3/primary"]
|
74
|
+
end
|
75
|
+
|
76
|
+
it "relative import with overrides" do
|
77
|
+
assert_equal "secret_config_test2", registry["mongo2/database"]
|
78
|
+
assert_equal "localhost:27017", registry["mongo3/primary"]
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|