secret_config 0.7.1 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +84 -42
- data/Rakefile +7 -7
- data/bin/secret_config +1 -1
- data/lib/secret_config.rb +30 -1
- data/lib/secret_config/config.rb +44 -0
- data/lib/secret_config/parser.rb +75 -0
- data/lib/secret_config/providers/file.rb +17 -4
- data/lib/secret_config/providers/ssm.rb +9 -2
- data/lib/secret_config/registry.rb +42 -29
- data/lib/secret_config/setting_interpolator.rb +4 -17
- data/lib/secret_config/string_interpolator.rb +5 -4
- data/lib/secret_config/utils.rb +1 -1
- data/lib/secret_config/version.rb +1 -1
- data/test/config/application.yml +34 -4
- data/test/parser_test.rb +82 -0
- data/test/providers/file_test.rb +4 -4
- data/test/providers/ssm_test.rb +37 -12
- data/test/registry_test.rb +51 -26
- data/test/secret_config_test.rb +35 -4
- data/test/setting_interpolator_test.rb +17 -17
- data/test/test_helper.rb +6 -6
- data/test/utils_test.rb +4 -4
- metadata +7 -3
@@ -0,0 +1,75 @@
|
|
1
|
+
module SecretConfig
|
2
|
+
class Parser
|
3
|
+
attr_reader :tree, :path, :registry, :interpolator
|
4
|
+
|
5
|
+
def initialize(path, registry)
|
6
|
+
@path = path
|
7
|
+
@registry = registry
|
8
|
+
@fetch_list = {}
|
9
|
+
@import_list = {}
|
10
|
+
@tree = {}
|
11
|
+
@interpolator = SettingInterpolator.new
|
12
|
+
end
|
13
|
+
|
14
|
+
# Returns a flat path of keys and values from the provider without looking in the local path.
|
15
|
+
# Keys are returned with path names relative to the supplied path.
|
16
|
+
def parse(key, value)
|
17
|
+
relative_key = relative_key?(key) ? key : key.sub("#{path}/", "")
|
18
|
+
tree[relative_key] = value.is_a?(String) && value.include?("%{") ? interpolator.parse(value) : value
|
19
|
+
end
|
20
|
+
|
21
|
+
# Returns a flat Hash of the rendered paths.
|
22
|
+
def render
|
23
|
+
# apply_fetches
|
24
|
+
apply_imports
|
25
|
+
tree
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# def apply_fetches
|
31
|
+
# tree[key] = relative_key?(fetch_key) ? registry[fetch_key] : registry.provider.fetch(fetch_key)
|
32
|
+
# end
|
33
|
+
|
34
|
+
# Import from the current registry as well as new fetches.
|
35
|
+
#
|
36
|
+
# Notes:
|
37
|
+
# - A lot of absolute key lookups can be expensive since each one is a separate call.
|
38
|
+
# - Imports cannot reference other imports at this time.
|
39
|
+
def apply_imports
|
40
|
+
tree.keys.each do |key|
|
41
|
+
next unless key =~ /\/__import__\Z/
|
42
|
+
|
43
|
+
import_key = tree.delete(key)
|
44
|
+
key, _ = ::File.split(key)
|
45
|
+
|
46
|
+
# binding.irb
|
47
|
+
|
48
|
+
# With a relative key, look for the values in the current registry.
|
49
|
+
# With an absolute key call the provider and fetch the value directly.
|
50
|
+
|
51
|
+
if relative_key?(import_key)
|
52
|
+
tree.keys.each do |current_key|
|
53
|
+
match = current_key.match(/\A#{import_key}\/(.*)/)
|
54
|
+
next unless match
|
55
|
+
|
56
|
+
imported_key = ::File.join(key, match[1])
|
57
|
+
tree[imported_key] = tree[current_key] unless tree.key?(imported_key)
|
58
|
+
end
|
59
|
+
else
|
60
|
+
relative_paths = registry.send(:fetch_path, import_key)
|
61
|
+
relative_paths.each_pair do |relative_key, value|
|
62
|
+
imported_key = ::File.join(key, relative_key)
|
63
|
+
tree[imported_key] = value unless tree.key?(imported_key)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Returns [true|false] whether the supplied key is considered a relative key.
|
70
|
+
def relative_key?(key)
|
71
|
+
!key.start_with?("/")
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
@@ -12,17 +12,30 @@ module SecretConfig
|
|
12
12
|
raise(ConfigurationError, "Cannot find config file: #{file_name}") unless ::File.exist?(file_name)
|
13
13
|
end
|
14
14
|
|
15
|
+
# Yields the key with its absolute path and corresponding string value
|
15
16
|
def each(path, &block)
|
16
|
-
|
17
|
-
|
18
|
-
paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
|
19
|
-
settings = config.dig(*paths)
|
17
|
+
settings = fetch_path(path)
|
20
18
|
|
21
19
|
raise(ConfigurationError, "Path #{paths.join('/')} not found in file: #{file_name}") unless settings
|
22
20
|
|
23
21
|
Utils.flatten_each(settings, path, &block)
|
24
22
|
nil
|
25
23
|
end
|
24
|
+
|
25
|
+
# Returns the value or `nil` if not found
|
26
|
+
def fetch(key)
|
27
|
+
values = fetch_path(path)
|
28
|
+
value.is_a?(Hash) ? nil : value
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def fetch_path(path)
|
34
|
+
config = YAML.load(ERB.new(::File.new(file_name).read).result)
|
35
|
+
|
36
|
+
paths = path.sub(%r{\A/*}, "").sub(%r{/*\Z}, "").split("/")
|
37
|
+
config.dig(*paths)
|
38
|
+
end
|
26
39
|
end
|
27
40
|
end
|
28
41
|
end
|
@@ -10,7 +10,13 @@ module SecretConfig
|
|
10
10
|
class Ssm < Provider
|
11
11
|
attr_reader :client, :key_id, :retry_count, :retry_max_ms, :logger
|
12
12
|
|
13
|
-
def initialize(
|
13
|
+
def initialize(
|
14
|
+
key_id: ENV["SECRET_CONFIG_KEY_ID"],
|
15
|
+
key_alias: ENV["SECRET_CONFIG_KEY_ALIAS"],
|
16
|
+
retry_count: 25,
|
17
|
+
retry_max_ms: 10_000,
|
18
|
+
**args
|
19
|
+
)
|
14
20
|
@key_id =
|
15
21
|
if key_alias
|
16
22
|
key_alias =~ %r{^alias/} ? key_alias : "alias/#{key_alias}"
|
@@ -20,9 +26,10 @@ module SecretConfig
|
|
20
26
|
@retry_count = retry_count
|
21
27
|
@retry_max_ms = retry_max_ms
|
22
28
|
@logger = SemanticLogger["Aws::SSM"] if defined?(SemanticLogger)
|
23
|
-
@client = Aws::SSM::Client.new(logger: logger)
|
29
|
+
@client = Aws::SSM::Client.new({logger: logger}.merge!(args))
|
24
30
|
end
|
25
31
|
|
32
|
+
# Yields the key with its absolute path and corresponding string value
|
26
33
|
def each(path)
|
27
34
|
retries = 0
|
28
35
|
token = nil
|
@@ -20,10 +20,14 @@ module SecretConfig
|
|
20
20
|
end
|
21
21
|
|
22
22
|
# Returns [Hash] a copy of the in memory configuration data.
|
23
|
-
|
23
|
+
#
|
24
|
+
# Supply the relative path to start from so that only keys and values in that
|
25
|
+
# path will be returned.
|
26
|
+
def configuration(path: nil, filters: SecretConfig.filters)
|
24
27
|
h = {}
|
25
28
|
cache.each_pair do |key, value|
|
26
|
-
key
|
29
|
+
next if path && !key.start_with?(path)
|
30
|
+
|
27
31
|
value = filter_value(key, value, filters)
|
28
32
|
Utils.decompose(key, value, h)
|
29
33
|
end
|
@@ -32,30 +36,36 @@ module SecretConfig
|
|
32
36
|
|
33
37
|
# Returns [String] configuration value for the supplied key, or nil when missing.
|
34
38
|
def [](key)
|
35
|
-
|
36
|
-
value = cache[full_key]
|
39
|
+
value = cache[key]
|
37
40
|
if value.nil? && SecretConfig.check_env_var?
|
38
|
-
value
|
39
|
-
cache[
|
41
|
+
value = env_var_override(key, value)
|
42
|
+
cache[key] = value unless value.nil?
|
40
43
|
end
|
41
|
-
value
|
44
|
+
value.nil? ? nil : value.to_s
|
42
45
|
end
|
43
46
|
|
44
47
|
# Returns [String] configuration value for the supplied key, or nil when missing.
|
45
48
|
def key?(key)
|
46
|
-
cache.key?(
|
49
|
+
cache.key?(key)
|
47
50
|
end
|
48
51
|
|
49
52
|
# Returns [String] configuration value for the supplied key
|
50
|
-
|
53
|
+
# Convert the string value into an array of values by supplying a `separator`.
|
54
|
+
def fetch(key, default: :no_default_supplied, type: :string, encoding: nil, separator: nil)
|
51
55
|
value = self[key]
|
52
56
|
if value.nil?
|
53
57
|
raise(MissingMandatoryKey, "Missing configuration value for #{path}/#{key}") if default == :no_default_supplied
|
58
|
+
|
54
59
|
value = block_given? ? yield : default
|
55
60
|
end
|
56
61
|
|
57
62
|
value = convert_encoding(encoding, value) if encoding
|
58
|
-
|
63
|
+
|
64
|
+
if separator
|
65
|
+
value.to_s.split(separator).collect { |element| convert_type(type, element.strip) }
|
66
|
+
else
|
67
|
+
convert_type(type, value)
|
68
|
+
end
|
59
69
|
end
|
60
70
|
|
61
71
|
# Set the value for a key in the centralized configuration store.
|
@@ -65,26 +75,25 @@ module SecretConfig
|
|
65
75
|
|
66
76
|
# Set the value for a key in the centralized configuration store.
|
67
77
|
def set(key, value)
|
68
|
-
|
69
|
-
provider.set(
|
78
|
+
full_key = expand_key(key)
|
79
|
+
provider.set(full_key, value)
|
70
80
|
cache[key] = value
|
71
81
|
end
|
72
82
|
|
73
83
|
# Delete a key from the centralized configuration store.
|
74
84
|
def delete(key)
|
75
|
-
|
76
|
-
provider.delete(
|
85
|
+
full_key = expand_key(key)
|
86
|
+
provider.delete(full_key)
|
77
87
|
cache.delete(key)
|
78
88
|
end
|
79
89
|
|
80
90
|
# Refresh the in-memory cached copy of the centralized configuration information.
|
81
|
-
# Environment variable values will take
|
91
|
+
# Environment variable values will take precedence over the central store values.
|
82
92
|
def refresh!
|
83
93
|
existing_keys = cache.keys
|
84
94
|
updated_keys = []
|
85
|
-
|
86
|
-
|
87
|
-
cache[key] = env_var_override(relative_key(key), value)
|
95
|
+
fetch_path(path).each_pair do |key, value|
|
96
|
+
cache[key] = env_var_override(key, value)
|
88
97
|
updated_keys << key
|
89
98
|
end
|
90
99
|
|
@@ -98,8 +107,17 @@ module SecretConfig
|
|
98
107
|
|
99
108
|
attr_reader :cache
|
100
109
|
|
101
|
-
|
102
|
-
|
110
|
+
# Returns [true|false] whether the supplied key is considered a relative key.
|
111
|
+
def relative_key?(key)
|
112
|
+
!key.start_with?("/")
|
113
|
+
end
|
114
|
+
|
115
|
+
# Returns a flat path of keys and values from the provider without looking in the local path.
|
116
|
+
# Keys are returned with path names relative to the supplied path.
|
117
|
+
def fetch_path(path)
|
118
|
+
parser = Parser.new(path, self)
|
119
|
+
provider.each(path) { |key, value| parser.parse(key, value) }
|
120
|
+
parser.render
|
103
121
|
end
|
104
122
|
|
105
123
|
# Returns the value from an env var if it is present,
|
@@ -113,12 +131,7 @@ module SecretConfig
|
|
113
131
|
|
114
132
|
# Add the path to the path if it is a relative path.
|
115
133
|
def expand_key(key)
|
116
|
-
|
117
|
-
end
|
118
|
-
|
119
|
-
# Convert the key to a relative path by removing the path.
|
120
|
-
def relative_key(key)
|
121
|
-
key.start_with?("/") ? key.sub("#{path}/", "") : key
|
134
|
+
relative_key?(key) ? "#{path}/#{key}" : key
|
122
135
|
end
|
123
136
|
|
124
137
|
def filter_value(key, value, filters)
|
@@ -140,12 +153,12 @@ module SecretConfig
|
|
140
153
|
|
141
154
|
def convert_type(type, value)
|
142
155
|
case type
|
156
|
+
when :string
|
157
|
+
value.to_s
|
143
158
|
when :integer
|
144
159
|
value.to_i
|
145
160
|
when :float
|
146
161
|
value.to_f
|
147
|
-
when :string
|
148
|
-
value
|
149
162
|
when :boolean
|
150
163
|
%w[true 1 t].include?(value.to_s.downcase)
|
151
164
|
when :symbol
|
@@ -171,7 +184,7 @@ module SecretConfig
|
|
171
184
|
|
172
185
|
raise(UndefinedRootError, "Either set env var 'SECRET_CONFIG_PATH' or call SecretConfig.use") unless path
|
173
186
|
|
174
|
-
|
187
|
+
relative_key?(path) ? "/#{path}" : path
|
175
188
|
end
|
176
189
|
|
177
190
|
def default_provider(provider)
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "date"
|
2
|
+
require "socket"
|
3
|
+
require "securerandom"
|
4
4
|
# * SecretConfig Interpolations
|
5
5
|
#
|
6
6
|
# Expanding values inline for date, time, hostname, pid and random values.
|
@@ -14,11 +14,6 @@ require 'securerandom'
|
|
14
14
|
# %{pid} # Process Id for this process.
|
15
15
|
# %{random} # URL safe Random 32 byte value.
|
16
16
|
# %{random:size} # URL safe Random value of `size` bytes.
|
17
|
-
#
|
18
|
-
# Retrieve values elsewhere in the registry.
|
19
|
-
# Paths can be relative to the current root, or absolute paths outside the current root.
|
20
|
-
# %{fetch:key} # Fetches a single value from a relative or absolute path
|
21
|
-
# %{include:path} # Fetches a path of keys and values
|
22
17
|
module SecretConfig
|
23
18
|
class SettingInterpolator < StringInterpolator
|
24
19
|
def date(format = "%Y%m%d")
|
@@ -35,7 +30,7 @@ module SecretConfig
|
|
35
30
|
|
36
31
|
def hostname(format = nil)
|
37
32
|
name = Socket.gethostname
|
38
|
-
name = name.split(
|
33
|
+
name = name.split(".")[0] if format == "short"
|
39
34
|
name
|
40
35
|
end
|
41
36
|
|
@@ -46,13 +41,5 @@ module SecretConfig
|
|
46
41
|
def random(size = 32)
|
47
42
|
SecureRandom.urlsafe_base64(size)
|
48
43
|
end
|
49
|
-
|
50
|
-
#def fetch(key)
|
51
|
-
# SecretConfig[key]
|
52
|
-
#end
|
53
|
-
#
|
54
|
-
#def include(path)
|
55
|
-
#
|
56
|
-
#end
|
57
44
|
end
|
58
45
|
end
|
@@ -16,14 +16,15 @@ module SecretConfig
|
|
16
16
|
|
17
17
|
def parse(string)
|
18
18
|
string.gsub(/%{1,2}\{([^}]+)\}/) do |match|
|
19
|
-
if match.start_with?(
|
19
|
+
if match.start_with?("%%")
|
20
20
|
match[1..-1]
|
21
21
|
else
|
22
|
-
expr =
|
23
|
-
key, args_str = expr.split(
|
22
|
+
expr = Regexp.last_match(1) || Regexp.last_match(2) || match.tr("%{}", "")
|
23
|
+
key, args_str = expr.split(":")
|
24
24
|
key = key.strip.to_sym
|
25
|
-
arguments = args_str&.split(
|
25
|
+
arguments = args_str&.split(",")&.map { |v| v.strip == "" ? nil : v.strip } || []
|
26
26
|
raise(InvalidInterpolation, "Invalid key: #{key} in string: #{match}") unless respond_to?(key)
|
27
|
+
|
27
28
|
public_send(key, *arguments)
|
28
29
|
end
|
29
30
|
end
|
data/lib/secret_config/utils.rb
CHANGED
data/test/config/application.yml
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
# Local application config goes here. Do not check in production secrets.
|
2
2
|
# These are for development and test only.
|
3
|
-
|
4
|
-
#
|
5
|
-
# Development - Local - Root: '/test/my_application'
|
6
|
-
#
|
7
3
|
development:
|
8
4
|
my_application:
|
9
5
|
symmetric_encryption:
|
@@ -31,12 +27,20 @@ test:
|
|
31
27
|
key: QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=
|
32
28
|
iv: QUJDREVGMTIzNDU2Nzg5MA==
|
33
29
|
version: 2
|
30
|
+
previous_key:
|
31
|
+
key: key1
|
32
|
+
iv: iv1
|
33
|
+
version: 1
|
34
34
|
|
35
35
|
mysql:
|
36
36
|
database: secret_config_test
|
37
37
|
username: secret_config
|
38
38
|
password: secret_configrules
|
39
39
|
host: 127.0.0.1
|
40
|
+
ports: "12345,5343,26815"
|
41
|
+
ports2: " 12345, 5343 , 26815"
|
42
|
+
hostnames: "primary.example.net,secondary.example.net,backup.example.net"
|
43
|
+
hostnames2: " primary.example.net, secondary.example.net , backup.example.net"
|
40
44
|
|
41
45
|
mongo:
|
42
46
|
database: secret_config_test
|
@@ -45,3 +49,29 @@ test:
|
|
45
49
|
|
46
50
|
secrets:
|
47
51
|
secret_key_base: somereallylongteststring
|
52
|
+
|
53
|
+
other_application:
|
54
|
+
symmetric_encryption:
|
55
|
+
version: 3
|
56
|
+
__import__: /test/my_application/symmetric_encryption
|
57
|
+
iv: MTIzNDU2Nzg5MEFCQ0RFRg==
|
58
|
+
previous_key:
|
59
|
+
key: key0
|
60
|
+
|
61
|
+
mysql:
|
62
|
+
# database: "%{fetch: /test/my_application/mysql/database }"
|
63
|
+
username: other
|
64
|
+
password: otherrules
|
65
|
+
host: "%{hostname}"
|
66
|
+
|
67
|
+
mongo:
|
68
|
+
database: secret_config_test
|
69
|
+
primary: localhost:27017
|
70
|
+
secondary: "%{hostname}:27018"
|
71
|
+
|
72
|
+
mongo2:
|
73
|
+
__import__: mongo
|
74
|
+
database: secret_config_test2
|
75
|
+
|
76
|
+
mongo3:
|
77
|
+
__import__: mongo
|
data/test/parser_test.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
require "socket"
|
3
|
+
|
4
|
+
class ParserTest < Minitest::Test
|
5
|
+
describe SecretConfig::Registry do
|
6
|
+
let :file_name do
|
7
|
+
File.join(File.dirname(__FILE__), "config", "application.yml")
|
8
|
+
end
|
9
|
+
|
10
|
+
let :path do
|
11
|
+
"/test/other_application"
|
12
|
+
end
|
13
|
+
|
14
|
+
let :provider do
|
15
|
+
SecretConfig::Providers::File.new(file_name: file_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
let :registry do
|
19
|
+
SecretConfig::Registry.new(path: path, provider: provider)
|
20
|
+
end
|
21
|
+
|
22
|
+
# let :parser do
|
23
|
+
# SecretConfig::Parser.new(path, registry)
|
24
|
+
# end
|
25
|
+
|
26
|
+
#
|
27
|
+
# Retrieve values elsewhere in the registry.
|
28
|
+
# Paths can be relative to the current root, or absolute paths outside the current root.
|
29
|
+
# %{fetch:key} # Fetches a single value from a relative or absolute path
|
30
|
+
# Return the value of the supplied key.
|
31
|
+
#
|
32
|
+
# With a relative key, look for the value in the current registry.
|
33
|
+
# With an absolute key call the provider and fetch the value directly.
|
34
|
+
#
|
35
|
+
# Notes:
|
36
|
+
# - A lot of absolute key lookups can be expensive since each one is a separate call.
|
37
|
+
# def fetch(key)
|
38
|
+
# fetch_list[key] = key
|
39
|
+
# end
|
40
|
+
# describe "#fetch" do
|
41
|
+
# it "inside current path" do
|
42
|
+
#
|
43
|
+
# end
|
44
|
+
#
|
45
|
+
# it "outside current path" do
|
46
|
+
#
|
47
|
+
# end
|
48
|
+
# end
|
49
|
+
|
50
|
+
# %{import:path} # Imports a path of keys and values into the current path
|
51
|
+
# Replace the current value with a tree of values with the supplied path.
|
52
|
+
#
|
53
|
+
describe "#import" do
|
54
|
+
it "removes import key" do
|
55
|
+
refute registry.key?("symmetric_encryption/__import__"), -> { registry.configuration(filters: nil).ai }
|
56
|
+
end
|
57
|
+
|
58
|
+
it "retains overrides" do
|
59
|
+
assert_equal "3", registry["symmetric_encryption/version"], -> { registry.configuration(filters: nil).ai }
|
60
|
+
assert_equal "MTIzNDU2Nzg5MEFCQ0RFRg==", registry["symmetric_encryption/iv"]
|
61
|
+
end
|
62
|
+
|
63
|
+
it "retains child overrides" do
|
64
|
+
assert_equal "key0", registry["symmetric_encryption/previous_key/key"], -> { registry.configuration(filters: nil).ai }
|
65
|
+
end
|
66
|
+
|
67
|
+
it "imports new fields" do
|
68
|
+
assert_equal "QUJDREVGMTIzNDU2Nzg5MEFCQ0RFRjEyMzQ1Njc4OTA=", registry["symmetric_encryption/key"]
|
69
|
+
end
|
70
|
+
|
71
|
+
it "relative import empty" do
|
72
|
+
assert_equal "secret_config_test", registry["mongo3/database"]
|
73
|
+
assert_equal "localhost:27017", registry["mongo3/primary"]
|
74
|
+
end
|
75
|
+
|
76
|
+
it "relative import with overrides" do
|
77
|
+
assert_equal "secret_config_test2", registry["mongo2/database"]
|
78
|
+
assert_equal "localhost:27017", registry["mongo3/primary"]
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|