cps-property-generator 0.2.14 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/cps-property-generator +7 -5
- data/lib/generator/generator.rb +22 -11
- data/lib/generator/globals.rb +3 -3
- data/lib/generator/service.rb +12 -5
- data/lib/linter/globals_linter.rb +0 -1
- data/lib/linter/services_linter.rb +4 -48
- data/spec/lib/global_spec.rb +26 -4
- data/spec/lib/service_spec.rb +73 -23
- data/spec/resources/globals/globals.yml +5 -1
- data/spec/resources/services/my-microservice-1.yml +15 -4
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f659265beb057cc6e807215df12702562307e50
|
4
|
+
data.tar.gz: 58a223834ecda08eea38a5f9a21ad6cbdf39f37d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbda7d164c992be4eab086b2f770dee1c8adbefe259ae6c12da695ba09d6f3c8ffbaa90a76e51e7572cb2143d7c26609c3ac39231d59a4af964daff555267e07
|
7
|
+
data.tar.gz: bf99d2880a883f34fc8047a61fff2b55c51307faa71d450780a30f01ef29f40d882c622e3d547df0e27e7d3ca127e4f73b12ce886c338c7971948e23f05284ff
|
data/bin/cps-property-generator
CHANGED
@@ -13,17 +13,19 @@ class GeneratorCLI < ::Thor
|
|
13
13
|
option 'upload_account', banner: 'UPLOAD_ACCOUNT', type: :string, desc: 'The account you are uploading properties to'
|
14
14
|
option 'upload_region', banner: 'UPLOAD_REGION', type: :string, desc: 'The region your property bucket is in'
|
15
15
|
option 'upload_bucket', banner: 'UPLOAD_BUCKET', type: :string, desc: 'The bucket you are uploading properties to.'
|
16
|
+
option 'upload_all', banner: 'UPLOAD_ALL', type: :boolean, desc: 'Whether to upload all envs and accounts to a single bucket', :default => false
|
16
17
|
|
17
18
|
def generate
|
18
19
|
|
19
20
|
generator = PropertyGenerator::Generator.new(options)
|
20
21
|
out = generator.generate
|
21
22
|
if options['upload']
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
abort('Did not specify an upload bucket') if options['upload_bucket'].nil?
|
24
|
+
abort('Did not specify an upload region') if options['upload_region'].nil?
|
25
|
+
|
26
|
+
abort('Did not specify upload configs') if !options['upload_all'] && options['upload_account'].nil?
|
27
|
+
|
28
|
+
generator.upload(out, options)
|
27
29
|
end
|
28
30
|
end
|
29
31
|
|
data/lib/generator/generator.rb
CHANGED
@@ -37,22 +37,33 @@ module PropertyGenerator
|
|
37
37
|
end
|
38
38
|
|
39
39
|
def upload(out, config)
|
40
|
-
|
40
|
+
upload_bucket = config['upload_bucket']
|
41
|
+
upload_region = config['upload_region']
|
41
42
|
|
42
|
-
if
|
43
|
-
|
44
|
-
|
43
|
+
if config['upload_all']
|
44
|
+
_upload_files(out.sort) do |file|
|
45
|
+
file_region = file.split("/")[-2]
|
46
|
+
file_account = file.split("/")[-3]
|
45
47
|
|
46
|
-
|
47
|
-
|
48
|
-
|
48
|
+
PropertyGenerator.sync(upload_region, file_account, upload_bucket, file, file_region)
|
49
|
+
end
|
50
|
+
else
|
51
|
+
upload_account = config['upload_account']
|
52
|
+
abort("The specified account (#{upload_account}) is not configured, please add it to config/config.yml") unless @accounts.include?(upload_account)
|
53
|
+
|
54
|
+
upload_out = out.select { |file| file.include?("#{upload_account}") && file.include?("#{upload_region}") }
|
55
|
+
_upload_files(upload_out) do |file|
|
56
|
+
file_region = file.split("/")[-2]
|
57
|
+
PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
49
61
|
|
50
|
-
|
51
|
-
|
62
|
+
def _upload_files(files)
|
63
|
+
files.each_slice(20) do |file_slice|
|
52
64
|
file_slice.map do |file|
|
53
65
|
Thread.new do
|
54
|
-
|
55
|
-
PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
|
66
|
+
yield file
|
56
67
|
end
|
57
68
|
end.each(&:join)
|
58
69
|
end
|
data/lib/generator/globals.rb
CHANGED
@@ -44,7 +44,7 @@ module PropertyGenerator
|
|
44
44
|
unless data[account][env]['encrypted'].nil?
|
45
45
|
encrypted = data[account][env]['encrypted'].dup
|
46
46
|
not_encrypted = data[account][env].reject { |k,_| k == 'encrypted' }
|
47
|
-
data[account][env] = not_encrypted.
|
47
|
+
data[account][env] = not_encrypted.deep_merge(encrypted)
|
48
48
|
end
|
49
49
|
end
|
50
50
|
end
|
@@ -67,7 +67,7 @@ module PropertyGenerator
|
|
67
67
|
env_global.each do |env, hash|
|
68
68
|
account_globals[account] ||= {}
|
69
69
|
# set the environment globals to be the account global merged with the env globals
|
70
|
-
env_global[env] = account_globals[account].
|
70
|
+
env_global[env] = account_globals[account].deep_merge(hash) unless hash.empty?
|
71
71
|
condensed[env] = env_global[env]
|
72
72
|
end
|
73
73
|
end
|
@@ -80,7 +80,7 @@ module PropertyGenerator
|
|
80
80
|
# We need to merge into the globals so any env configs overwrite main global configs.
|
81
81
|
# Dup so we dont modify the original object
|
82
82
|
main_global_dup = main_global.dup
|
83
|
-
condensed[env] = main_global_dup.
|
83
|
+
condensed[env] = main_global_dup.deep_merge(condensed[env])
|
84
84
|
end
|
85
85
|
end
|
86
86
|
condensed
|
data/lib/generator/service.rb
CHANGED
@@ -50,7 +50,9 @@ module PropertyGenerator
|
|
50
50
|
|
51
51
|
# Recursively interate through the properties for an environment and gsub the config
|
52
52
|
# with defined interpolations.
|
53
|
-
|
53
|
+
service_env = Marshal.load(Marshal.dump(@service[env]))
|
54
|
+
interpolate_nested_properties(service_env, interpolations)
|
55
|
+
@service[env] = service_env
|
54
56
|
end
|
55
57
|
service
|
56
58
|
end
|
@@ -58,12 +60,17 @@ module PropertyGenerator
|
|
58
60
|
def interpolate_nested_properties(service_env, interpolations)
|
59
61
|
interpolations.each do |matcher_key, matcher_value|
|
60
62
|
service_env.each { |k,v| service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}")}
|
61
|
-
service_env.values.each
|
63
|
+
service_env.values.each do |v|
|
64
|
+
interpolate_nested_properties(v, interpolations) if v.class == Hash
|
65
|
+
v.each_with_index do |val, idx|
|
66
|
+
v[idx] = val.gsub("{#{matcher_key}}", matcher_value) if val.class == String && val.include?("{#{matcher_key}}")
|
67
|
+
end if v.class == Array
|
68
|
+
end
|
62
69
|
end
|
63
70
|
end
|
64
71
|
|
65
72
|
def merge_env_default(data, environments)
|
66
|
-
#creates a hash of the
|
73
|
+
#creates a hash of the environments merged with the defaults
|
67
74
|
# {service => {env1 => {properties},
|
68
75
|
# env2 => {properties}
|
69
76
|
# }
|
@@ -84,7 +91,7 @@ module PropertyGenerator
|
|
84
91
|
if default_clone.nil?
|
85
92
|
merged = environment_data
|
86
93
|
else
|
87
|
-
merged = default_clone.
|
94
|
+
merged = default_clone.deep_merge(environment_data)
|
88
95
|
end
|
89
96
|
output[env] = merged
|
90
97
|
end
|
@@ -100,7 +107,7 @@ module PropertyGenerator
|
|
100
107
|
if globals_clone[env].nil? || globals_clone[env] == false
|
101
108
|
merged = service_data[env]
|
102
109
|
else
|
103
|
-
merged = globals_clone[env].
|
110
|
+
merged = globals_clone[env].deep_merge(service_data[env])
|
104
111
|
end
|
105
112
|
output[env] = merged
|
106
113
|
end
|
@@ -16,9 +16,7 @@ module PropertyGenerator
|
|
16
16
|
tests = [
|
17
17
|
'services_have_accepted_keys',
|
18
18
|
'service_environments_are_not_empty',
|
19
|
-
'service_defaults_have_no_hashes_as_values',
|
20
19
|
'service_environments_match_config_environments',
|
21
|
-
'service_environments_have_no_hashes_as_values',
|
22
20
|
'service_encrypted_environments_match_config_environments',
|
23
21
|
'service_encrypted_fields_are_correct',
|
24
22
|
'service_encrypted_region_field_is_accepted'
|
@@ -48,7 +46,7 @@ module PropertyGenerator
|
|
48
46
|
|
49
47
|
def services_have_accepted_keys
|
50
48
|
status = {status: 'pass', error: ''}
|
51
|
-
accepted_keys = ['default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels']
|
49
|
+
accepted_keys = ['default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels', 'label']
|
52
50
|
services_with_unacceptable_keys = []
|
53
51
|
@services.each do |path, loaded|
|
54
52
|
loaded.keys.each do |service_key|
|
@@ -59,26 +57,7 @@ module PropertyGenerator
|
|
59
57
|
end
|
60
58
|
if services_with_unacceptable_keys != []
|
61
59
|
status[:status] = 'fail'
|
62
|
-
status[:error] = "Service files: #{services_with_unacceptable_keys} have keys other than 'default', 'environments', 'encrypted', 'configname', 'stringdata' 'configlabels' or '
|
63
|
-
end
|
64
|
-
status
|
65
|
-
end
|
66
|
-
|
67
|
-
def service_defaults_have_no_hashes_as_values
|
68
|
-
status = {status: 'pass', error: ''}
|
69
|
-
services_with_hashes_in_defaults = []
|
70
|
-
@services.each do |path, loaded|
|
71
|
-
unless loaded['default'] == nil
|
72
|
-
loaded['default'].each do |defaultkey, defaultvalue|
|
73
|
-
if defaultvalue.class == Hash
|
74
|
-
services_with_hashes_in_defaults << {path => defaultkey}
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
if services_with_hashes_in_defaults != []
|
80
|
-
status[:status] = 'fail'
|
81
|
-
status[:error] = "Service files: #{services_with_hashes_in_defaults} have default properties with values as hashes."
|
60
|
+
status[:error] = "Service files: #{services_with_unacceptable_keys} have keys other than 'default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels' or 'label'"
|
82
61
|
end
|
83
62
|
status
|
84
63
|
end
|
@@ -107,29 +86,6 @@ module PropertyGenerator
|
|
107
86
|
status
|
108
87
|
end
|
109
88
|
|
110
|
-
def service_environments_have_no_hashes_as_values
|
111
|
-
status = {status: 'pass', error: ''}
|
112
|
-
services_with_hashes_in_environments = []
|
113
|
-
@services.each do |path, loaded|
|
114
|
-
unless loaded['environments'] == nil
|
115
|
-
loaded['environments'].each do |environments, properties|
|
116
|
-
unless properties == nil
|
117
|
-
properties.each do |key, value|
|
118
|
-
if value.class == Hash
|
119
|
-
services_with_hashes_in_environments << {path => {environments => key}}
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
123
|
-
end
|
124
|
-
end
|
125
|
-
end
|
126
|
-
if services_with_hashes_in_environments != []
|
127
|
-
status[:status] = 'fail'
|
128
|
-
status[:error] = "Service files #{services_with_hashes_in_environments} have environment properties with values as hashes."
|
129
|
-
end
|
130
|
-
status
|
131
|
-
end
|
132
|
-
|
133
89
|
def service_encrypted_environments_match_config_environments
|
134
90
|
status = {status: 'pass', error: ''}
|
135
91
|
missmatched_environments = []
|
@@ -156,7 +112,7 @@ module PropertyGenerator
|
|
156
112
|
|
157
113
|
def service_encrypted_fields_are_correct
|
158
114
|
status = {status: 'pass', error: ''}
|
159
|
-
accepted_keys = ['region', 'encrypted', 'service']
|
115
|
+
accepted_keys = ['region', 'encrypted', 'service', 'label']
|
160
116
|
services_with_unacceptable_keys = []
|
161
117
|
@services.each do |path, loaded|
|
162
118
|
if loaded['encrypted'] != nil
|
@@ -181,7 +137,7 @@ module PropertyGenerator
|
|
181
137
|
end
|
182
138
|
if services_with_unacceptable_keys != []
|
183
139
|
status[:status] = 'fail'
|
184
|
-
status[:error] = "Service files: #{services_with_unacceptable_keys} have encrypted properties with bad indentation or keys other than 'region'
|
140
|
+
status[:error] = "Service files: #{services_with_unacceptable_keys} have encrypted properties with bad indentation or keys other than 'region', 'encrypted' or 'label'."
|
185
141
|
end
|
186
142
|
status
|
187
143
|
end
|
data/spec/lib/global_spec.rb
CHANGED
@@ -8,7 +8,7 @@ module PropertyGenerator
|
|
8
8
|
subject(:global) {described_class.new(File.expand_path("./spec/resources"), config)}
|
9
9
|
|
10
10
|
it 'should read the main global file' do
|
11
|
-
expect(global.get_main_global).to eq({'foo'=>'bar'})
|
11
|
+
expect(global.get_main_global).to eq({'foo'=>'bar', 'map' => {'key1' => 'val1', 'key2' => 'val2', 'key4' => '{domain}'}})
|
12
12
|
end
|
13
13
|
|
14
14
|
it 'should read the account globals' do
|
@@ -20,9 +20,31 @@ module PropertyGenerator
|
|
20
20
|
end
|
21
21
|
|
22
22
|
it 'should condense the globals accurately' do
|
23
|
-
expect(global.condense_globals).to eq({'my-test-env1'=>{
|
24
|
-
|
23
|
+
expect(global.condense_globals).to eq({'my-test-env1'=>{
|
24
|
+
'foo' => 'bar',
|
25
|
+
'map' => {
|
26
|
+
'key1' => 'val1',
|
27
|
+
'key2' => 'val2',
|
28
|
+
'key4' => '{domain}'
|
29
|
+
},
|
30
|
+
'my_account'=>123456789012,
|
31
|
+
'my_env'=>'my-test-env1',
|
32
|
+
'test_encrypted' => {
|
33
|
+
'$ssm' => {
|
34
|
+
'region' => 'region',
|
35
|
+
'encrypted' => 'encrypted_value'
|
36
|
+
}
|
37
|
+
}
|
38
|
+
},
|
39
|
+
'my-test-env2' => {
|
40
|
+
'foo' => 'bar',
|
41
|
+
'map' => {
|
42
|
+
'key1' => 'val1',
|
43
|
+
'key2' => 'val2',
|
44
|
+
'key4' => '{domain}'
|
45
|
+
}
|
46
|
+
}})
|
25
47
|
end
|
26
48
|
|
27
49
|
end
|
28
|
-
end
|
50
|
+
end
|
data/spec/lib/service_spec.rb
CHANGED
@@ -5,33 +5,83 @@ require_relative "../../lib/generator/config"
|
|
5
5
|
require "pp"
|
6
6
|
module PropertyGenerator
|
7
7
|
describe Service do
|
8
|
-
subject(:config) {PropertyGenerator::Config.new(File.expand_path("./spec/resources"))}
|
9
|
-
subject(:globals) {PropertyGenerator::Globals.new(File.expand_path("./spec/resources"), config)}
|
10
|
-
subject(:service) {described_class.new(YAML.load_file("./spec/resources/services/my-microservice-1.yml"), config, globals.globals)}
|
8
|
+
subject(:config) { PropertyGenerator::Config.new(File.expand_path("./spec/resources")) }
|
9
|
+
subject(:globals) { PropertyGenerator::Globals.new(File.expand_path("./spec/resources"), config) }
|
10
|
+
subject(:service) { described_class.new(YAML.load_file("./spec/resources/services/my-microservice-1.yml"), config, globals.globals) }
|
11
11
|
|
12
|
-
it "Parses and condenses a service\"s defaults and environment definitions"
|
13
|
-
expect(service.service).to eq({
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
12
|
+
it "Parses and condenses a service\"s defaults and environment definitions" do
|
13
|
+
expect(service.service).to eq({
|
14
|
+
"my-test-env1" => {
|
15
|
+
"foo" => "bar",
|
16
|
+
"map" => {
|
17
|
+
"key1" => "notval1",
|
18
|
+
"key2" => "val2",
|
19
|
+
"key3" => "{cloud}-{region}",
|
20
|
+
"key4" => "{domain}",
|
21
|
+
"arr" => %w[one two {domain}]
|
22
|
+
},
|
23
|
+
"my_account" => 123456789012,
|
24
|
+
"my_env" => "my-test-env1",
|
25
|
+
"test_encrypted" => {
|
26
|
+
"$ssm" => {
|
27
|
+
"region" => "region",
|
28
|
+
"encrypted" => "encrypted_value"
|
29
|
+
}
|
30
|
+
},
|
31
|
+
"database.host" => "my.database.{domain}",
|
32
|
+
"database.port" => 3306,
|
33
|
+
"thread.pool.size" => 12
|
34
|
+
},
|
35
|
+
"my-test-env2" => {
|
36
|
+
"foo" => "bar",
|
37
|
+
"map" => {
|
38
|
+
"key1" => "notval1",
|
39
|
+
"key2" => "val2",
|
40
|
+
"key3" => "{cloud}-{region}",
|
41
|
+
"key4" => "{domain}",
|
42
|
+
"arr" => %w[one two {domain}]
|
43
|
+
},
|
44
|
+
"database.host" => "my.database.{domain}",
|
45
|
+
"database.port" => 3306,
|
46
|
+
"thread.pool.size" => 8,
|
47
|
+
"new_arr" => %w[{region} {cloud} {domain}]
|
48
|
+
}
|
49
|
+
})
|
22
50
|
end
|
23
51
|
|
24
52
|
it "Tests interpolations work for a service" do
|
25
|
-
expect(service.interpolate).to eq({
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
53
|
+
expect(service.interpolate).to eq({
|
54
|
+
"my-test-env1" => {
|
55
|
+
"foo" => "bar",
|
56
|
+
"map" => {
|
57
|
+
"key1" => "notval1",
|
58
|
+
"key2" => "val2",
|
59
|
+
"key3" => "test-cloud-1-us-east-1",
|
60
|
+
"key4" => "my1.com",
|
61
|
+
"arr" => %w[one two my1.com]
|
62
|
+
},
|
63
|
+
"my_account" => 123456789012,
|
64
|
+
"my_env" => "my-test-env1",
|
65
|
+
"test_encrypted" => { "$ssm" => { "region" => "region", "encrypted" => "encrypted_value" } },
|
66
|
+
"database.host" => "my.database.my1.com",
|
67
|
+
"database.port" => 3306,
|
68
|
+
"thread.pool.size" => 12
|
69
|
+
},
|
70
|
+
"my-test-env2" => {
|
71
|
+
"foo" => "bar",
|
72
|
+
"map" => {
|
73
|
+
"key1" => "notval1",
|
74
|
+
"key2" => "val2",
|
75
|
+
"key3" => "test-cloud-2-eu-central-1",
|
76
|
+
"key4" => "my2.com",
|
77
|
+
"arr" => %w[one two my2.com]
|
78
|
+
},
|
79
|
+
"database.host" => "my.database.my2.com",
|
80
|
+
"database.port" => 3306,
|
81
|
+
"thread.pool.size" => 8,
|
82
|
+
"new_arr" => %w[eu-central-1 test-cloud-2 my2.com]
|
83
|
+
} })
|
34
84
|
end
|
35
85
|
|
36
86
|
end
|
37
|
-
end
|
87
|
+
end
|
@@ -1,16 +1,27 @@
|
|
1
1
|
default:
|
2
2
|
database.host: 'my.database.{domain}'
|
3
3
|
database.port: 3306
|
4
|
+
map:
|
5
|
+
key1: notval1
|
6
|
+
key3: '{cloud}-{region}'
|
7
|
+
arr:
|
8
|
+
- one
|
9
|
+
- two
|
10
|
+
- '{domain}'
|
4
11
|
|
5
12
|
environments:
|
6
|
-
my-test-
|
13
|
+
my-test-env1:
|
7
14
|
thread.pool.size: 12
|
8
|
-
my-test-
|
15
|
+
my-test-env2:
|
9
16
|
thread.pool.size: 8
|
17
|
+
new_arr:
|
18
|
+
- '{region}'
|
19
|
+
- '{cloud}'
|
20
|
+
- '{domain}'
|
10
21
|
|
11
22
|
#encrypted:
|
12
|
-
# my-test-
|
23
|
+
# my-test-env1:
|
13
24
|
# my.encrypted.property:
|
14
25
|
# $ssm:
|
15
26
|
# region: us-east-1
|
16
|
-
# encrypted: PRETEND_ENCRYPTED_PROPERTY_CIPHERTEXT
|
27
|
+
# encrypted: PRETEND_ENCRYPTED_PROPERTY_CIPHERTEXT
|