cps-property-generator 0.2.14 → 0.2.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a92aa70bf1eb0d003090e5cb5d7d42d6e3fd2353
-  data.tar.gz: d313236d4357af37d7ceca7e4e1703cc982971ee
+  metadata.gz: 2f659265beb057cc6e807215df12702562307e50
+  data.tar.gz: 58a223834ecda08eea38a5f9a21ad6cbdf39f37d
 SHA512:
-  metadata.gz: 057bd3529819bdcb57760b5692b5f8be89bcd61b00ae0ebbe703030f50c80d534717029dabfea0b0d7eb5f76ae4dc3a325066b39bd0cacba02d6c76914addd4c
-  data.tar.gz: beca6f7166b0b8eaa71598433f1b0f81eae384e42ffe31d9a6524f1c3c01ef62e2cbe187d5acadc352085218c629c7848b47df0cfe9eca60332126c7f2adf06d
+  metadata.gz: dbda7d164c992be4eab086b2f770dee1c8adbefe259ae6c12da695ba09d6f3c8ffbaa90a76e51e7572cb2143d7c26609c3ac39231d59a4af964daff555267e07
+  data.tar.gz: bf99d2880a883f34fc8047a61fff2b55c51307faa71d450780a30f01ef29f40d882c622e3d547df0e27e7d3ca127e4f73b12ce886c338c7971948e23f05284ff

data/bin/cps-property-generator

@@ -13,17 +13,19 @@ class GeneratorCLI < ::Thor
   option 'upload_account', banner: 'UPLOAD_ACCOUNT', type: :string, desc: 'The account you are uploading properties to'
   option 'upload_region', banner: 'UPLOAD_REGION', type: :string, desc: 'The region your property bucket is in'
   option 'upload_bucket', banner: 'UPLOAD_BUCKET', type: :string, desc: 'The bucket you are uploading properties to.'
+  option 'upload_all', banner: 'UPLOAD_ALL', type: :boolean, desc: 'Whether to upload all envs and accounts to a single bucket', :default => false
 
   def generate
 
     generator = PropertyGenerator::Generator.new(options)
     out = generator.generate
     if options['upload']
-      if options['upload_account'].nil? || options['upload_region'].nil? && options['upload_bucket'].nil?
-        puts 'Please specify upload configs'
-      else
-        generator.upload(out, options)
-      end
+      abort('Did not specify an upload bucket') if options['upload_bucket'].nil?
+      abort('Did not specify an upload region') if options['upload_region'].nil?
+
+      abort('Did not specify upload configs') if !options['upload_all'] && options['upload_account'].nil?
+
+      generator.upload(out, options)
     end
   end
 

data/lib/generator/generator.rb

@@ -37,22 +37,33 @@ module PropertyGenerator
     end
 
     def upload(out, config)
-      account = config['upload_account']
+      upload_bucket = config['upload_bucket']
+      upload_region = config['upload_region']
 
-      if !@accounts.include?(account.to_i)
-        abort("The specified account (#{account}) is not configured, please add it to config/config.yml")
-      end
+      if config['upload_all']
+        _upload_files(out.sort) do |file|
+          file_region = file.split("/")[-2]
+          file_account = file.split("/")[-3]
 
-      upload_account = config['upload_account']
-      upload_region = config['upload_region']
-      upload_bucket = config['upload_bucket']
+          PropertyGenerator.sync(upload_region, file_account, upload_bucket, file, file_region)
+        end
+      else
+        upload_account = config['upload_account']
+        abort("The specified account (#{upload_account}) is not configured, please add it to config/config.yml") unless @accounts.include?(upload_account)
+
+        upload_out = out.select { |file| file.include?("#{upload_account}") && file.include?("#{upload_region}") }
+        _upload_files(upload_out) do |file|
+          file_region = file.split("/")[-2]
+          PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
+        end
+      end
+    end
 
-      upload_out = out.select { |file| file.include?("#{upload_account}") && file.include?("#{upload_region}") }
-      upload_out.each_slice(20) do |file_slice|
+    def _upload_files(files)
+      files.each_slice(20) do |file_slice|
         file_slice.map do |file|
           Thread.new do
-            file_region = file.split("/")[-2]
-            PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
+            yield file
           end
         end.each(&:join)
       end

data/lib/generator/globals.rb

@@ -44,7 +44,7 @@ module PropertyGenerator
           unless data[account][env]['encrypted'].nil?
             encrypted = data[account][env]['encrypted'].dup
             not_encrypted = data[account][env].reject { |k,_| k == 'encrypted' }
-            data[account][env] = not_encrypted.merge(encrypted)
+            data[account][env] = not_encrypted.deep_merge(encrypted)
           end
         end
       end
@@ -67,7 +67,7 @@ module PropertyGenerator
         env_global.each do |env, hash|
           account_globals[account] ||= {}
           # set the environment globals to be the account global merged with the env globals
-          env_global[env] = account_globals[account].merge(hash) unless hash.empty?
+          env_global[env] = account_globals[account].deep_merge(hash) unless hash.empty?
           condensed[env] = env_global[env]
         end
       end
@@ -80,7 +80,7 @@ module PropertyGenerator
           # We need to merge into the globals so any env configs overwrite main global configs.
           # Dup so we dont modify the original object
           main_global_dup = main_global.dup
-          condensed[env] = main_global_dup.merge(condensed[env])
+          condensed[env] = main_global_dup.deep_merge(condensed[env])
         end
       end
       condensed

data/lib/generator/service.rb

@@ -50,7 +50,9 @@ module PropertyGenerator
 
         # Recursively interate through the properties for an environment and gsub the config
         # with defined interpolations.
-        interpolate_nested_properties(@service[env], interpolations)
+        service_env = Marshal.load(Marshal.dump(@service[env]))
+        interpolate_nested_properties(service_env, interpolations)
+        @service[env] = service_env
       end
       service
     end
@@ -58,12 +60,17 @@ module PropertyGenerator
     def interpolate_nested_properties(service_env, interpolations)
       interpolations.each do |matcher_key, matcher_value|
         service_env.each { |k,v|  service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}")}
-        service_env.values.each { |v| interpolate_nested_properties(v, interpolations)  if v.class == Hash }
+        service_env.values.each do |v|
+          interpolate_nested_properties(v, interpolations) if v.class == Hash
+          v.each_with_index do |val, idx|
+            v[idx] = val.gsub("{#{matcher_key}}", matcher_value) if val.class == String && val.include?("{#{matcher_key}}")
+          end if v.class == Array
+        end
       end
     end
 
     def merge_env_default(data, environments)
-      #creates a hash of the enviornments merged with the defaults
+      #creates a hash of the environments merged with the defaults
       # {service => {env1 =>  {properties},
       #             env2 => {properties}
       #           }
@@ -84,7 +91,7 @@ module PropertyGenerator
         if default_clone.nil?
           merged = environment_data
         else
-          merged = default_clone.merge(environment_data)
+          merged = default_clone.deep_merge(environment_data)
         end
         output[env] = merged
       end
@@ -100,7 +107,7 @@ module PropertyGenerator
         if globals_clone[env].nil? || globals_clone[env] == false
           merged = service_data[env]
         else
-          merged = globals_clone[env].merge(service_data[env])
+          merged = globals_clone[env].deep_merge(service_data[env])
         end
         output[env] = merged
       end

data/lib/linter/globals_linter.rb

@@ -14,7 +14,6 @@ module PropertyGenerator
 
     def run_globals_tests
       tests = ['globals_load_as_hashes',
-               'globals_have_no_hashes_as_values',
                'globals_are_defined_for_valid_environemnts',
                ]
       results = PropertyGenerator.test_runner(self, tests)

data/lib/linter/services_linter.rb

@@ -16,9 +16,7 @@ module PropertyGenerator
       tests = [
         'services_have_accepted_keys',
         'service_environments_are_not_empty',
-        'service_defaults_have_no_hashes_as_values',
         'service_environments_match_config_environments',
-        'service_environments_have_no_hashes_as_values',
         'service_encrypted_environments_match_config_environments',
         'service_encrypted_fields_are_correct',
         'service_encrypted_region_field_is_accepted'
@@ -48,7 +46,7 @@ module PropertyGenerator
 
     def services_have_accepted_keys
       status = {status: 'pass', error: ''}
-      accepted_keys = ['default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels']
+      accepted_keys = ['default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels', 'label']
       services_with_unacceptable_keys = []
       @services.each do |path, loaded|
         loaded.keys.each do |service_key|
@@ -59,26 +57,7 @@ module PropertyGenerator
       end
       if services_with_unacceptable_keys != []
         status[:status] = 'fail'
-        status[:error] = "Service files: #{services_with_unacceptable_keys} have keys other than 'default', 'environments', 'encrypted', 'configname', 'stringdata' 'configlabels' or 'secretlabels'"
-      end
-      status
-    end
-
-    def service_defaults_have_no_hashes_as_values
-      status = {status: 'pass', error: ''}
-      services_with_hashes_in_defaults = []
-      @services.each do |path, loaded|
-        unless loaded['default'] == nil
-          loaded['default'].each do |defaultkey, defaultvalue|
-            if defaultvalue.class == Hash
-              services_with_hashes_in_defaults << {path => defaultkey}
-            end
-          end
-        end
-      end
-      if services_with_hashes_in_defaults != []
-        status[:status] = 'fail'
-        status[:error] = "Service files: #{services_with_hashes_in_defaults} have default properties with values as hashes."
+        status[:error] = "Service files: #{services_with_unacceptable_keys} have keys other than 'default', 'environments', 'encrypted', 'configname', 'stringdata', 'configlabels', 'secretlabels' or 'label'"
       end
       status
     end
@@ -107,29 +86,6 @@ module PropertyGenerator
       status
     end
 
-    def service_environments_have_no_hashes_as_values
-      status = {status: 'pass', error: ''}
-      services_with_hashes_in_environments = []
-      @services.each do |path, loaded|
-        unless loaded['environments'] == nil
-          loaded['environments'].each do |environments, properties|
-            unless properties == nil
-              properties.each do |key, value|
-                if value.class == Hash
-                  services_with_hashes_in_environments << {path => {environments => key}}
-                end
-              end
-            end
-          end
-        end
-      end
-      if services_with_hashes_in_environments != []
-        status[:status] = 'fail'
-        status[:error] = "Service files #{services_with_hashes_in_environments} have environment properties with values as hashes."
-      end
-      status
-    end
-
     def service_encrypted_environments_match_config_environments
       status = {status: 'pass', error: ''}
       missmatched_environments = []
@@ -156,7 +112,7 @@ module PropertyGenerator
 
     def service_encrypted_fields_are_correct
       status = {status: 'pass', error: ''}
-      accepted_keys = ['region', 'encrypted', 'service']
+      accepted_keys = ['region', 'encrypted', 'service', 'label']
       services_with_unacceptable_keys = []
       @services.each do |path, loaded|
         if loaded['encrypted'] != nil
@@ -181,7 +137,7 @@ module PropertyGenerator
       end
       if services_with_unacceptable_keys != []
         status[:status] = 'fail'
-        status[:error] = "Service files: #{services_with_unacceptable_keys} have encrypted properties with bad indentation or keys other than 'region' and 'encrypted'."
+        status[:error] = "Service files: #{services_with_unacceptable_keys} have encrypted properties with bad indentation or keys other than 'region', 'encrypted' or 'label'."
       end
       status
     end

data/spec/lib/global_spec.rb

@@ -8,7 +8,7 @@ module PropertyGenerator
     subject(:global) {described_class.new(File.expand_path("./spec/resources"), config)}
 
     it 'should read the main global file' do
-      expect(global.get_main_global).to eq({'foo'=>'bar'})
+      expect(global.get_main_global).to eq({'foo'=>'bar', 'map' => {'key1' => 'val1', 'key2' => 'val2', 'key4' => '{domain}'}})
     end
 
     it 'should read the account globals' do
@@ -20,9 +20,31 @@ module PropertyGenerator
     end
 
     it 'should condense the globals accurately' do
-      expect(global.condense_globals).to eq({'my-test-env1'=>{'foo' => 'bar', 'my_account'=>123456789012, 'my_env'=>'my-test-env1', 'test_encrypted' => { '$ssm' => { 'region' => 'region', 'encrypted' => 'encrypted_value' }}},
-                                             'my-test-env2' => {'foo' => 'bar'}})
+      expect(global.condense_globals).to eq({'my-test-env1'=>{
+        'foo' => 'bar',
+        'map' => {
+          'key1' => 'val1',
+          'key2' => 'val2',
+          'key4' => '{domain}'
+        },
+        'my_account'=>123456789012,
+        'my_env'=>'my-test-env1',
+        'test_encrypted' => {
+          '$ssm' => {
+            'region' => 'region',
+            'encrypted' => 'encrypted_value'
+          }
+        }
+      },
+      'my-test-env2' => {
+        'foo' => 'bar',
+        'map' => {
+          'key1' => 'val1',
+          'key2' => 'val2',
+          'key4' => '{domain}'
+        }
+      }})
     end
 
   end
-end
+end

data/spec/lib/service_spec.rb

@@ -5,33 +5,83 @@ require_relative "../../lib/generator/config"
 require "pp"
 module PropertyGenerator
   describe Service do
-    subject(:config) {PropertyGenerator::Config.new(File.expand_path("./spec/resources"))}
-    subject(:globals) {PropertyGenerator::Globals.new(File.expand_path("./spec/resources"), config)}
-    subject(:service) {described_class.new(YAML.load_file("./spec/resources/services/my-microservice-1.yml"), config, globals.globals)}
+    subject(:config) { PropertyGenerator::Config.new(File.expand_path("./spec/resources")) }
+    subject(:globals) { PropertyGenerator::Globals.new(File.expand_path("./spec/resources"), config) }
+    subject(:service) { described_class.new(YAML.load_file("./spec/resources/services/my-microservice-1.yml"), config, globals.globals) }
 
-    it "Parses and condenses a service\"s defaults and environment definitions"  do
-      expect(service.service).to eq({"my-test-env1"=> {"foo"=>"bar",
-                                                       "my_account"=>123456789012,
-                                                       "my_env"=>"my-test-env1",
-                                                       "test_encrypted" => { "$ssm" => { "region" => "region", "encrypted" => "encrypted_value" }},
-                                                       "database.host"=>"my.database.{domain}",
-                                                       "database.port"=>3306},
-                                     "my-test-env2"=> {"foo"=>"bar",
-                                                       "database.host"=>"my.database.{domain}",
-                                                       "database.port"=>3306}})
+    it "Parses and condenses a service\"s defaults and environment definitions" do
+      expect(service.service).to eq({
+                                      "my-test-env1" => {
+                                        "foo" => "bar",
+                                        "map" => {
+                                          "key1" => "notval1",
+                                          "key2" => "val2",
+                                          "key3" => "{cloud}-{region}",
+                                          "key4" => "{domain}",
+                                          "arr" => %w[one two {domain}]
+                                        },
+                                        "my_account" => 123456789012,
+                                        "my_env" => "my-test-env1",
+                                        "test_encrypted" => {
+                                          "$ssm" => {
+                                            "region" => "region",
+                                            "encrypted" => "encrypted_value"
+                                          }
+                                        },
+                                        "database.host" => "my.database.{domain}",
+                                        "database.port" => 3306,
+                                        "thread.pool.size" => 12
+                                      },
+                                      "my-test-env2" => {
+                                        "foo" => "bar",
+                                        "map" => {
+                                          "key1" => "notval1",
+                                          "key2" => "val2",
+                                          "key3" => "{cloud}-{region}",
+                                          "key4" => "{domain}",
+                                          "arr" => %w[one two {domain}]
+                                        },
+                                        "database.host" => "my.database.{domain}",
+                                        "database.port" => 3306,
+                                        "thread.pool.size" => 8,
+                                        "new_arr" => %w[{region} {cloud} {domain}]
+                                      }
+                                    })
     end
 
     it "Tests interpolations work for a service" do
-      expect(service.interpolate).to eq({"my-test-env1"=> {"foo"=>"bar",
-                                                           "my_account"=>123456789012,
-                                                           "my_env"=>"my-test-env1",
-                                                           "test_encrypted" => { "$ssm" => { "region" => "region", "encrypted" => "encrypted_value" }},
-                                                           "database.host"=>"my.database.my1.com",
-                                                           "database.port"=>3306},
-                                         "my-test-env2"=> {"foo"=>"bar",
-                                                           "database.host"=>"my.database.my2.com",
-                                                           "database.port"=>3306}})
+      expect(service.interpolate).to eq({
+                                          "my-test-env1" => {
+                                            "foo" => "bar",
+                                            "map" => {
+                                              "key1" => "notval1",
+                                              "key2" => "val2",
+                                              "key3" => "test-cloud-1-us-east-1",
+                                              "key4" => "my1.com",
+                                              "arr" => %w[one two my1.com]
+                                            },
+                                            "my_account" => 123456789012,
+                                            "my_env" => "my-test-env1",
+                                            "test_encrypted" => { "$ssm" => { "region" => "region", "encrypted" => "encrypted_value" } },
+                                            "database.host" => "my.database.my1.com",
+                                            "database.port" => 3306,
+                                            "thread.pool.size" => 12
+                                          },
+                                          "my-test-env2" => {
+                                            "foo" => "bar",
+                                            "map" => {
+                                              "key1" => "notval1",
+                                              "key2" => "val2",
+                                              "key3" => "test-cloud-2-eu-central-1",
+                                              "key4" => "my2.com",
+                                              "arr" => %w[one two my2.com]
+                                            },
+                                            "database.host" => "my.database.my2.com",
+                                            "database.port" => 3306,
+                                            "thread.pool.size" => 8,
+                                            "new_arr" => %w[eu-central-1 test-cloud-2 my2.com]
+                                          } })
     end
 
   end
-end
+end

data/spec/resources/globals/globals.yml

@@ -1 +1,5 @@
-foo: "bar"
+foo: "bar"
+map:
+  key1: val1
+  key2: val2
+  key4: '{domain}'

data/spec/resources/services/my-microservice-1.yml

@@ -1,16 +1,27 @@
 default:
   database.host: 'my.database.{domain}'
   database.port: 3306
+  map:
+    key1: notval1
+    key3: '{cloud}-{region}'
+    arr:
+      - one
+      - two
+      - '{domain}'
 
 environments:
-  my-test-env-1:
+  my-test-env1:
     thread.pool.size: 12
-  my-test-env-2:
+  my-test-env2:
     thread.pool.size: 8
+    new_arr:
+      - '{region}'
+      - '{cloud}'
+      - '{domain}'
 
 #encrypted:
-#  my-test-env-1:
+#  my-test-env1:
 #   my.encrypted.property:
 #      $ssm:
 #        region: us-east-1
-#        encrypted: PRETEND_ENCRYPTED_PROPERTY_CIPHERTEXT
+#        encrypted: PRETEND_ENCRYPTED_PROPERTY_CIPHERTEXT

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cps-property-generator
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.20
 platform: ruby
 authors:
 - Bryan Call