cps-property-generator 0.2.16 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84b92342668cccfdeb1e097c19fdd1641fef1589
-  data.tar.gz: d4c5c0bf42cbbec4a3a2203a4fbdd990f88a9a97
+  metadata.gz: ee35c7e51b0de1094566e73a39784a663442903c
+  data.tar.gz: 16ade81e5e6922856112a7516dd768e029935374
 SHA512:
-  metadata.gz: 4883e4c5fef3a9ea680b6401c9b3629371f16b967ab634a9fb2c041ee60940cab2e16c5469a2191a7964ee947fc80a6b190862aa3b0193caeb22c87be29849ba
-  data.tar.gz: 9381609f8a882ade931cb10c438b0fb6b9ad3619e4620d80929b8d52693bb2fdd6d245e65e68ebe805a5108ba444225976c565d82152d18c3eca9c5b5193b0de
+  metadata.gz: 81d1e90f92865481d6c6ce7242c0bba8c1802ab92ef8c04b961310ef4aa730b50358a68addc1912530ebc045aa3edd77525dfeffcda9bb6be83096a12f334a4c
+  data.tar.gz: 4363ca29e85b30e93ab04615fabc809cc285592f9e3f49e9b640ec36ce5bd81e59be05f533c1556d49ae9b4fd8c9c365f0d9b950963743901a7740c0f4d3a521

data/bin/cps-property-generator

@@ -1,11 +1,10 @@
 #!/usr/bin/env ruby
-#
 require 'thor'
 require 'yaml'
-require_relative '../lib/generator/generator.rb'
-require_relative '../lib/linter/linter.rb'
-class GeneratorCLI < ::Thor
+require_relative '../lib/generator/generator'
+require_relative '../lib/linter/linter'
 
+class GeneratorCLI < ::Thor
   desc 'generate', 'Generate properties'
   option 'project_path', banner: 'PROJECT_PATH', type: :string, desc: 'Path to the property project to generate properties for'
   option 'output', banner: 'OUTPUT', type: :string, desc: 'Output path for locally dumping generated outputs', :default => '/tmp/'
@@ -13,17 +12,18 @@ class GeneratorCLI < ::Thor
   option 'upload_account', banner: 'UPLOAD_ACCOUNT', type: :string, desc: 'The account you are uploading properties to'
   option 'upload_region', banner: 'UPLOAD_REGION', type: :string, desc: 'The region your property bucket is in'
   option 'upload_bucket', banner: 'UPLOAD_BUCKET', type: :string, desc: 'The bucket you are uploading properties to.'
+  option 'upload_all', banner: 'UPLOAD_ALL', type: :boolean, desc: 'Whether to upload all envs and accounts to a single bucket', :default => false
 
   def generate
-
     generator = PropertyGenerator::Generator.new(options)
     out = generator.generate
     if options['upload']
-      if options['upload_account'].nil? || options['upload_region'].nil? && options['upload_bucket'].nil?
-        puts 'Please specify upload configs'
-      else
-        generator.upload(out, options)
-      end
+      abort('Did not specify an upload bucket') if options['upload_bucket'].nil?
+      abort('Did not specify an upload region') if options['upload_region'].nil?
+
+      abort('Did not specify upload configs') if !options['upload_all'] && options['upload_account'].nil?
+
+      generator.upload(out, options)
     end
   end
 

data/lib/generator/config.rb

@@ -1,7 +1,6 @@
 module PropertyGenerator
   require 'yaml'
   class Config
-
     attr_accessor :configs
 
     def initialize(project_path)
@@ -29,4 +28,4 @@ module PropertyGenerator
       YAML.load_file(file)
     end
   end
-end
+end

data/lib/generator/generator.rb

@@ -11,20 +11,20 @@ module PropertyGenerator
     # purpose: initialize globals and configs
     # serve as a broker between tasks
     def initialize(options)
-      project_path =  File.expand_path(options['project_path'])
+      project_path = File.expand_path(options['project_path'])
       @configs = PropertyGenerator::Config.new(project_path)
       @globals = PropertyGenerator::Globals.new(project_path, @configs)
       @globals = @globals.globals
       @accounts = @configs.accounts
 
-      @output_path =  "#{File.expand_path(options['output'])}/properties/#{SecureRandom.hex}"
+      @output_path = "#{File.expand_path(options['output'])}/properties/#{SecureRandom.hex}"
       puts "Properties will be output here #{@output_path}"
       @service_list = PropertyGenerator.read_services(project_path)
     end
 
     def generate
       output = []
-      @service_list.each do | service, path|
+      @service_list.each do |service, path|
         PropertyGenerator.config_enforcer(@configs.environment_configs)
         service_instance = PropertyGenerator::Service.new(YAML.load_file(path), @configs, @globals)
         service_instance.service
@@ -37,27 +37,38 @@ module PropertyGenerator
     end
 
     def upload(out, config)
-      account = config['upload_account']
+      upload_bucket = config['upload_bucket']
+      upload_region = config['upload_region']
 
-      if !@accounts.include?(account.to_i)
-        abort("The specified account (#{account}) is not configured, please add it to config/config.yml")
-      end
+      if config['upload_all']
+        _upload_files(out.sort) do |file|
+          file_region = file.split('/')[-2]
+          file_account = file.split('/')[-3]
 
-      upload_account = config['upload_account']
-      upload_region = config['upload_region']
-      upload_bucket = config['upload_bucket']
+          PropertyGenerator.sync(upload_region, file_account, upload_bucket, file, file_region)
+        end
+      else
+        upload_account = config['upload_account'].strip
+        unless @accounts.map { |a| a.to_s.strip }.include?(upload_account)
+          abort("The specified account (#{upload_account}) is not configured, please add it to config/config.yml")
+        end
 
-      upload_out = out.select { |file| file.include?("#{upload_account}") && file.include?("#{upload_region}") }
-      upload_out.each_slice(20) do |file_slice|
+        upload_out = out.select { |file| file.include?(upload_account) && file.include?(upload_region) }
+        _upload_files(upload_out) do |file|
+          file_region = file.split('/')[-2]
+          PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
+        end
+      end
+    end
+
+    def _upload_files(files)
+      files.each_slice(20) do |file_slice|
         file_slice.map do |file|
           Thread.new do
-            file_region = file.split("/")[-2]
-            PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
+            yield file
           end
         end.each(&:join)
       end
     end
-
   end
 end
-

data/lib/generator/globals.rb

@@ -14,10 +14,9 @@ module PropertyGenerator
       @globals ||= condense_globals
     end
 
-
     def get_main_global
       top_level = {}
-      if File.exists?("#{@project_path}/globals/globals.yml")
+      if File.exist?("#{@project_path}/globals/globals.yml")
         top_level = YAML.load_file("#{@project_path}/globals/globals.yml")
       end
       top_level
@@ -26,9 +25,10 @@ module PropertyGenerator
     def get_account_globals
       data = {}
       @accounts.each do |account|
-        next unless Dir.exists?("#{@project_path}/globals/accounts/#{account}")
+        next unless Dir.exist?("#{@project_path}/globals/accounts/#{account}")
+
         account_default_file = "#{@project_path}/globals/accounts/#{account}/#{account}.yml"
-        data[account] = YAML.load_file(account_default_file) if File.exists?(account_default_file)
+        data[account] = YAML.load_file(account_default_file) if File.exist?(account_default_file)
       end
       data
     end
@@ -36,23 +36,24 @@ module PropertyGenerator
     def get_environment_globals
       data = {}
       @accounts.each do |account|
-        next unless Dir.exists?("#{@project_path}/globals/accounts/#{account}/environments")
+        next unless Dir.exist?("#{@project_path}/globals/accounts/#{account}/environments")
+
         data[account] = {}
         @environments.each do |env|
-          next unless File.exists?("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
+          next unless File.exist?("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
+
           data[account][env] = YAML.load_file("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
           unless data[account][env]['encrypted'].nil?
             encrypted = data[account][env]['encrypted'].dup
-            not_encrypted = data[account][env].reject { |k,_| k == 'encrypted' }
-            data[account][env] = not_encrypted.merge(encrypted)
+            not_encrypted = data[account][env].reject { |k, _| k == 'encrypted' }
+            data[account][env] = not_encrypted.deep_merge(encrypted)
           end
         end
       end
       data
     end
 
-
-    #merge environment globals with account globals.
+    # merge environment globals with account globals.
     def condense_globals
       condensed = {}
       # get account and the environmental hash's for said account
@@ -62,12 +63,12 @@ module PropertyGenerator
       # nothing to do here if everything is empty
       return condensed if environment_globals.empty? && account_globals.empty? && main_global.empty?
 
-      environment_globals.each do |account, env_global |
+      environment_globals.each do |account, env_global|
         # get the env and the values
         env_global.each do |env, hash|
           account_globals[account] ||= {}
           # set the environment globals to be the account global merged with the env globals
-          env_global[env] = account_globals[account].merge(hash) unless hash.empty?
+          env_global[env] = account_globals[account].deep_merge(hash) unless hash.empty?
           condensed[env] = env_global[env]
         end
       end
@@ -80,11 +81,10 @@ module PropertyGenerator
           # We need to merge into the globals so any env configs overwrite main global configs.
           # Dup so we dont modify the original object
           main_global_dup = main_global.dup
-          condensed[env] = main_global_dup.merge(condensed[env])
+          condensed[env] = main_global_dup.deep_merge(condensed[env])
         end
       end
       condensed
     end
-
   end
 end

data/lib/generator/service.rb

@@ -1,9 +1,9 @@
 module PropertyGenerator
-
   class Service
     require 'active_support/core_ext/hash'
 
     attr_accessor :service
+    attr_reader :additional_options
 
     def initialize(service_data, config, globals)
       @service_data = service_data
@@ -28,42 +28,41 @@ module PropertyGenerator
       @additional_options['secretlabels'] = @service_data['secretlabels'].nil? ? nil : @service_data['secretlabels']
     end
 
-    def additional_options
-      @additional_options
-    end
-
-    def service
-      @service
-    end
-
     def configmap_name
       @configmapname
     end
 
     def interpolate
       environments = @environments
-      #read in config
-      #interate through environment and substitute config for values for that environment
-      environments.each do | env|
-        #get the map of config for a env
+      # read in config
+      # interate through environment and substitute config for values for that environment
+      environments.each do |env|
+        # get the map of config for a env
         interpolations = @environment_configs[env]['interpolations']
 
         # Recursively interate through the properties for an environment and gsub the config
         # with defined interpolations.
-        interpolate_nested_properties(@service[env], interpolations)
+        service_env = Marshal.load(Marshal.dump(@service[env]))
+        interpolate_nested_properties(service_env, interpolations)
+        @service[env] = service_env
       end
       service
     end
 
     def interpolate_nested_properties(service_env, interpolations)
       interpolations.each do |matcher_key, matcher_value|
-        service_env.each { |k,v|  service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}")}
-        service_env.values.each { |v| interpolate_nested_properties(v, interpolations)  if v.class == Hash }
+        service_env.each { |k, v| service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}") }
+        service_env.values.each do |v|
+          interpolate_nested_properties(v, interpolations) if v.class == Hash
+          v.each_with_index do |val, idx|
+            v[idx] = val.gsub("{#{matcher_key}}", matcher_value) if val.class == String && val.include?("{#{matcher_key}}")
+          end if v.class == Array
+        end
       end
     end
 
     def merge_env_default(data, environments)
-      #creates a hash of the enviornments merged with the defaults
+      # creates a hash of the environments merged with the defaults
       # {service => {env1 =>  {properties},
       #             env2 => {properties}
       #           }
@@ -73,7 +72,7 @@ module PropertyGenerator
 
       environments.each do |env|
         default_clone = default.dup
-        #if nil, use set to environments as nothing to merge env with
+        # if nil, use set to environments as nothing to merge env with
         data['environments'] ||= {}
         data['environments'][env] ||= {}
         environment_data = data['environments'][env].dup
@@ -84,7 +83,7 @@ module PropertyGenerator
         if default_clone.nil?
           merged = environment_data
         else
-          merged = default_clone.merge(environment_data)
+          merged = default_clone.deep_merge(environment_data)
         end
         output[env] = merged
       end
@@ -92,7 +91,7 @@ module PropertyGenerator
     end
 
     def merge_service_with_globals(globals_data, service_data, environments)
-      #service will now overwrite globals, merging will be done for each environment
+      # service will now overwrite globals, merging will be done for each environment
       output = {}
       envs = environments
       envs.each do |env|
@@ -100,13 +99,11 @@ module PropertyGenerator
         if globals_clone[env].nil? || globals_clone[env] == false
           merged = service_data[env]
         else
-          merged = globals_clone[env].merge(service_data[env])
+          merged = globals_clone[env].deep_merge(service_data[env])
         end
         output[env] = merged
       end
       output
     end
-
   end
-
 end

data/lib/helpers/helpers.rb

@@ -1,10 +1,10 @@
 require_relative '../helpers/helpers'
+
 module PropertyGenerator
   require 'json'
   require 'fileutils'
   require 'aws-sdk-s3'
   class << self
-
     def test_runner(object, test_list)
       results = {}
       test_list.each do |test|
@@ -14,9 +14,9 @@ module PropertyGenerator
     end
 
     def get_list_of_files(path, ignore_list)
-      #Returns a list of files in given path
-      #Ignores files in a given ignore list
-      Dir.glob(path + "/**/*").select{ |e| File.file? e unless ignore_list.include?(e.split('/')[(e.split('/')).length - 1])}
+      # Returns a list of files in given path
+      # Ignores files in a given ignore list
+      Dir.glob("#{path}/**/*").select { |e| File.file?(e) unless ignore_list.include?(e.split('/')[-1]) }
     end
 
     def valid_paths(path)
@@ -26,7 +26,7 @@ module PropertyGenerator
         begin
           YAML.load_file(file_path)
           valid_paths << file_path
-        rescue
+        rescue StandardError
           next
         end
       end
@@ -39,7 +39,7 @@ module PropertyGenerator
       list_of_file_paths.each do |file_path|
         begin
           YAML.load_file(file_path)
-        rescue
+        rescue StandardError
           invalid_paths << file_path
         end
       end
@@ -57,21 +57,21 @@ module PropertyGenerator
     def writer(service_name, finalized, configs, output_path, additional_options)
       output = []
       envs = configs.environments
-      environmental_configs =  configs.environment_configs
-      envs.each do | env|
-        account = environmental_configs[env]["account"]
-        region = environmental_configs[env]["region"]
-        hash = { "properties" => finalized[env] }
+      environmental_configs = configs.environment_configs
+      envs.each do |env|
+        account = environmental_configs[env]['account']
+        region = environmental_configs[env]['region']
+        hash = { 'properties' => finalized[env] }
         ['configname', 'stringdata', 'configlabels', 'secretlabels'].each do |setting|
-          hash[setting] = additional_options[setting] if [setting] unless additional_options[setting].nil?
+          hash[setting] = additional_options[setting] unless additional_options[setting].nil?
         end
         json = JSON.pretty_generate(hash)
-        #IF users are specifing a vpc then we will drop property files under the dir that corresponds to the vpc
-        if environmental_configs[env].key?("vpc") && !environmental_configs[env]["vpc"].nil?
-          vpc_dir = "#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}"
+        # IF users are specifing a vpc then we will drop property files under the dir that corresponds to the vpc
+        if environmental_configs[env].key?('vpc') && !environmental_configs[env]['vpc'].nil?
+          vpc_dir = "#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}"
           FileUtils.mkdir_p("#{vpc_dir}/") unless Dir.exist?(vpc_dir)
-          File.write("#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}/#{service_name}.json", json)
-          output << "#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}/#{service_name}.json"
+          File.write("#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}/#{service_name}.json", json)
+          output << "#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}/#{service_name}.json"
         else
           FileUtils.mkdir_p("#{output_path}/#{account}/#{region}/") unless Dir.exist?("#{output_path}/#{account}/#{region}/")
           File.write("#{output_path}/#{account}/#{region}/#{service_name}.json", json)
@@ -83,22 +83,22 @@ module PropertyGenerator
 
     def sync(region, account, bucket, file, file_region)
       s3 = Aws::S3::Resource.new(region: region)
-      filename = file.split("/").last
+      filename = file.split('/').last
       puts "Destination: #{account}/#{file_region}/#{filename}"
       puts "Uploading: #{file}"
       obj = s3.bucket(bucket).object("#{account}/#{file_region}/#{filename}")
       obj.upload_file(file)
     end
 
-    #Force users to specify VPCs for all environments if specified for one environment.
-    #This allows us to skip having conditional logic downstream in CPS requests
+    # Force users to specify VPCs for all environments if specified for one environment.
+    # This allows us to skip having conditional logic downstream in CPS requests
     def config_enforcer(environment_configs)
       a_vpc_exists = false
       environment_configs.each do |environment, config|
-        if config.key?("vpc")
+        if config.key?('vpc')
           a_vpc_exists = true
         elsif a_vpc_exists
-          raise("If you are using VPCs then a VPC must be specified for all environments in the environment_configs")
+          raise('If you are using VPCs then a VPC must be specified for all environments in the environment_configs')
         end
       end
     end

data/lib/linter/config_linter.rb

@@ -1,50 +1,53 @@
 module PropertyGenerator
   require 'yaml'
   class ConfigLinter
+    TESTS = [
+      'config_has_correct_keys',
+      'environment_configs_match_environments_list',
+      'environment_configs_have_valid_region_and_account_values',
+      'environment_configs_have_well_formatted_interpolations',
+      'config_file_is_present'
+    ].freeze
 
     attr_accessor :configs
 
-    def initialize(path)
+    def initialize(path, ignored_tests)
       @configs = check_for_config(path)
+      @ignored_tests = ignored_tests
     end
 
     def run_config_tests
       if @configs == {}
         tests = ['config_file_is_present']
       else
-        tests = ['config_has_correct_keys',
-                 'environment_configs_match_environments_list',
-                 'environment_configs_have_valid_region_and_account_values',
-                 'environment_configs_have_well_formatted_interpolations',
-                 'config_file_is_present']
+        tests = TESTS
       end
-      results = PropertyGenerator.test_runner(self, tests)
-      results
+      tests -= @ignored_tests
+
+      PropertyGenerator.test_runner(self, tests)
     end
 
     def check_for_config(path)
-      #Tries to load the config file - if is unable to load config.yml it returns an empty hash
-      #Empty hash is returned so that the rest of the files are still able to be linted instead of stopping at this point.
-      begin
-        YAML.load_file(path)
-      rescue
-        {}
-      end
+      # Tries to load the config file - if is unable to load config.yml it returns an empty hash
+      # Empty hash is returned so that the rest of the files are still able to be linted instead of stopping at this point.
+
+      YAML.load_file(path)
+    rescue StandardError
+      {}
     end
 
     def config_file_is_present
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       if @configs == {}
         status[:status] = 'fail'
-        status[:error] = "Config.yml file is missing, it is required."
+        status[:error] = 'Config.yml file is missing, it is required.'
       end
       status
     end
 
     def config_has_correct_keys
-      status = {status: 'pass', error: ''}
-      config_keys = ['environments', 'accounts',
-                     'environment_configs']
+      status = { status: 'pass', error: '' }
+      config_keys = ['environments', 'accounts', 'environment_configs']
       if @configs.keys != config_keys
         status[:status] = 'fail'
         status[:error] = "Config keys should be 'environments', 'accounts', and 'environment_configs'."
@@ -53,40 +56,37 @@ module PropertyGenerator
     end
 
     def environment_configs_match_environments_list
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       if @configs['environments'] != @configs['environment_configs'].keys
         status[:status] = 'fail'
-        status[:error] = "Environments in environment_configs do not match environments listed in config environments."
+        status[:error] = 'Environments in environment_configs do not match environments listed in config environments.'
       end
       status
     end
 
     def environment_configs_have_valid_region_and_account_values
-      status = {status: 'pass', error: ''}
-      environments_missmatch_values = []
-      any_missmatches = false
-      @configs['environment_configs'].keys.each do |environment|
-        unless (@configs['environment_configs'][environment].key?('region')) && @configs['accounts'].include?(@configs['environment_configs'][environment]['account'])
-          environments_missmatch_values << environment
-          any_missmatches = true
-        end
-        if any_missmatches
-          status[:status] = 'fail'
-          status[:error] = "Environments: #{environments_missmatch_values} in environment_configs have a region or account value not listed in top level."
-        end
+      status = { status: 'pass', error: '' }
+      environments_missmatch_values = @configs['environment_configs'].reject do |_, env_config|
+        env_config.key?('region') && @configs['accounts'].include?(env_config['account'])
+      end.keys
+
+      unless environments_missmatch_values.empty?
+        status[:status] = 'fail'
+        status[:error] = "Environments: #{environments_missmatch_values} in environment_configs have a region or account value not listed in top level."
       end
+
       status
     end
 
     def environment_configs_have_well_formatted_interpolations
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       environments_with_bad_interpolations = []
       any_mistakes = false
       @configs['environment_configs'].keys.each do |environment|
         if @configs['environment_configs'][environment]['interpolations'].class == Hash
           @configs['environment_configs'][environment]['interpolations'].each do |interpolation, value|
             if value.class != String && value.class != Integer && value.class != Float && value.class != Fixnum
-              environments_with_bad_interpolations << {environment => interpolation}
+              environments_with_bad_interpolations << { environment => interpolation }
               any_mistakes = true
             end
           end
@@ -101,6 +101,5 @@ module PropertyGenerator
       end
       status
     end
-
   end
 end