cps-property-generator 0.2.20 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f659265beb057cc6e807215df12702562307e50
-  data.tar.gz: 58a223834ecda08eea38a5f9a21ad6cbdf39f37d
+  metadata.gz: aad999a530e887f24055f48e8481d342c2c9513a
+  data.tar.gz: 581fa068ac84ebb424ef6bd5bfe7d8e2eb3bb555
 SHA512:
-  metadata.gz: dbda7d164c992be4eab086b2f770dee1c8adbefe259ae6c12da695ba09d6f3c8ffbaa90a76e51e7572cb2143d7c26609c3ac39231d59a4af964daff555267e07
-  data.tar.gz: bf99d2880a883f34fc8047a61fff2b55c51307faa71d450780a30f01ef29f40d882c622e3d547df0e27e7d3ca127e4f73b12ce886c338c7971948e23f05284ff
+  metadata.gz: a1bad9217c2f69cd37ae6d92ec2a3d7473b3d5372b114c681ef5c06dc5761d1a0f9e7bf074852ac10d760a14c9a13031e6ca2e735ce35db41d7b55265d02bd63
+  data.tar.gz: 4ba83409ee857e560b09ab8047c22265d56cf95ecfec19199001f7c39eea5ea137a60c0b310a53786c449e35ee793eb62051ff36a52991da87b9f7975773e00f

data/README.md

@@ -77,6 +77,9 @@ touch globals.yml
 1. Create a folder called `environments` inside your folder named after your account.
 2. Inside the `environments` folder create a yaml file named after the environment you would like to define globals for. Only environments defined in your config are supported. The environment must also be mapped in the config to the account the sharing the same name as the folder the environment global yaml file lives in. 
 3. In the newly created environment's yaml file you may define your globals.
+##### Hierarchical Merging
+Hierarchical merging follows the following equation `(account.yml < service::default) < (region.yml < service::region)`. So the main thing to note that may be counterintuitive is that service defaults do not overwrite a region default. 
+
 ###### Encrypted Properties in Environment Globals
 If you would like to pass in an encrypted property to all services in a given environment, you can pass in ckrt encrypted values by having an `encrypted` block in your environment global file.
 ```yaml

data/bin/cps-property-generator

@@ -1,11 +1,10 @@
 #!/usr/bin/env ruby
-#
 require 'thor'
 require 'yaml'
-require_relative '../lib/generator/generator.rb'
-require_relative '../lib/linter/linter.rb'
-class GeneratorCLI < ::Thor
+require_relative '../lib/generator/generator'
+require_relative '../lib/linter/linter'
 
+class GeneratorCLI < ::Thor
   desc 'generate', 'Generate properties'
   option 'project_path', banner: 'PROJECT_PATH', type: :string, desc: 'Path to the property project to generate properties for'
   option 'output', banner: 'OUTPUT', type: :string, desc: 'Output path for locally dumping generated outputs', :default => '/tmp/'
@@ -16,7 +15,6 @@ class GeneratorCLI < ::Thor
   option 'upload_all', banner: 'UPLOAD_ALL', type: :boolean, desc: 'Whether to upload all envs and accounts to a single bucket', :default => false
 
   def generate
-
     generator = PropertyGenerator::Generator.new(options)
     out = generator.generate
     if options['upload']

data/lib/generator/config.rb

@@ -1,7 +1,6 @@
 module PropertyGenerator
   require 'yaml'
   class Config
-
     attr_accessor :configs
 
     def initialize(project_path)
@@ -29,4 +28,4 @@ module PropertyGenerator
       YAML.load_file(file)
     end
   end
-end
+end

data/lib/generator/generator.rb

@@ -11,20 +11,20 @@ module PropertyGenerator
     # purpose: initialize globals and configs
     # serve as a broker between tasks
     def initialize(options)
-      project_path =  File.expand_path(options['project_path'])
+      project_path = File.expand_path(options['project_path'])
       @configs = PropertyGenerator::Config.new(project_path)
       @globals = PropertyGenerator::Globals.new(project_path, @configs)
       @globals = @globals.globals
       @accounts = @configs.accounts
 
-      @output_path =  "#{File.expand_path(options['output'])}/properties/#{SecureRandom.hex}"
+      @output_path = "#{File.expand_path(options['output'])}/properties/#{SecureRandom.hex}"
       puts "Properties will be output here #{@output_path}"
       @service_list = PropertyGenerator.read_services(project_path)
     end
 
     def generate
       output = []
-      @service_list.each do | service, path|
+      @service_list.each do |service, path|
         PropertyGenerator.config_enforcer(@configs.environment_configs)
         service_instance = PropertyGenerator::Service.new(YAML.load_file(path), @configs, @globals)
         service_instance.service
@@ -42,18 +42,20 @@ module PropertyGenerator
 
       if config['upload_all']
         _upload_files(out.sort) do |file|
-          file_region = file.split("/")[-2]
-          file_account = file.split("/")[-3]
+          file_region = file.split('/')[-2]
+          file_account = file.split('/')[-3]
 
           PropertyGenerator.sync(upload_region, file_account, upload_bucket, file, file_region)
         end
       else
-        upload_account = config['upload_account']
-        abort("The specified account (#{upload_account}) is not configured, please add it to config/config.yml") unless @accounts.include?(upload_account)
+        upload_account = config['upload_account'].strip
+        unless @accounts.map { |a| a.to_s.strip }.include?(upload_account)
+          abort("The specified account (#{upload_account}) is not configured, please add it to config/config.yml")
+        end
 
-        upload_out = out.select { |file| file.include?("#{upload_account}") && file.include?("#{upload_region}") }
+        upload_out = out.select { |file| file.include?(upload_account) && file.include?(upload_region) }
         _upload_files(upload_out) do |file|
-          file_region = file.split("/")[-2]
+          file_region = file.split('/')[-2]
           PropertyGenerator.sync(upload_region, upload_account, upload_bucket, file, file_region)
         end
       end
@@ -68,7 +70,5 @@ module PropertyGenerator
         end.each(&:join)
       end
     end
-
   end
 end
-

data/lib/generator/globals.rb

@@ -14,10 +14,9 @@ module PropertyGenerator
       @globals ||= condense_globals
     end
 
-
     def get_main_global
       top_level = {}
-      if File.exists?("#{@project_path}/globals/globals.yml")
+      if File.exist?("#{@project_path}/globals/globals.yml")
         top_level = YAML.load_file("#{@project_path}/globals/globals.yml")
       end
       top_level
@@ -26,9 +25,10 @@ module PropertyGenerator
     def get_account_globals
       data = {}
       @accounts.each do |account|
-        next unless Dir.exists?("#{@project_path}/globals/accounts/#{account}")
+        next unless Dir.exist?("#{@project_path}/globals/accounts/#{account}")
+
         account_default_file = "#{@project_path}/globals/accounts/#{account}/#{account}.yml"
-        data[account] = YAML.load_file(account_default_file) if File.exists?(account_default_file)
+        data[account] = YAML.load_file(account_default_file) if File.exist?(account_default_file)
       end
       data
     end
@@ -36,14 +36,16 @@ module PropertyGenerator
     def get_environment_globals
       data = {}
       @accounts.each do |account|
-        next unless Dir.exists?("#{@project_path}/globals/accounts/#{account}/environments")
+        next unless Dir.exist?("#{@project_path}/globals/accounts/#{account}/environments")
+
         data[account] = {}
         @environments.each do |env|
-          next unless File.exists?("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
+          next unless File.exist?("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
+
           data[account][env] = YAML.load_file("#{@project_path}/globals/accounts/#{account}/environments/#{env}.yml")
           unless data[account][env]['encrypted'].nil?
             encrypted = data[account][env]['encrypted'].dup
-            not_encrypted = data[account][env].reject { |k,_| k == 'encrypted' }
+            not_encrypted = data[account][env].reject { |k, _| k == 'encrypted' }
             data[account][env] = not_encrypted.deep_merge(encrypted)
           end
         end
@@ -51,8 +53,7 @@ module PropertyGenerator
       data
     end
 
-
-    #merge environment globals with account globals.
+    # merge environment globals with account globals.
     def condense_globals
       condensed = {}
       # get account and the environmental hash's for said account
@@ -62,7 +63,7 @@ module PropertyGenerator
       # nothing to do here if everything is empty
       return condensed if environment_globals.empty? && account_globals.empty? && main_global.empty?
 
-      environment_globals.each do |account, env_global |
+      environment_globals.each do |account, env_global|
         # get the env and the values
         env_global.each do |env, hash|
           account_globals[account] ||= {}
@@ -85,6 +86,5 @@ module PropertyGenerator
       end
       condensed
     end
-
   end
 end

data/lib/generator/service.rb

@@ -1,9 +1,9 @@
 module PropertyGenerator
-
   class Service
     require 'active_support/core_ext/hash'
 
     attr_accessor :service
+    attr_reader :additional_options
 
     def initialize(service_data, config, globals)
       @service_data = service_data
@@ -28,24 +28,16 @@ module PropertyGenerator
       @additional_options['secretlabels'] = @service_data['secretlabels'].nil? ? nil : @service_data['secretlabels']
     end
 
-    def additional_options
-      @additional_options
-    end
-
-    def service
-      @service
-    end
-
     def configmap_name
       @configmapname
     end
 
     def interpolate
       environments = @environments
-      #read in config
-      #interate through environment and substitute config for values for that environment
-      environments.each do | env|
-        #get the map of config for a env
+      # read in config
+      # interate through environment and substitute config for values for that environment
+      environments.each do |env|
+        # get the map of config for a env
         interpolations = @environment_configs[env]['interpolations']
 
         # Recursively interate through the properties for an environment and gsub the config
@@ -59,7 +51,7 @@ module PropertyGenerator
 
     def interpolate_nested_properties(service_env, interpolations)
       interpolations.each do |matcher_key, matcher_value|
-        service_env.each { |k,v|  service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}")}
+        service_env.each { |k, v| service_env[k] = v.gsub("{#{matcher_key}}", matcher_value) if v.class == String && v.include?("{#{matcher_key}}") }
         service_env.values.each do |v|
           interpolate_nested_properties(v, interpolations) if v.class == Hash
           v.each_with_index do |val, idx|
@@ -70,7 +62,7 @@ module PropertyGenerator
     end
 
     def merge_env_default(data, environments)
-      #creates a hash of the environments merged with the defaults
+      # creates a hash of the environments merged with the defaults
       # {service => {env1 =>  {properties},
       #             env2 => {properties}
       #           }
@@ -80,7 +72,7 @@ module PropertyGenerator
 
       environments.each do |env|
         default_clone = default.dup
-        #if nil, use set to environments as nothing to merge env with
+        # if nil, use set to environments as nothing to merge env with
         data['environments'] ||= {}
         data['environments'][env] ||= {}
         environment_data = data['environments'][env].dup
@@ -99,7 +91,7 @@ module PropertyGenerator
     end
 
     def merge_service_with_globals(globals_data, service_data, environments)
-      #service will now overwrite globals, merging will be done for each environment
+      # service will now overwrite globals, merging will be done for each environment
       output = {}
       envs = environments
       envs.each do |env|
@@ -113,7 +105,5 @@ module PropertyGenerator
       end
       output
     end
-
   end
-
 end

data/lib/helpers/helpers.rb

@@ -1,10 +1,10 @@
 require_relative '../helpers/helpers'
+
 module PropertyGenerator
   require 'json'
   require 'fileutils'
   require 'aws-sdk-s3'
   class << self
-
     def test_runner(object, test_list)
       results = {}
       test_list.each do |test|
@@ -14,9 +14,9 @@ module PropertyGenerator
     end
 
     def get_list_of_files(path, ignore_list)
-      #Returns a list of files in given path
-      #Ignores files in a given ignore list
-      Dir.glob(path + "/**/*").select{ |e| File.file? e unless ignore_list.include?(e.split('/')[(e.split('/')).length - 1])}
+      # Returns a list of files in given path
+      # Ignores files in a given ignore list
+      Dir.glob("#{path}/**/*").select { |e| File.file?(e) unless ignore_list.include?(e.split('/')[-1]) }
     end
 
     def valid_paths(path)
@@ -26,7 +26,7 @@ module PropertyGenerator
         begin
           YAML.load_file(file_path)
           valid_paths << file_path
-        rescue
+        rescue StandardError
           next
         end
       end
@@ -39,7 +39,7 @@ module PropertyGenerator
       list_of_file_paths.each do |file_path|
         begin
           YAML.load_file(file_path)
-        rescue
+        rescue StandardError
           invalid_paths << file_path
         end
       end
@@ -57,21 +57,21 @@ module PropertyGenerator
     def writer(service_name, finalized, configs, output_path, additional_options)
       output = []
       envs = configs.environments
-      environmental_configs =  configs.environment_configs
-      envs.each do | env|
-        account = environmental_configs[env]["account"]
-        region = environmental_configs[env]["region"]
-        hash = { "properties" => finalized[env] }
+      environmental_configs = configs.environment_configs
+      envs.each do |env|
+        account = environmental_configs[env]['account']
+        region = environmental_configs[env]['region']
+        hash = { 'properties' => finalized[env] }
         ['configname', 'stringdata', 'configlabels', 'secretlabels'].each do |setting|
-          hash[setting] = additional_options[setting] if [setting] unless additional_options[setting].nil?
+          hash[setting] = additional_options[setting] unless additional_options[setting].nil?
         end
         json = JSON.pretty_generate(hash)
-        #IF users are specifing a vpc then we will drop property files under the dir that corresponds to the vpc
-        if environmental_configs[env].key?("vpc") && !environmental_configs[env]["vpc"].nil?
-          vpc_dir = "#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}"
+        # IF users are specifing a vpc then we will drop property files under the dir that corresponds to the vpc
+        if environmental_configs[env].key?('vpc') && !environmental_configs[env]['vpc'].nil?
+          vpc_dir = "#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}"
           FileUtils.mkdir_p("#{vpc_dir}/") unless Dir.exist?(vpc_dir)
-          File.write("#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}/#{service_name}.json", json)
-          output << "#{output_path}/#{account}/#{region}/#{environmental_configs[env]["vpc"]}/#{service_name}.json"
+          File.write("#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}/#{service_name}.json", json)
+          output << "#{output_path}/#{account}/#{region}/#{environmental_configs[env]['vpc']}/#{service_name}.json"
         else
           FileUtils.mkdir_p("#{output_path}/#{account}/#{region}/") unless Dir.exist?("#{output_path}/#{account}/#{region}/")
           File.write("#{output_path}/#{account}/#{region}/#{service_name}.json", json)
@@ -83,22 +83,22 @@ module PropertyGenerator
 
     def sync(region, account, bucket, file, file_region)
       s3 = Aws::S3::Resource.new(region: region)
-      filename = file.split("/").last
+      filename = file.split('/').last
       puts "Destination: #{account}/#{file_region}/#{filename}"
       puts "Uploading: #{file}"
       obj = s3.bucket(bucket).object("#{account}/#{file_region}/#{filename}")
       obj.upload_file(file)
     end
 
-    #Force users to specify VPCs for all environments if specified for one environment.
-    #This allows us to skip having conditional logic downstream in CPS requests
+    # Force users to specify VPCs for all environments if specified for one environment.
+    # This allows us to skip having conditional logic downstream in CPS requests
     def config_enforcer(environment_configs)
       a_vpc_exists = false
       environment_configs.each do |environment, config|
-        if config.key?("vpc")
+        if config.key?('vpc')
           a_vpc_exists = true
         elsif a_vpc_exists
-          raise("If you are using VPCs then a VPC must be specified for all environments in the environment_configs")
+          raise('If you are using VPCs then a VPC must be specified for all environments in the environment_configs')
         end
       end
     end

data/lib/linter/config_linter.rb

@@ -1,50 +1,53 @@
 module PropertyGenerator
   require 'yaml'
   class ConfigLinter
+    TESTS = [
+      'config_has_correct_keys',
+      'environment_configs_match_environments_list',
+      'environment_configs_have_valid_region_and_account_values',
+      'environment_configs_have_well_formatted_interpolations',
+      'config_file_is_present'
+    ].freeze
 
     attr_accessor :configs
 
-    def initialize(path)
+    def initialize(path, ignored_tests)
       @configs = check_for_config(path)
+      @ignored_tests = ignored_tests
     end
 
     def run_config_tests
       if @configs == {}
         tests = ['config_file_is_present']
       else
-        tests = ['config_has_correct_keys',
-                 'environment_configs_match_environments_list',
-                 'environment_configs_have_valid_region_and_account_values',
-                 'environment_configs_have_well_formatted_interpolations',
-                 'config_file_is_present']
+        tests = TESTS
       end
-      results = PropertyGenerator.test_runner(self, tests)
-      results
+      tests -= @ignored_tests
+
+      PropertyGenerator.test_runner(self, tests)
     end
 
     def check_for_config(path)
-      #Tries to load the config file - if is unable to load config.yml it returns an empty hash
-      #Empty hash is returned so that the rest of the files are still able to be linted instead of stopping at this point.
-      begin
-        YAML.load_file(path)
-      rescue
-        {}
-      end
+      # Tries to load the config file - if is unable to load config.yml it returns an empty hash
+      # Empty hash is returned so that the rest of the files are still able to be linted instead of stopping at this point.
+
+      YAML.load_file(path)
+    rescue StandardError
+      {}
     end
 
     def config_file_is_present
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       if @configs == {}
         status[:status] = 'fail'
-        status[:error] = "Config.yml file is missing, it is required."
+        status[:error] = 'Config.yml file is missing, it is required.'
       end
       status
     end
 
     def config_has_correct_keys
-      status = {status: 'pass', error: ''}
-      config_keys = ['environments', 'accounts',
-                     'environment_configs']
+      status = { status: 'pass', error: '' }
+      config_keys = ['environments', 'accounts', 'environment_configs']
       if @configs.keys != config_keys
         status[:status] = 'fail'
         status[:error] = "Config keys should be 'environments', 'accounts', and 'environment_configs'."
@@ -53,40 +56,37 @@ module PropertyGenerator
     end
 
     def environment_configs_match_environments_list
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       if @configs['environments'] != @configs['environment_configs'].keys
         status[:status] = 'fail'
-        status[:error] = "Environments in environment_configs do not match environments listed in config environments."
+        status[:error] = 'Environments in environment_configs do not match environments listed in config environments.'
       end
       status
     end
 
     def environment_configs_have_valid_region_and_account_values
-      status = {status: 'pass', error: ''}
-      environments_missmatch_values = []
-      any_missmatches = false
-      @configs['environment_configs'].keys.each do |environment|
-        unless (@configs['environment_configs'][environment].key?('region')) && @configs['accounts'].include?(@configs['environment_configs'][environment]['account'])
-          environments_missmatch_values << environment
-          any_missmatches = true
-        end
-        if any_missmatches
-          status[:status] = 'fail'
-          status[:error] = "Environments: #{environments_missmatch_values} in environment_configs have a region or account value not listed in top level."
-        end
+      status = { status: 'pass', error: '' }
+      environments_missmatch_values = @configs['environment_configs'].reject do |_, env_config|
+        env_config.key?('region') && @configs['accounts'].include?(env_config['account'])
+      end.keys
+
+      unless environments_missmatch_values.empty?
+        status[:status] = 'fail'
+        status[:error] = "Environments: #{environments_missmatch_values} in environment_configs have a region or account value not listed in top level."
       end
+
       status
     end
 
     def environment_configs_have_well_formatted_interpolations
-      status = {status: 'pass', error: ''}
+      status = { status: 'pass', error: '' }
       environments_with_bad_interpolations = []
       any_mistakes = false
       @configs['environment_configs'].keys.each do |environment|
         if @configs['environment_configs'][environment]['interpolations'].class == Hash
           @configs['environment_configs'][environment]['interpolations'].each do |interpolation, value|
             if value.class != String && value.class != Integer && value.class != Float && value.class != Fixnum
-              environments_with_bad_interpolations << {environment => interpolation}
+              environments_with_bad_interpolations << { environment => interpolation }
               any_mistakes = true
             end
           end
@@ -101,6 +101,5 @@ module PropertyGenerator
       end
       status
     end
-
   end
 end