config_o_mat 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6e9883d10f0ab6349aa447ef2b199692e05c02aeb8f17a95b8c1a7825ffab9b
-  data.tar.gz: 2486dea3da2ad9f2113e88d966efcff92616fd1baec02e1ac4dacf392df19710
+  metadata.gz: 3f95d6871d5b5bb42f5c48900cd449e6b5cb899dbe2457ce9461b51e52893bd0
+  data.tar.gz: ef07242ebd3cc3ab3c79a047e5574567821a528d9661953eb65fa0fd9a61ee61
 SHA512:
-  metadata.gz: edfc8bb95b506e73c191fba4f476e38cc18d3478a2917d159ee448e9392662c042b00991c1ad5f985b29c73f988a580c8fff1814155a71c3f120dc6dd0a117c2
-  data.tar.gz: 80df6494340c67f9ea7b57b558080ae3e369d0249e9505511db33c7c070fc434144c3bba096adff07176a66085b57bbc4a8013329453b57343ef0024305018e5
+  metadata.gz: ebf154bdf525fdc70a3d5f447a3485eee808e790da404a77b2be5da0a0700986fd0724f1a6fa245dcc12afadf0082fcab8d248f49802c4d68676b881fe2fbecc
+  data.tar.gz: 67c3561c1ec2b839ba343fb04bc71c2886ead05e81d888ca031d1fd42b3105984cc055fec6341a2447de032f821a1e4b70305d671bb371249d0526efb8dc9605

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.5.0
+
+NEW FEATURES:
+
+* Profiles can now specify an s3_fallback key. If AWS AppConfig is unreachable, we will fall back to loading the specified S3 object from the top level configured fallback_s3_bucket and interpreting it as a response from AWS AppConfig. For best results enable versioning on the bucket. It is an error to specify an s3_fallback key if the configuration does not have a top level fallback_s3_bucket.
+* AppConfig responses can request that the s3_fallback be used by setting a top level "aws:chaos_config" key to true. This allows for deployments which AppConfig deployments which can test the fallback mechanism. If a chaos config load fails, the source AppConfig profile will remain in use. Chaos config is ignored in error recovery scenarios.
+
+BUG FIXES:
+
+* config_o_mat-configurator/meta_configurator now correctly log the op that errored.
+
 ## 0.4.4
 
 BUG FIXES:

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    config_o_mat (0.4.0)
+    config_o_mat (0.4.4)
       aws-sdk-appconfig (~> 1.18)
+      aws-sdk-s3 (~> 1.114)
       aws-sdk-secretsmanager (~> 1.57)
       facter (~> 4.2, >= 4.2.8)
       lifecycle_vm (~> 0.1.1)
@@ -14,23 +15,30 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.2.0)
-    aws-partitions (1.574.0)
+    aws-partitions (1.593.0)
     aws-sdk-appconfig (1.25.0)
       aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.130.0)
+    aws-sdk-core (3.131.1)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-secretsmanager (1.59.0)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.57.0)
       aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.4.0)
+    aws-sdk-s3 (1.114.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.4)
+    aws-sdk-secretsmanager (1.62.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.5.0)
       aws-eventstream (~> 1, >= 1.0.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
-    facter (4.2.9)
+    facter (4.2.10)
       hocon (~> 1.3)
       thor (>= 1.0.1, < 2.0)
     hocon (1.3.1)

data/bin/config_o_mat-configurator

@@ -36,12 +36,12 @@ vm = ConfigOMat::Configurator::VM.new(memory)
 vm.call
 
 if vm.errors?
-  warn "Errored executing #{vm.error_op.class.name}"
+  warn "Errored executing #{vm.error_op.op.name}"
   warn "Errors: #{vm.errors}"
 
   if vm.recovery_errors?
     warn ''
-    warn "Errors recovering from error. Errored executing recovery #{vm.current_op.class.name}"
+    warn "Errors recovering from error. Errored executing recovery #{vm.current_op.op.name}"
     warn "Errors: #{vm.recovery_errors}"
   end
 

data/bin/config_o_mat-meta_configurator

@@ -36,12 +36,12 @@ vm = ConfigOMat::MetaConfigurator::VM.new(memory)
 vm.call
 
 if vm.errors?
-  warn "Errored executing #{vm.error_op.class.name}"
+  warn "Errored executing #{vm.error_op.op.name}"
   warn "Errors: #{vm.errors}"
 
   if vm.recovery_errors?
     warn ''
-    warn "Errors recovering from error. Errored executing recovery #{vm.current_op.class.name}"
+    warn "Errors recovering from error. Errored executing recovery #{vm.current_op.op.name}"
     warn "Errors: #{vm.recovery_errors}"
   end
 

data/config_o_mat.gemspec

@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
   spec.executables   = ["config_o_mat-configurator", "config_o_mat-meta_configurator"]
 
   spec.add_dependency('aws-sdk-appconfig', '~> 1.18')
+  spec.add_dependency('aws-sdk-s3', '~> 1.114')
   spec.add_dependency('aws-sdk-secretsmanager', '~> 1.57')
   spec.add_dependency('logsformyfamily', '~> 0.2')
   spec.add_dependency('lifecycle_vm', '~> 0.1.1')

data/lib/config_o_mat/configurator/cond/retries_left.rb

@@ -22,13 +22,19 @@ module ConfigOMat
       reads :error_op, :retries_left, :applying_profile
 
       def call
-        logger&.error(:op_failure, op: error_op.class.name, errors: error_op.errors)
+        logger&.error(:retry_from_failure, state: error_op.state_name, op: error_op.op.name, errors: error_op.errors)
 
         # If we aren't currently applying a profile then there's nothing for us to retry.
-        return false if applying_profile.nil?
+        if applying_profile.nil?
+          logger&.error(:cannot_retry, reason: 'not applying profile')
+          return false
+        end
 
         # If we're out of retries, well, no more retrying
-        return false if retries_left.zero?
+        if retries_left.zero?
+          logger&.error(:cannot_retry, reason: 'out of retries')
+          return false
+        end
 
         true
       end

data/lib/config_o_mat/configurator/memory.rb

@@ -25,8 +25,9 @@ module ConfigOMat
                     :client_id, :compiled_templates, :applied_profiles, :applying_profile,
                     :generated_templates, :services_to_reload, :profiles_to_apply,
                     :last_refresh_time, :next_state, :retry_count, :retries_left, :retry_wait,
-                    :region, :appconfig_client, :secretsmanager_client, :systemd_interface,
-                    :secrets_loader_memory, :gc_compact, :gc_stat
+                    :region, :appconfig_client, :secretsmanager_client, :s3_client,
+                    :systemd_interface, :secrets_loader_memory, :gc_compact, :gc_stat,
+                    :fallback_s3_bucket
 
       def initialize(
         argv: [],
@@ -57,10 +58,12 @@ module ConfigOMat
         region: nil,
         appconfig_client: nil,
         secretsmanager_client: nil,
+        s3_client: nil,
         systemd_interface: nil,
         secrets_loader_memory: nil,
         gc_compact: 0,
-        gc_stat: 0
+        gc_stat: 0,
+        fallback_s3_bucket: nil
       )
         super()
 
@@ -92,10 +95,12 @@ module ConfigOMat
         @region = region
         @appconfig_client = appconfig_client
         @secretsmanager_client = secretsmanager_client
+        @s3_client = s3_client
         @systemd_interface = systemd_interface
         @secrets_loader_memory = secrets_loader_memory
         @gc_compact = gc_compact
         @gc_stat = gc_stat
+        @fallback_s3_bucket = fallback_s3_bucket
       end
     end
   end

data/lib/config_o_mat/configurator/op/connect_to_aws.rb

@@ -17,19 +17,21 @@
 require 'lifecycle_vm/op_base'
 
 require 'aws-sdk-appconfig'
+require 'aws-sdk-s3'
 require 'aws-sdk-secretsmanager'
 
 module ConfigOMat
   module Op
     class ConnectToAws < LifecycleVM::OpBase
       reads :region
-      writes :appconfig_client, :secretsmanager_client
+      writes :appconfig_client, :secretsmanager_client, :s3_client
 
       def call
         client_opts = { logger: logger }
         client_opts[:region] = region if region
 
         self.appconfig_client = Aws::AppConfig::Client.new(client_opts)
+        self.s3_client = Aws::S3::Client.new(client_opts)
         self.secretsmanager_client = Aws::SecretsManager::Client.new(client_opts)
       end
     end

data/lib/config_o_mat/configurator/op/refresh_all_profiles.rb

@@ -21,7 +21,8 @@ require 'config_o_mat/secrets_loader'
 module ConfigOMat
   module Op
     class RefreshAllProfiles < LifecycleVM::OpBase
-      reads :profile_defs, :applied_profiles, :client_id, :appconfig_client, :secrets_loader_memory, :secretsmanager_client
+      reads :profile_defs, :applied_profiles, :client_id, :appconfig_client, :secrets_loader_memory,
+            :secretsmanager_client, :s3_client, :fallback_s3_bucket
       writes :profiles_to_apply, :last_refresh_time, :secrets_loader_memory
 
       def call
@@ -55,6 +56,21 @@ module ConfigOMat
         profiles_to_apply << LoadedProfile.new(new_profile, nil)
       end
 
+      def request_from_s3(profile_name, definition, ignore_errors)
+        begin
+          s3_response = s3_client.get_object(bucket: fallback_s3_bucket, key: definition.s3_fallback)
+          OpenStruct.new(
+            content: s3_response.body,
+            content_type: s3_response.content_type,
+            configuration_version: s3_response.version_id || s3_response.etag
+          )
+        rescue StandardError => e
+          logger&.error(:s3_fallback_load_failed, name: profile_name, reason: e)
+          error profile_name, e unless ignore_errors
+          nil
+        end
+      end
+
       def refresh_appconfig_profile(profile_name, definition)
         request = {
           application: definition.application, environment: definition.environment,
@@ -63,13 +79,20 @@ module ConfigOMat
 
         current_version = applied_profiles&.fetch(profile_name, nil)&.version
         request[:client_configuration_version] = current_version if current_version
+        fallback = false
 
         response =
           begin
             appconfig_client.get_configuration(request)
           rescue StandardError => e
-            error profile_name, e
-            nil
+            if definition.s3_fallback
+              fallback = true
+              logger&.error(:s3_fallback_request, name: profile_name, reason: e)
+              request_from_s3(profile_name, definition, false)
+            else
+              error profile_name, e
+              nil
+            end
           end
 
         return if response.nil?
@@ -83,9 +106,27 @@ module ConfigOMat
         )
 
         profile = LoadedAppconfigProfile.new(
-          profile_name, loaded_version, response.content.read, response.content_type
+          profile_name, loaded_version, response.content.read, response.content_type, fallback
         )
 
+        if profile.chaos_config? && !fallback
+          logger&.notice(:chaos_config_requested, name: profile_name)
+          if !definition.s3_fallback
+            logger&.error(:refusing_chaos_config, name: profile_name, reason: 'no s3 fallback provided')
+          else
+            response = request_from_s3(profile_name, definition, true)
+            if !response
+              logger&.error(:chaos_config_load_failed, name: profile_name)
+            else
+              loaded_version = response.configuration_version
+              logger&.notice(:chaos_config_profile_loaded, name: profile_name, new_version: loaded_version)
+              profile = LoadedAppconfigProfile.new(
+                profile_name, loaded_version, response.content.read, response.content_type, true
+              )
+            end
+          end
+        end
+
         loaded_secrets = nil
 
         if !profile.secret_defs.empty?

data/lib/config_o_mat/configurator/op/refresh_profile.rb

@@ -19,7 +19,8 @@ require 'lifecycle_vm/op_base'
 module ConfigOMat
   module Op
     class RefreshProfile < LifecycleVM::OpBase
-      reads :profile_defs, :client_id, :applying_profile, :appconfig_client, :secretsmanager_client, :secrets_loader_memory
+      reads :profile_defs, :client_id, :applying_profile, :appconfig_client, :secretsmanager_client,
+            :secrets_loader_memory, :s3_client, :fallback_s3_bucket
       writes :applying_profile, :secrets_loader_memory
 
       def call
@@ -54,6 +55,20 @@ module ConfigOMat
         self.applying_profile = LoadedProfile.new(new_profile, nil)
       end
 
+      def request_from_s3(profile_name, definition)
+        begin
+          s3_response = s3_client.get_object(bucket: fallback_s3_bucket, key: definition.s3_fallback)
+          OpenStruct.new(
+            content: s3_response.body,
+            content_type: s3_response.content_type,
+            configuration_version: s3_response.version_id || s3_response.etag
+          )
+        rescue StandardError => e
+          error profile_name, e
+          nil
+        end
+      end
+
       def refresh_appconfig_profile
         profile_name = applying_profile.name
         profile_version = applying_profile.version
@@ -63,13 +78,20 @@ module ConfigOMat
           configuration: definition.profile, client_id: client_id,
           client_configuration_version: profile_version
         }
+        fallback = false
 
         response =
           begin
             appconfig_client.get_configuration(request)
           rescue StandardError => e
-            error profile_name, e
-            nil
+            if definition.s3_fallback
+              fallback = true
+              logger&.error(:s3_fallback_request, name: profile_name, reason: e)
+              request_from_s3(profile_name, definition)
+            else
+              error profile_name, e
+              nil
+            end
           end
 
         return if response.nil? || errors?
@@ -89,7 +111,7 @@ module ConfigOMat
         )
 
         profile = LoadedAppconfigProfile.new(
-          profile_name, loaded_version, response.content.read, response.content_type
+          profile_name, loaded_version, response.content.read, response.content_type, fallback
         )
 
         loaded_secrets = nil

data/lib/config_o_mat/meta_configurator/memory.rb

@@ -23,7 +23,8 @@ module ConfigOMat
                     :systemd_directory, :logs_directory, :profile_defs,
                     :template_defs, :service_defs, :dependencies, :refresh_interval,
                     :client_id, :retry_count, :retries_left, :retry_wait,
-                    :region, :systemd_interface, :gc_stat, :gc_compact
+                    :region, :systemd_interface, :gc_stat, :gc_compact,
+                    :fallback_s3_bucket
 
       def initialize(
         argv: [],
@@ -46,7 +47,8 @@ module ConfigOMat
         region: nil,
         systemd_interface: nil,
         gc_stat: 0,
-        gc_copmact: 0
+        gc_copmact: 0,
+        fallback_s3_bucket: nil
       )
         super()
 
@@ -71,6 +73,7 @@ module ConfigOMat
         @systemd_interface = systemd_interface
         @gc_stat = gc_stat
         @gc_compact = gc_compact
+        @fallback_s3_bucket = fallback_s3_bucket
       end
     end
   end

data/lib/config_o_mat/shared/op/load_meta_config.rb

@@ -63,7 +63,8 @@ module ConfigOMat
       reads :configuration_directory, :logs_directory, :env
       writes :profile_defs, :template_defs, :service_defs, :dependencies,
              :refresh_interval, :client_id, :logger, :retry_count, :retries_left,
-             :retry_wait, :region, :systemd_interface, :gc_stat, :gc_compact
+             :retry_wait, :region, :systemd_interface, :gc_stat, :gc_compact,
+             :fallback_s3_bucket
 
       def call
         default_config = {
@@ -127,6 +128,11 @@ module ConfigOMat
         self.service_defs = instantiate.call(:services, Service)
         self.template_defs = instantiate.call(:templates, Template)
         self.profile_defs = instantiate.call(:profiles, Profile)
+        self.fallback_s3_bucket = merged_config[:fallback_s3_bucket]
+
+        if profile_defs.values.any? { |pd| pd.s3_fallback } && (fallback_s3_bucket.nil? || fallback_s3_bucket.empty?)
+          error :fallback_s3_bucket, 'must be present to use s3_fallback on profiles'
+        end
 
         facter = merged_config[:facter]
         if facter

data/lib/config_o_mat/shared/types.rb

@@ -164,27 +164,40 @@ module ConfigOMat
   end
 
   class Profile < ConfigItem
-    attr_reader :application, :environment, :profile
+    attr_reader :application, :environment, :profile, :s3_fallback
 
     def initialize(opts)
       @application = opts[:application]
       @environment = opts[:environment]
       @profile = opts[:profile]
+      @s3_fallback = opts[:s3_fallback]
     end
 
     def validate
       error :application, 'must be present' if @application.nil? || @application.empty?
       error :environment, 'must be present' if @environment.nil? || @environment.empty?
       error :profile, 'must be present' if @profile.nil? || @profile.empty?
+      if !@s3_fallback.nil?
+        if @s3_fallback.kind_of?(String)
+          error :s3_fallback, 'must be non-empty' if @s3_fallback.empty?
+        else
+          error :s3_fallback, 'must be a string'
+        end
+      end
     end
 
     def hash
-      application.hash ^ environment.hash ^ profile.hash
+      application.hash ^ environment.hash ^ profile.hash ^ s3_fallback.hash
     end
 
     def eql?(other)
       return false if !super(other)
-      return false if other.application != application || other.environment != environment || other.profile != profile
+      if other.application != application ||
+         other.environment != environment ||
+         other.profile != profile ||
+         other.s3_fallback != s3_fallback
+        return false
+      end
       true
     end
   end
@@ -207,6 +220,10 @@ module ConfigOMat
       @version = contents.hash
     end
 
+    def fallback?
+      false
+    end
+
     def validate
       error :name, 'must be present' if @name.nil? || @name.empty?
       error :contents, 'must be present' if @contents.nil? || @contents.empty?
@@ -275,10 +292,12 @@ module ConfigOMat
       'application/x-yaml' => proc { |str| YAML.safe_load(str, symbolize_names: true) }
     }.freeze
 
-    def initialize(name, version, contents, content_type)
+    def initialize(name, version, contents, content_type, fallback = false)
       @name = name
       @version = version
       @secret_defs = {}
+      @fallback = fallback
+      @chaos_config = false
 
       parser = PARSERS[content_type]
 
@@ -287,6 +306,7 @@ module ConfigOMat
           @contents = parser.call(contents)
           if @contents.kind_of?(Hash)
             parse_secrets
+            @chaos_config = @contents.fetch(:"aws:chaos_config", false)
             @contents.default_proc = proc do |hash, key|
               raise KeyError.new("No key #{key.inspect} in profile #{name}", key: key, receiver: hash)
             end
@@ -307,16 +327,30 @@ module ConfigOMat
     end
 
     def hash
-      @name.hash ^ @version.hash ^ @contents.hash
+      @name.hash ^ @version.hash ^ @contents.hash ^ @fallback.hash ^ @chaos_config.hash
     end
 
     def to_h
       @contents
     end
 
+    def fallback?
+      @fallback
+    end
+
+    def chaos_config?
+      @chaos_config
+    end
+
     def eql?(other)
       return false if !super(other)
-      return false if other.version != version || other.contents != contents || other.name != name
+      if other.version != version ||
+         other.contents != contents ||
+         other.name != name ||
+         other.fallback? != fallback? ||
+         other.chaos_config? != chaos_config?
+        return false
+      end
       true
     end
 
@@ -426,7 +460,7 @@ module ConfigOMat
 
     attr_reader :secrets, :loaded_profile_data
 
-    def_delegators :@loaded_profile_data, :name, :version, :contents
+    def_delegators :@loaded_profile_data, :name, :version, :contents, :fallback?
 
     def initialize(loaded_profile_data, secrets)
       @loaded_profile_data = loaded_profile_data

data/lib/config_o_mat/version.rb

@@ -15,5 +15,5 @@
 # limitations under the License.
 
 module ConfigOMat
-  VERSION = "0.4.4"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: config_o_mat
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Alex Scarborough
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-14 00:00:00.000000000 Z
+date: 2022-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-appconfig
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.114'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.114'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-secretsmanager
   requirement: !ruby/object:Gem::Requirement