config_o_mat 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd094e1ed00e6019a324570d47475448e924c6c31abe2c9fb7c31cd37064ddfe
-  data.tar.gz: e9712d360ae7da7adf5c15142bf3a2365465832071998790ebd8e5a089429f9a
+  metadata.gz: 26063499446b5cf904e29c5ef7e147c845bc6a2e72e8e37b32148c35f8b4eb96
+  data.tar.gz: 0cce14aaef3a7b479bce31bbe8dfeeb4f4159153c999de199227aa8dd49b9602
 SHA512:
-  metadata.gz: 1a63b431f2fe289c0defaadc88a7105910d0198bf8dc3862da8effb908953f2b480687cc7750fc4f282cc410dcc6f3dc4e8d54659523e909c07f5e8e8059a1e3
-  data.tar.gz: 8763d1eff5f21f0cb2cfc603595fb8b4e198dad30ccbb8f2727ee930df1438d8b5a5c5075de674e269b99a25313f6648be160b72ed6caa3d921d595af7c8fb83
+  metadata.gz: 94869014f18070dfcb57e3907ad2c16b24d06f5693a353a7979f57e7b5333813ec9a4c458584173ec1622240c3fc6126dae0895ac53facef31fad11775879388
+  data.tar.gz: eff0f317b20431cb779765aa04af447a0402fc5ede9f46c69ddcbdc5403b11b85b79989e7315cb0f21262cad1818b06bb6bccf1b7e21a91638e91aba91db53ec

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+## 0.4.0
+
+NEW FEATURES:
+
+* Facter support. In your configomat config set a top level `facter` key to either truthy or a string. If truthy, facter data will be exposed in a profile named `facter` in all templates. If set to a string, facter data will be exposed in a profile with the given name in all templates.
+* Attempting to access a configuration variable from a profile or secret that does not exist using `#[]` will now raise an exception indicating the key being incorrectly accessed. If you need to access optionally present configuration variables from profiles or secrets use `#fetch(key, nil)`.
+
+## 0.3.0
+
+NEW FEATURES:
+
+* AWS Secrets Manager support, through an AWS AppConfig configuration. Values to pull from AWS Secrets Manager can be set using an "aws:secrets" key in any loaded AWS AppConfig JSON or YAML configuration. The value must be a dictionary. Keys in this dictionary will be exposed in the #secrets hash on the profile in templates. Values in this dictionary must be a dictionary containing at a minimum a `secret_id` key, which must be the id of an AWS Secrets Manager Secret. The dictionary may also contain a `version_id` or `version_stage` key to indicate which version of the secret to load, and a `content_type` key which may be one of `text/plan`, `application/json`, or `application/x-yaml` to indicate how the secret data should be parsed.
+
+## 0.2.1
+
+BUG FIXES:
+
+* restart_all actually works.
+
+## 0.2.0
+
+NEW FEATURES:
+
+* restart_all restart_mode for services to restart all running instances of an instantiated service.
+
+## 0.1.0
+
+Initial release

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    config_o_mat (0.2.1)
+    config_o_mat (0.4.0)
       aws-sdk-appconfig (~> 1.18)
       aws-sdk-secretsmanager (~> 1.57)
+      facter (~> 4.2, >= 4.2.8)
       lifecycle_vm (~> 0.1.1)
       logsformyfamily (~> 0.2)
       ruby-dbus (~> 0.16.0)
@@ -13,25 +14,29 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.2.0)
-    aws-partitions (1.554.0)
-    aws-sdk-appconfig (1.24.0)
-      aws-sdk-core (~> 3, >= 3.126.0)
+    aws-partitions (1.574.0)
+    aws-sdk-appconfig (1.25.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.126.1)
+    aws-sdk-core (3.130.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-secretsmanager (1.57.0)
-      aws-sdk-core (~> 3, >= 3.126.0)
+    aws-sdk-secretsmanager (1.59.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
-    jmespath (1.6.0)
-    lifecycle_vm (0.1.2)
-    logsformyfamily (0.2.3)
+    facter (4.2.9)
+      hocon (~> 1.3)
+      thor (>= 1.0.1, < 2.0)
+    hocon (1.3.1)
+    jmespath (1.6.1)
+    lifecycle_vm (0.1.3)
+    logsformyfamily (0.3.0)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -53,6 +58,7 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    thor (1.2.1)
 
 PLATFORMS
   ruby

data/config_o_mat.gemspec

@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency('lifecycle_vm', '~> 0.1.1')
   spec.add_dependency('ruby-dbus', '~> 0.16.0')
   spec.add_dependency('sd_notify', '~> 0.1.1')
+  spec.add_dependency('facter', ['~> 4.2', '>= 4.2.8'])
 
   spec.add_development_dependency('simplecov', '~> 0.21.2')
   spec.add_development_dependency('rspec', '~> 3.10')

data/lib/config_o_mat/configurator/op/refresh_all_profiles.rb

@@ -29,56 +29,82 @@ module ConfigOMat
         self.last_refresh_time = Time.now.to_i
 
         profile_defs.each do |(profile_name, definition)|
-          request = {
-            application: definition.application, environment: definition.environment,
-            configuration: definition.profile, client_id: client_id
-          }
+          if definition.kind_of?(ConfigOMat::Profile)
+            refresh_appconfig_profile(profile_name, definition)
+          elsif definition.kind_of?(ConfigOMat::FacterProfile)
+            refresh_facter_profile(profile_name)
+          else
+            error profile_name, "unknown profile type #{definition.class.name}"
+          end
+        end
+      end
+
+    private
+
+      def refresh_facter_profile(profile_name)
+        current_version = applied_profiles&.fetch(profile_name, nil)&.version
+        new_profile = ConfigOMat::LoadedFacterProfile.new(profile_name)
 
-          current_version = applied_profiles&.fetch(profile_name, nil)&.version
-          request[:client_configuration_version] = current_version if current_version
+        return if new_profile.version == current_version
 
-          response =
-            begin
-              appconfig_client.get_configuration(request)
-            rescue StandardError => e
-              error profile_name, e
-              nil
-            end
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: current_version, new_version: new_profile.version
+        )
 
-          next if response.nil?
+        profiles_to_apply << LoadedProfile.new(new_profile, nil)
+      end
+
+      def refresh_appconfig_profile(profile_name, definition)
+        request = {
+          application: definition.application, environment: definition.environment,
+          configuration: definition.profile, client_id: client_id
+        }
+
+        current_version = applied_profiles&.fetch(profile_name, nil)&.version
+        request[:client_configuration_version] = current_version if current_version
+
+        response =
+          begin
+            appconfig_client.get_configuration(request)
+          rescue StandardError => e
+            error profile_name, e
+            nil
+          end
 
-          loaded_version = response.configuration_version
-          next if current_version && loaded_version == current_version
+        return if response.nil?
 
-          logger&.notice(
-            :updated_profile,
-            name: profile_name, previous_version: current_version, new_version: loaded_version
-          )
+        loaded_version = response.configuration_version
+        return if current_version && loaded_version == current_version
 
-          profile = LoadedAppconfigProfile.new(
-            profile_name, loaded_version, response.content.read, response.content_type
-          )
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: current_version, new_version: loaded_version
+        )
 
-          loaded_secrets = nil
+        profile = LoadedAppconfigProfile.new(
+          profile_name, loaded_version, response.content.read, response.content_type
+        )
 
-          if !profile.secret_defs.empty?
-            self.secrets_loader_memory ||= ConfigOMat::SecretsLoader::Memory.new(secretsmanager_client: secretsmanager_client)
-            secrets_loader_memory.update_secret_defs_to_load(profile.secret_defs.values)
+        loaded_secrets = nil
 
-            vm = ConfigOMat::SecretsLoader::VM.new(secrets_loader_memory).call
+        if !profile.secret_defs.empty?
+          self.secrets_loader_memory ||= ConfigOMat::SecretsLoader::Memory.new(secretsmanager_client: secretsmanager_client)
+          secrets_loader_memory.update_secret_defs_to_load(profile.secret_defs.values)
 
-            if vm.errors?
-              error :"#{profile_name}_secrets", vm.errors
-              next
-            end
+          vm = ConfigOMat::SecretsLoader::VM.new(secrets_loader_memory).call
 
-            loaded_secrets = secrets_loader_memory.loaded_secrets.each_with_object({}) do |(key, value), hash|
-              hash[value.name] = value
-            end
+          if vm.errors?
+            error :"#{profile_name}_secrets", vm.errors
+            return
           end
 
-          profiles_to_apply << LoadedProfile.new(profile, loaded_secrets)
+          loaded_secrets = secrets_loader_memory.loaded_secrets.each_with_object({}) do |(key, value), hash|
+            hash[value.name] = value
+          end
         end
+
+        profiles_to_apply << LoadedProfile.new(profile, loaded_secrets)
       end
     end
   end

data/lib/config_o_mat/configurator/op/refresh_profile.rb

@@ -23,6 +23,38 @@ module ConfigOMat
       writes :applying_profile, :secrets_loader_memory
 
       def call
+        if applying_profile.loaded_profile_data.kind_of?(ConfigOMat::LoadedAppconfigProfile)
+          refresh_appconfig_profile
+        elsif applying_profile.loaded_profile_data.kind_of?(ConfigOMat::LoadedFacterProfile)
+          refresh_facter_profile
+        end
+      end
+
+    private
+
+      def refresh_facter_profile
+        profile_name = applying_profile.name
+        profile_version = applying_profile.version
+        definition = profile_defs[profile_name]
+        new_profile = ConfigOMat::LoadedFacterProfile.new(profile_name)
+
+        if new_profile.version == profile_version
+          logger&.warning(
+            :no_update,
+            name: profile_name, version: profile_version
+          )
+          return
+        end
+
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: profile_version, new_version: new_profile.version
+        )
+
+        self.applying_profile = LoadedProfile.new(new_profile, nil)
+      end
+
+      def refresh_appconfig_profile
         profile_name = applying_profile.name
         profile_version = applying_profile.version
         definition = profile_defs[profile_name]

data/lib/config_o_mat/secrets_loader/memory.rb

@@ -38,7 +38,7 @@ module ConfigOMat
         @logger = logger
       end
 
-       def update_secret_defs_to_load(defs)
+      def update_secret_defs_to_load(defs)
         @loaded_secrets = {}
         @secret_defs_to_load = defs
         self

data/lib/config_o_mat/shared/op/load_meta_config.rb

@@ -126,6 +126,16 @@ module ConfigOMat
         self.template_defs = instantiate.call(:templates, Template)
         self.profile_defs = instantiate.call(:profiles, Profile)
 
+        facter = merged_config[:facter]
+        if facter
+          facter_key = facter.kind_of?(String) ? facter.to_sym : :facter
+          if profile_defs.key?(facter_key)
+            error :facter, "conflicts with profile #{facter_key}"
+          else
+            profile_defs[facter_key] = ConfigOMat::FacterProfile.new
+          end
+        end
+
         self.logger = LogsForMyFamily::Logger.new if !logger
 
         log_type = merged_config[:log_type]&.to_sym || :stdout

data/lib/config_o_mat/shared/types.rb

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'facter'
+
 require 'json'
 require 'yaml'
 require 'digest'
@@ -187,6 +189,83 @@ module ConfigOMat
     end
   end
 
+  class FacterProfile < ConfigItem
+  end
+
+  class LoadedFacterProfile < ConfigItem
+    CLEAR_FROM_FACTER = [
+      "memoryfree", "memoryfree_mb", "load_averages", "uptime", "system_uptime", "uptime_seconds", "uptime_hours", "uptime_days",
+      {"ec2_metadata" => ["identity-credentials"]},
+      {"memory" => [{"system" => ["capacity", "available_bytes", "used", "used_bytes", "available"] }] }
+    ].freeze
+
+    attr_reader :name, :version, :contents
+
+    def initialize(name)
+      @name = name
+      load_from_facter
+      @version = contents.hash
+    end
+
+    def validate
+      error :name, 'must be present' if @name.nil? || @name.empty?
+      error :contents, 'must be present' if @contents.nil? || @contents.empty?
+      error :contents, 'must be a hash' if !@contents.kind_of?(Hash)
+    end
+
+    def hash
+      @name.hash ^ @version
+    end
+
+    def to_h
+      @contents
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.version != version || other.name != name
+      true
+    end
+
+  private
+
+    def load_from_facter
+      Facter.clear
+      # This is to work around a bug in Facter wherein it fails to invalidate a second cache of the
+      # IMDSv2 token.
+      Facter::Resolvers::Ec2.instance_variable_set(:@v2_token, nil)
+      new_facts = Facter.to_hash
+      clear(new_facts, CLEAR_FROM_FACTER)
+      transform(new_facts)
+      @contents = new_facts
+    end
+
+    def clear(hash, diffs)
+      diffs.each do |diff|
+        if diff.kind_of?(Hash)
+          diff.each do |(key, values)|
+            clear(hash[key], values) if hash.key?(key)
+          end
+        elsif hash
+          hash.delete(diff)
+        end
+      end
+    end
+
+    def transform(hash)
+      return unless hash.kind_of?(Hash)
+      hash.transform_keys!(&:to_sym)
+      hash.default_proc = proc { |hash, key| raise KeyError.new("No key #{key.inspect} in profile #{name}", key: key, receiver: hash) }
+      hash.each_value do |value|
+        if value.kind_of?(Hash)
+          transform(value)
+        elsif value.kind_of?(Array)
+          value.each { |v| transform(v) }
+        end
+      end
+    end
+  end
+
   class LoadedAppconfigProfile < ConfigItem
     attr_reader :name, :version, :contents, :secret_defs
 
@@ -206,7 +285,12 @@ module ConfigOMat
       if parser
         begin
           @contents = parser.call(contents)
-          parse_secrets if @contents.kind_of?(Hash)
+          if @contents.kind_of?(Hash)
+            parse_secrets
+            @contents.default_proc = proc do |hash, key|
+              raise KeyError.new("No key #{key.inspect} in profile #{name}", key: key, receiver: hash)
+            end
+          end
         rescue StandardError => e
           error :contents, e
         end
@@ -239,7 +323,7 @@ module ConfigOMat
   private
 
     def parse_secrets
-      secret_entries = @contents[:"aws:secrets"]
+      secret_entries = @contents.fetch(:"aws:secrets", nil)
       return if secret_entries.nil?
 
       error :contents_secrets, 'must be a dictionary' if !secret_entries.kind_of?(Hash)
@@ -305,6 +389,11 @@ module ConfigOMat
 
       begin
         @contents = LoadedAppconfigProfile::PARSERS[content_type].call(secret_string)
+        if @contents.kind_of?(Hash)
+          @contents.default_proc = proc do |hash, key|
+            raise KeyError.new("No key #{key.inspect} in secret #{name}", key: key, receiver: hash)
+          end
+        end
       rescue StandardError => e
         error :contents, e
       end
@@ -335,22 +424,22 @@ module ConfigOMat
   class LoadedProfile < ConfigItem
     extend Forwardable
 
-    attr_reader :secrets, :loaded_appconfig_profile
+    attr_reader :secrets, :loaded_profile_data
 
-    def_delegators :@loaded_appconfig_profile, :name, :version, :contents
+    def_delegators :@loaded_profile_data, :name, :version, :contents
 
-    def initialize(loaded_appconfig_profile, secrets)
-      @loaded_appconfig_profile = loaded_appconfig_profile
+    def initialize(loaded_profile_data, secrets)
+      @loaded_profile_data = loaded_profile_data
       @secrets = secrets || {}
 
-      @errors = @loaded_appconfig_profile.errors if @loaded_appconfig_profile.errors?
+      @errors = @loaded_profile_data.errors if @loaded_profile_data.errors?
     end
 
     def validate
     end
 
     def hash
-      @loaded_appconfig_profile.hash ^ @secrets.hash
+      @loaded_profile_data.hash ^ @secrets.hash
     end
 
     def to_h
@@ -359,7 +448,7 @@ module ConfigOMat
 
     def eql?(other)
       return false if !super(other)
-      return false if other.loaded_appconfig_profile != @loaded_appconfig_profile || other.secrets != @secrets
+      return false if other.loaded_profile_data != @loaded_profile_data || other.secrets != @secrets
       true
     end
   end

data/lib/config_o_mat/version.rb

@@ -15,5 +15,5 @@
 # limitations under the License.
 
 module ConfigOMat
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: config_o_mat
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Alex Scarborough
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-16 00:00:00.000000000 Z
+date: 2022-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-appconfig
@@ -94,6 +94,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.8
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +155,7 @@ files:
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE