config_o_mat 0.2.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35eaeccb6abdf69afb14c8cbe951c4579734f27af4844673b1761db70dfd1bf1
-  data.tar.gz: 392a61b3ee55ba8d9351c2b7d69d39de7d591bbde87f3374fac77fc425e628b8
+  metadata.gz: df6fa2a90625e5927b75d670dce66e5e4823cdb9af32ce64f1bdae0280a3146d
+  data.tar.gz: ac2ec11e30427a04952478b8b4293eb9be53ba6af3299ac8a7e6156f6eca14ac
 SHA512:
-  metadata.gz: 31f9277ada3b1a7fc09efe007a3b530fb3f100c480c09ab0af83a9879ab1fa3ea0b0db3252300fe8dbe9bb9877acf2244727a94554a6384bf52641a5a95c685b
-  data.tar.gz: 4346bae4957800e67e238c0231ddf1b424e1c5afe60a86e2d29e2134f480d56979d320514e24b412557f7f31b239fd7342ffdce459c387ba2b3ce56ce3e3e054
+  metadata.gz: f9efe6b0f7c05308fe8acfd7e569ca95adb3c85f8952fd70a820ef82f652054c0428ce0c7e773248c1e31d8e47c01d3a70791fedf34266016eeab07fe73eb224
+  data.tar.gz: a43d73f798e16f4388551666da9a8d573fa3e97442288a3d8b215014fa4593dbe07253c2cde4346146b79871b4daba3856342cf41588a6d43d18f7df01c40755

data/.gitignore

@@ -7,3 +7,6 @@ coverage/
 # YARD doc db and output
 .yardoc/
 doc/
+
+# Gem builds
+pkg/

data/CHANGELOG.md

@@ -0,0 +1,34 @@
+## 0.4.1
+
+ENHANCEMENTS:
+
+* Omit empty lines in generated config when using <% %> ERB templates.
+
+## 0.4.0
+
+NEW FEATURES:
+
+* Facter support. In your configomat config set a top level `facter` key to either truthy or a string. If truthy, facter data will be exposed in a profile named `facter` in all templates. If set to a string, facter data will be exposed in a profile with the given name in all templates.
+* Attempting to access a configuration variable from a profile or secret that does not exist using `#[]` will now raise an exception indicating the key being incorrectly accessed. If you need to access optionally present configuration variables from profiles or secrets use `#fetch(key, nil)`.
+
+## 0.3.0
+
+NEW FEATURES:
+
+* AWS Secrets Manager support, through an AWS AppConfig configuration. Values to pull from AWS Secrets Manager can be set using an "aws:secrets" key in any loaded AWS AppConfig JSON or YAML configuration. The value must be a dictionary. Keys in this dictionary will be exposed in the #secrets hash on the profile in templates. Values in this dictionary must be a dictionary containing at a minimum a `secret_id` key, which must be the id of an AWS Secrets Manager Secret. The dictionary may also contain a `version_id` or `version_stage` key to indicate which version of the secret to load, and a `content_type` key which may be one of `text/plan`, `application/json`, or `application/x-yaml` to indicate how the secret data should be parsed.
+
+## 0.2.1
+
+BUG FIXES:
+
+* restart_all actually works.
+
+## 0.2.0
+
+NEW FEATURES:
+
+* restart_all restart_mode for services to restart all running instances of an instantiated service.
+
+## 0.1.0
+
+Initial release

data/Gemfile

@@ -16,11 +16,4 @@
 
 source 'https://rubygems.org'
 
-gem 'aws-sdk-appconfig', '~> 1.18', require: false
-gem 'logsformyfamily', '~> 0.2', require: false
-gem 'lifecycle_vm', '~> 0.1.1', require: false
-gem 'ruby-dbus', '~> 0.16.0', require: false
-gem 'sd_notify', '~> 0.1', require: false
-
-gem 'rspec', '~> 3.10', group: :test, require: false
-gem 'simplecov', '~> 0.21', group: :test, require: false
+gemspec

data/Gemfile.lock

@@ -1,36 +1,55 @@
+PATH
+  remote: .
+  specs:
+    config_o_mat (0.4.0)
+      aws-sdk-appconfig (~> 1.18)
+      aws-sdk-secretsmanager (~> 1.57)
+      facter (~> 4.2, >= 4.2.8)
+      lifecycle_vm (~> 0.1.1)
+      logsformyfamily (~> 0.2)
+      ruby-dbus (~> 0.16.0)
+      sd_notify (~> 0.1.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.2.0)
-    aws-partitions (1.525.0)
-    aws-sdk-appconfig (1.19.0)
-      aws-sdk-core (~> 3, >= 3.122.0)
+    aws-partitions (1.574.0)
+    aws-sdk-appconfig (1.25.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.122.0)
+    aws-sdk-core (3.130.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
+    aws-sdk-secretsmanager (1.59.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
+      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    jmespath (1.4.0)
-    lifecycle_vm (0.1.2)
-    logsformyfamily (0.2.3)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    facter (4.2.9)
+      hocon (~> 1.3)
+      thor (>= 1.0.1, < 2.0)
+    hocon (1.3.1)
+    jmespath (1.6.1)
+    lifecycle_vm (0.1.3)
+    logsformyfamily (0.3.0)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     ruby-dbus (0.16.0)
     sd_notify (0.1.1)
     simplecov (0.21.2)
@@ -38,19 +57,16 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  aws-sdk-appconfig (~> 1.18)
-  lifecycle_vm (~> 0.1.1)
-  logsformyfamily (~> 0.2)
+  config_o_mat!
   rspec (~> 3.10)
-  ruby-dbus (~> 0.16.0)
-  sd_notify (~> 0.1)
-  simplecov (~> 0.21)
+  simplecov (~> 0.21.2)
 
 BUNDLED WITH
    2.1.4

data/config_o_mat.gemspec

@@ -39,10 +39,12 @@ Gem::Specification.new do |spec|
   spec.executables   = ["config_o_mat-configurator", "config_o_mat-meta_configurator"]
 
   spec.add_dependency('aws-sdk-appconfig', '~> 1.18')
+  spec.add_dependency('aws-sdk-secretsmanager', '~> 1.57')
   spec.add_dependency('logsformyfamily', '~> 0.2')
   spec.add_dependency('lifecycle_vm', '~> 0.1.1')
   spec.add_dependency('ruby-dbus', '~> 0.16.0')
   spec.add_dependency('sd_notify', '~> 0.1.1')
+  spec.add_dependency('facter', ['~> 4.2', '>= 4.2.8'])
 
   spec.add_development_dependency('simplecov', '~> 0.21.2')
   spec.add_development_dependency('rspec', '~> 3.10')

data/lib/config_o_mat/configurator/memory.rb

@@ -25,7 +25,8 @@ module ConfigOMat
                     :client_id, :compiled_templates, :applied_profiles, :applying_profile,
                     :generated_templates, :services_to_reload, :profiles_to_apply,
                     :last_refresh_time, :next_state, :retry_count, :retries_left, :retry_wait,
-                    :region, :appconfig_client, :systemd_interface
+                    :region, :appconfig_client, :secretsmanager_client, :systemd_interface,
+                    :secrets_loader_memory
 
       def initialize(
         argv: [],
@@ -55,7 +56,9 @@ module ConfigOMat
         retry_wait: 2,
         region: nil,
         appconfig_client: nil,
-        systemd_interface: nil
+        secretsmanager_client: nil,
+        systemd_interface: nil,
+        secrets_loader_memory: nil
       )
         super()
 
@@ -86,7 +89,9 @@ module ConfigOMat
         @retry_wait = retry_wait
         @region = region
         @appconfig_client = appconfig_client
+        @secretsmanager_client = secretsmanager_client
         @systemd_interface = systemd_interface
+        @secrets_loader_memory = secrets_loader_memory
       end
     end
   end

data/lib/config_o_mat/configurator/op/compile_templates.rb

@@ -28,7 +28,7 @@ module ConfigOMat
         self.compiled_templates = template_defs.each_with_object({}) do |(key, templ_def), hash|
           filename = File.join(configuration_directory, 'templates', templ_def.src)
           begin
-            templ = ERB.new(File.read(filename))
+            templ = ERB.new(File.read(filename), trim_mode: '<>')
             templ.filename = filename
             hash[key] = templ.def_class(Object, 'render(profiles)').new
           rescue SyntaxError, StandardError => e

data/lib/config_o_mat/configurator/op/{connect_to_appconfig.rb → connect_to_aws.rb}

@@ -17,18 +17,20 @@
 require 'lifecycle_vm/op_base'
 
 require 'aws-sdk-appconfig'
+require 'aws-sdk-secretsmanager'
 
 module ConfigOMat
   module Op
-    class ConnectToAppconfig < LifecycleVM::OpBase
+    class ConnectToAws < LifecycleVM::OpBase
       reads :region
-      writes :appconfig_client
+      writes :appconfig_client, :secretsmanager_client
 
       def call
         client_opts = { logger: logger }
         client_opts[:region] = region if region
 
         self.appconfig_client = Aws::AppConfig::Client.new(client_opts)
+        self.secretsmanager_client = Aws::SecretsManager::Client.new(client_opts)
       end
     end
   end

data/lib/config_o_mat/configurator/op/refresh_all_profiles.rb

@@ -16,47 +16,95 @@
 
 require 'lifecycle_vm/op_base'
 
+require 'config_o_mat/secrets_loader'
+
 module ConfigOMat
   module Op
     class RefreshAllProfiles < LifecycleVM::OpBase
-      reads :profile_defs, :applied_profiles, :client_id, :appconfig_client
-      writes :profiles_to_apply, :last_refresh_time
+      reads :profile_defs, :applied_profiles, :client_id, :appconfig_client, :secrets_loader_memory, :secretsmanager_client
+      writes :profiles_to_apply, :last_refresh_time, :secrets_loader_memory
 
       def call
         self.profiles_to_apply = []
         self.last_refresh_time = Time.now.to_i
 
         profile_defs.each do |(profile_name, definition)|
-          request = {
-            application: definition.application, environment: definition.environment,
-            configuration: definition.profile, client_id: client_id
-          }
-
-          current_version = applied_profiles&.fetch(profile_name, nil)&.version
-          request[:client_configuration_version] = current_version if current_version
-
-          response =
-            begin
-              appconfig_client.get_configuration(request)
-            rescue StandardError => e
-              error profile_name, e
-              nil
-            end
-
-          next if response.nil?
-
-          loaded_version = response.configuration_version
-          next if current_version && loaded_version == current_version
-
-          logger&.notice(
-            :updated_profile,
-            name: profile_name, previous_version: current_version, new_version: loaded_version
-          )
-
-          profiles_to_apply << LoadedProfile.new(
-            profile_name, loaded_version, response.content.read, response.content_type
-          )
+          if definition.kind_of?(ConfigOMat::Profile)
+            refresh_appconfig_profile(profile_name, definition)
+          elsif definition.kind_of?(ConfigOMat::FacterProfile)
+            refresh_facter_profile(profile_name)
+          else
+            error profile_name, "unknown profile type #{definition.class.name}"
+          end
+        end
+      end
+
+    private
+
+      def refresh_facter_profile(profile_name)
+        current_version = applied_profiles&.fetch(profile_name, nil)&.version
+        new_profile = ConfigOMat::LoadedFacterProfile.new(profile_name)
+
+        return if new_profile.version == current_version
+
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: current_version, new_version: new_profile.version
+        )
+
+        profiles_to_apply << LoadedProfile.new(new_profile, nil)
+      end
+
+      def refresh_appconfig_profile(profile_name, definition)
+        request = {
+          application: definition.application, environment: definition.environment,
+          configuration: definition.profile, client_id: client_id
+        }
+
+        current_version = applied_profiles&.fetch(profile_name, nil)&.version
+        request[:client_configuration_version] = current_version if current_version
+
+        response =
+          begin
+            appconfig_client.get_configuration(request)
+          rescue StandardError => e
+            error profile_name, e
+            nil
+          end
+
+        return if response.nil?
+
+        loaded_version = response.configuration_version
+        return if current_version && loaded_version == current_version
+
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: current_version, new_version: loaded_version
+        )
+
+        profile = LoadedAppconfigProfile.new(
+          profile_name, loaded_version, response.content.read, response.content_type
+        )
+
+        loaded_secrets = nil
+
+        if !profile.secret_defs.empty?
+          self.secrets_loader_memory ||= ConfigOMat::SecretsLoader::Memory.new(secretsmanager_client: secretsmanager_client)
+          secrets_loader_memory.update_secret_defs_to_load(profile.secret_defs.values)
+
+          vm = ConfigOMat::SecretsLoader::VM.new(secrets_loader_memory).call
+
+          if vm.errors?
+            error :"#{profile_name}_secrets", vm.errors
+            return
+          end
+
+          loaded_secrets = secrets_loader_memory.loaded_secrets.each_with_object({}) do |(key, value), hash|
+            hash[value.name] = value
+          end
         end
+
+        profiles_to_apply << LoadedProfile.new(profile, loaded_secrets)
       end
     end
   end

data/lib/config_o_mat/configurator/op/refresh_profile.rb

@@ -19,10 +19,42 @@ require 'lifecycle_vm/op_base'
 module ConfigOMat
   module Op
     class RefreshProfile < LifecycleVM::OpBase
-      reads :profile_defs, :client_id, :applying_profile, :appconfig_client
-      writes :applying_profile
+      reads :profile_defs, :client_id, :applying_profile, :appconfig_client, :secretsmanager_client, :secrets_loader_memory
+      writes :applying_profile, :secrets_loader_memory
 
       def call
+        if applying_profile.loaded_profile_data.kind_of?(ConfigOMat::LoadedAppconfigProfile)
+          refresh_appconfig_profile
+        elsif applying_profile.loaded_profile_data.kind_of?(ConfigOMat::LoadedFacterProfile)
+          refresh_facter_profile
+        end
+      end
+
+    private
+
+      def refresh_facter_profile
+        profile_name = applying_profile.name
+        profile_version = applying_profile.version
+        definition = profile_defs[profile_name]
+        new_profile = ConfigOMat::LoadedFacterProfile.new(profile_name)
+
+        if new_profile.version == profile_version
+          logger&.warning(
+            :no_update,
+            name: profile_name, version: profile_version
+          )
+          return
+        end
+
+        logger&.notice(
+          :updated_profile,
+          name: profile_name, previous_version: profile_version, new_version: new_profile.version
+        )
+
+        self.applying_profile = LoadedProfile.new(new_profile, nil)
+      end
+
+      def refresh_appconfig_profile
         profile_name = applying_profile.name
         profile_version = applying_profile.version
         definition = profile_defs[profile_name]
@@ -56,9 +88,29 @@ module ConfigOMat
           name: profile_name, previous_version: profile_version, new_version: loaded_version
         )
 
-        self.applying_profile = LoadedProfile.new(
+        profile = LoadedAppconfigProfile.new(
           profile_name, loaded_version, response.content.read, response.content_type
         )
+
+        loaded_secrets = nil
+
+        if !profile.secret_defs.empty?
+          self.secrets_loader_memory ||= ConfigOMat::SecretsLoader::Memory.new(secretsmanager_client: secretsmanager_client)
+          secrets_loader_memory.update_secret_defs_to_load(profile.secret_defs.values)
+
+          vm = ConfigOMat::SecretsLoader::VM.new(secrets_loader_memory).call
+
+          if vm.errors?
+            error :"#{profile_name}_secrets", vm.errors
+            return
+          end
+
+          loaded_secrets = secrets_loader_memory.loaded_secrets.each_with_object({}) do |(key, value), hash|
+            hash[value.name] = value
+          end
+        end
+
+        self.applying_profile = LoadedProfile.new(profile, loaded_secrets)
       end
     end
   end

data/lib/config_o_mat/configurator.rb

@@ -41,9 +41,9 @@ module ConfigOMat
 
       on :reading_meta_config, do: Op::LoadMetaConfig, then: :compiling_templates
 
-      on :compiling_templates, do: Op::CompileTemplates, then: :connecting_to_appconfig
+      on :compiling_templates, do: Op::CompileTemplates, then: :connecting_to_aws
 
-      on :connecting_to_appconfig, do: Op::ConnectToAppconfig, then: :refreshing_profiles
+      on :connecting_to_aws, do: Op::ConnectToAws, then: :refreshing_profiles
 
       on :refreshing_profiles,
          do: Op::RefreshAllProfiles,
@@ -116,7 +116,7 @@ module ConfigOMat
            }
          }
 
-      on :refreshing_profile, do: Op::RefreshProfile, then: :applying_profile
+      on :refreshing_profile, do: Op::RefreshProfile, then: :generating_templates
     end
   end
 end

data/lib/config_o_mat/secrets_loader/cond/requires_load.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class RequiresLoad < LifecycleVM::CondBase
+      reads :loading_secret
+
+      def call
+        !loading_secret.nil?
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/cond/secrets_to_load.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class SecretsToLoad < LifecycleVM::CondBase
+      reads :secret_defs_to_load
+
+      def call
+        !secret_defs_to_load.empty?
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/cond.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Dir[File.join(__dir__, 'cond', '*')].sort.each { |file| require file }

data/lib/config_o_mat/secrets_loader/memory.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm'
+
+module ConfigOMat
+  module SecretsLoader
+    class Memory < LifecycleVM::Memory
+      attr_reader :secretsmanager_client
+
+      attr_accessor :secrets_cache, :secret_defs_to_load, :loaded_secrets, :loading_secret
+
+      def initialize(
+        secretsmanager_client: nil,
+        secret_defs_to_load: [],
+        secrets_cache: {},
+        loading_secret: nil,
+        logger: nil
+      )
+        @secretsmanager_client = secretsmanager_client
+        @secret_defs_to_load = secret_defs_to_load
+        @secrets_cache = secrets_cache
+        @loading_secret = loading_secret
+        @loaded_secrets = {}
+        @logger = logger
+      end
+
+      def update_secret_defs_to_load(defs)
+        @loaded_secrets = {}
+        @secret_defs_to_load = defs
+        self
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/op/check_cache.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+module ConfigOMat
+  module Op
+    class CheckCache < LifecycleVM::OpBase
+      reads :loading_secret, :secrets_cache, :loaded_secrets
+      writes :loading_secret, :loaded_secrets
+
+      def call
+        # If we're referencing a secret by its version stage, never cache it, since these can be
+        # updated out from under us.
+        return unless loading_secret.version_stage.nil? || loading_secret.version_stage.empty?
+
+        cached_secret = secrets_cache[loading_secret.secret_id]
+
+        if cached_secret && cached_secret.version_id == loading_secret.version_id
+          logger&.info(:cached_secret, name: loading_secret.name, version_id: loading_secret.version_id)
+          loaded_secrets[loading_secret] = cached_secret
+          self.loading_secret = nil
+        elsif cached_secret
+          logger&.info(
+            :invalid_cached_secret,
+            name: loading_secret.name,
+            expected_version_id: loading_secret.version_id,
+            cached_version_id: cached_secret.version_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/op/load_secret.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+require 'config_o_mat/shared/types'
+
+module ConfigOMat
+  module Op
+    class LoadSecret < LifecycleVM::OpBase
+      reads :loading_secret, :secrets_cache, :loaded_secrets, :secretsmanager_client
+      writes :loading_secret, :loaded_secrets, :secrets_cache
+
+      def call
+        opts = {
+          secret_id: loading_secret.secret_id
+        }
+
+        if loading_secret.version_id
+          opts[:version_id] = loading_secret.version_id
+        else
+          opts[:version_stage] = loading_secret.version_stage
+        end
+
+        response =
+          begin
+            secretsmanager_client.get_secret_value(opts)
+          rescue StandardError => e
+            error loading_secret.name, e
+            nil
+          end
+
+        return if response.nil? || errors?
+
+        loaded_secret = LoadedSecret.new(
+          loading_secret.name, loading_secret.secret_id, response.version_id,
+          response.secret_string, loading_secret.content_type
+        )
+
+        logger&.info(
+          :loaded_secret, name: loading_secret.name, arn: response.arn,
+          version_id: response.version_id
+        )
+
+        begin
+          loaded_secret.validate!
+        rescue StandardError => e
+          logger&.error(
+            :invalid_secret, name: loading_secret.name, arn: response.arn,
+            version_id: response.version_id, errors: e
+          )
+          error loaded_secret.name, e
+        end
+
+        return if errors?
+
+        secrets_cache[loading_secret.secret_id] = loaded_secret
+        loaded_secrets[loading_secret] = loaded_secret
+        self.loading_secret = nil
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/op/stage_one_secret.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+module ConfigOMat
+  module Op
+    class StageOneSecret < LifecycleVM::OpBase
+      reads :secret_defs_to_load
+      writes :loading_secret, :secret_defs_to_load
+
+      def call
+        self.loading_secret = secret_defs_to_load.pop
+      end
+    end
+  end
+end

data/lib/config_o_mat/secrets_loader/op.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Dir[File.join(__dir__, 'op', '*')].sort.each { |file| require file }

data/lib/config_o_mat/secrets_loader.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm'
+
+require 'config_o_mat/secrets_loader/op'
+require 'config_o_mat/secrets_loader/cond'
+require 'config_o_mat/secrets_loader/memory'
+
+module ConfigOMat
+  module SecretsLoader
+    class VM < LifecycleVM::VM
+      memory_class ConfigOMat::SecretsLoader::Memory
+
+      on :start, then: {
+        case: Cond::SecretsToLoad,
+        when: {
+          true => {
+            do: Op::StageOneSecret,
+            then: :check_cache
+          },
+          false => {
+            then: :exit
+          }
+        }
+      }
+
+      on :check_cache, do: Op::CheckCache, then: {
+        case: Cond::RequiresLoad,
+        when: {
+          true => :load_secret,
+          false => :start
+        }
+      }
+
+      on :load_secret, do: Op::LoadSecret, then: :start
+    end
+  end
+end

data/lib/config_o_mat/shared/op/load_meta_config.rb

@@ -126,6 +126,16 @@ module ConfigOMat
         self.template_defs = instantiate.call(:templates, Template)
         self.profile_defs = instantiate.call(:profiles, Profile)
 
+        facter = merged_config[:facter]
+        if facter
+          facter_key = facter.kind_of?(String) ? facter.to_sym : :facter
+          if profile_defs.key?(facter_key)
+            error :facter, "conflicts with profile #{facter_key}"
+          else
+            profile_defs[facter_key] = ConfigOMat::FacterProfile.new
+          end
+        end
+
         self.logger = LogsForMyFamily::Logger.new if !logger
 
         log_type = merged_config[:log_type]&.to_sym || :stdout

data/lib/config_o_mat/shared/types.rb

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'facter'
+
 require 'json'
 require 'yaml'
 require 'digest'
@@ -187,9 +189,86 @@ module ConfigOMat
     end
   end
 
-  class LoadedProfile < ConfigItem
+  class FacterProfile < ConfigItem
+  end
+
+  class LoadedFacterProfile < ConfigItem
+    CLEAR_FROM_FACTER = [
+      "memoryfree", "memoryfree_mb", "load_averages", "uptime", "system_uptime", "uptime_seconds", "uptime_hours", "uptime_days",
+      {"ec2_metadata" => ["identity-credentials"]},
+      {"memory" => [{"system" => ["capacity", "available_bytes", "used", "used_bytes", "available"] }] }
+    ].freeze
+
     attr_reader :name, :version, :contents
 
+    def initialize(name)
+      @name = name
+      load_from_facter
+      @version = contents.hash
+    end
+
+    def validate
+      error :name, 'must be present' if @name.nil? || @name.empty?
+      error :contents, 'must be present' if @contents.nil? || @contents.empty?
+      error :contents, 'must be a hash' if !@contents.kind_of?(Hash)
+    end
+
+    def hash
+      @name.hash ^ @version
+    end
+
+    def to_h
+      @contents
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.version != version || other.name != name
+      true
+    end
+
+  private
+
+    def load_from_facter
+      Facter.clear
+      # This is to work around a bug in Facter wherein it fails to invalidate a second cache of the
+      # IMDSv2 token.
+      Facter::Resolvers::Ec2.instance_variable_set(:@v2_token, nil)
+      new_facts = Facter.to_hash
+      clear(new_facts, CLEAR_FROM_FACTER)
+      transform(new_facts)
+      @contents = new_facts
+    end
+
+    def clear(hash, diffs)
+      diffs.each do |diff|
+        if diff.kind_of?(Hash)
+          diff.each do |(key, values)|
+            clear(hash[key], values) if hash.key?(key)
+          end
+        elsif hash
+          hash.delete(diff)
+        end
+      end
+    end
+
+    def transform(hash)
+      return unless hash.kind_of?(Hash)
+      hash.transform_keys!(&:to_sym)
+      hash.default_proc = proc { |hash, key| raise KeyError.new("No key #{key.inspect} in profile #{name}", key: key, receiver: hash) }
+      hash.each_value do |value|
+        if value.kind_of?(Hash)
+          transform(value)
+        elsif value.kind_of?(Array)
+          value.each { |v| transform(v) }
+        end
+      end
+    end
+  end
+
+  class LoadedAppconfigProfile < ConfigItem
+    attr_reader :name, :version, :contents, :secret_defs
+
     PARSERS = {
       'text/plain' => proc { |str| str },
       'application/json' => proc { |str| JSON.parse(str, symbolize_names: true) },
@@ -199,12 +278,19 @@ module ConfigOMat
     def initialize(name, version, contents, content_type)
       @name = name
       @version = version
+      @secret_defs = {}
 
       parser = PARSERS[content_type]
 
       if parser
         begin
           @contents = parser.call(contents)
+          if @contents.kind_of?(Hash)
+            parse_secrets
+            @contents.default_proc = proc do |hash, key|
+              raise KeyError.new("No key #{key.inspect} in profile #{name}", key: key, receiver: hash)
+            end
+          end
         rescue StandardError => e
           error :contents, e
         end
@@ -233,6 +319,138 @@ module ConfigOMat
       return false if other.version != version || other.contents != contents || other.name != name
       true
     end
+
+  private
+
+    def parse_secrets
+      secret_entries = @contents.fetch(:"aws:secrets", nil)
+      return if secret_entries.nil?
+
+      error :contents_secrets, 'must be a dictionary' if !secret_entries.kind_of?(Hash)
+
+      secret_entries.each do |(secret_name, secret_conf)|
+        secret_def = Secret.new(secret_name, secret_conf)
+        secret_def.validate
+        error :"contents_secrets_#{secret_name}", secret_def.errors if secret_def.errors?
+
+        @secret_defs[secret_name] = secret_def
+      end
+    end
+  end
+
+  class Secret < ConfigItem
+    VALID_CONTENT_TYPES = LoadedAppconfigProfile::PARSERS.keys.freeze
+
+    attr_reader :name, :secret_id, :version_id, :version_stage, :content_type
+
+    def initialize(name, opts)
+      @name = name
+      @secret_id = opts[:secret_id]
+      @version_id = opts[:version_id]
+      @version_stage = opts[:version_stage]
+      @content_type = opts[:content_type]&.downcase
+
+      if (@version_id.nil? || @version_id.empty?) && (@version_stage.nil? || @version_stage.empty?)
+        @version_stage = 'AWSCURRENT'
+      end
+
+      @content_type ||= 'application/json'
+    end
+
+    def validate
+      error :secret_id, 'must be present' if @secret_id.nil? || @secret_id.empty?
+      error :content_type, "must be one of #{VALID_CONTENT_TYPES}" unless VALID_CONTENT_TYPES.include?(@content_type)
+    end
+
+    def hash
+      secret_id.hash ^ version_id.hash ^ version_stage.hash & content_type.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      if other.name != name ||
+         other.secret_id != secret_id ||
+         other.version_id != version_id ||
+         other.version_stage != version_stage ||
+         other.content_type != content_type
+        return false
+      end
+      true
+    end
+  end
+
+  class LoadedSecret < ConfigItem
+    attr_reader :name, :secret_id, :version_id, :contents
+
+    def initialize(name, secret_id, version_id, secret_string, content_type)
+      @name = name
+      @secret_id = secret_id
+      @version_id = version_id
+
+      begin
+        @contents = LoadedAppconfigProfile::PARSERS[content_type].call(secret_string)
+        if @contents.kind_of?(Hash)
+          @contents.default_proc = proc do |hash, key|
+            raise KeyError.new("No key #{key.inspect} in secret #{name}", key: key, receiver: hash)
+          end
+        end
+      rescue StandardError => e
+        error :contents, e
+      end
+    end
+
+    def validate
+      # Since name and version_id are coming from AWS and must be present, I'm not going to check
+      # them here.
+    end
+
+    def hash
+      @name.hash ^ @secret_id.hash ^ @version_id.hash ^ @contents.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      if other.name != name ||
+         other.secret_id != secret_id ||
+         other.version_id != version_id ||
+         other.contents != contents
+        return false
+      end
+      true
+    end
+  end
+
+
+  class LoadedProfile < ConfigItem
+    extend Forwardable
+
+    attr_reader :secrets, :loaded_profile_data
+
+    def_delegators :@loaded_profile_data, :name, :version, :contents
+
+    def initialize(loaded_profile_data, secrets)
+      @loaded_profile_data = loaded_profile_data
+      @secrets = secrets || {}
+
+      @errors = @loaded_profile_data.errors if @loaded_profile_data.errors?
+    end
+
+    def validate
+    end
+
+    def hash
+      @loaded_profile_data.hash ^ @secrets.hash
+    end
+
+    def to_h
+      contents
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.loaded_profile_data != @loaded_profile_data || other.secrets != @secrets
+      true
+    end
   end
 
   class GeneratedTemplate < ConfigItem

data/lib/config_o_mat/version.rb

@@ -15,5 +15,5 @@
 # limitations under the License.
 
 module ConfigOMat
-  VERSION = "0.2.1"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: config_o_mat
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.4.1
 platform: ruby
 authors:
 - Alex Scarborough
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-15 00:00:00.000000000 Z
+date: 2022-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-appconfig
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-secretsmanager
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.57'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.57'
 - !ruby/object:Gem::Dependency
   name: logsformyfamily
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.8
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -121,6 +155,7 @@ files:
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -144,7 +179,7 @@ files:
 - lib/config_o_mat/configurator/op/apply_all_profiles.rb
 - lib/config_o_mat/configurator/op/commit_staged_profile.rb
 - lib/config_o_mat/configurator/op/compile_templates.rb
-- lib/config_o_mat/configurator/op/connect_to_appconfig.rb
+- lib/config_o_mat/configurator/op/connect_to_aws.rb
 - lib/config_o_mat/configurator/op/generate_all_templates.rb
 - lib/config_o_mat/configurator/op/next_tick.rb
 - lib/config_o_mat/configurator/op/notify_systemd_start.rb
@@ -170,6 +205,15 @@ files:
 - lib/config_o_mat/meta_configurator/op.rb
 - lib/config_o_mat/meta_configurator/op/generate_systemd_config.rb
 - lib/config_o_mat/meta_configurator/op/parse_meta_cli.rb
+- lib/config_o_mat/secrets_loader.rb
+- lib/config_o_mat/secrets_loader/cond.rb
+- lib/config_o_mat/secrets_loader/cond/requires_load.rb
+- lib/config_o_mat/secrets_loader/cond/secrets_to_load.rb
+- lib/config_o_mat/secrets_loader/memory.rb
+- lib/config_o_mat/secrets_loader/op.rb
+- lib/config_o_mat/secrets_loader/op/check_cache.rb
+- lib/config_o_mat/secrets_loader/op/load_secret.rb
+- lib/config_o_mat/secrets_loader/op/stage_one_secret.rb
 - lib/config_o_mat/shared/cond.rb
 - lib/config_o_mat/shared/cond/early_exit.rb
 - lib/config_o_mat/shared/op.rb