config_o_mat 0.1.0

data/lib/config_o_mat/configurator/cond/first_run.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class FirstRun < LifecycleVM::CondBase
+      reads :run_count
+
+      def call
+        run_count.zero?
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/cond/next_state.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class NextState < LifecycleVM::CondBase
+      reads :next_state
+
+      def call
+        next_state
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/cond/profiles_to_apply.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class ProfilesToApply < LifecycleVM::CondBase
+      reads :profiles_to_apply
+
+      def call
+        !profiles_to_apply.empty?
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/cond/retries_left.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class RetriesLeft < LifecycleVM::CondBase
+      reads :error_op, :retries_left, :applying_profile
+
+      def call
+        logger&.error(:op_failure, op: error_op.class.name, errors: error_op.errors)
+
+        # If we aren't currently applying a profile then there's nothing for us to retry.
+        return false if applying_profile.nil?
+
+        # If we're out of retries, well, no more retrying
+        return false if retries_left.zero?
+
+        true
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/cond/services_to_reload.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/cond_base'
+
+module ConfigOMat
+  module Cond
+    class ServicesToReload < LifecycleVM::CondBase
+      reads :services_to_reload
+
+      def call
+        !services_to_reload.empty?
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/cond.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Dir[File.join(__dir__, 'cond', '*')].sort.each { |file| require file }

data/lib/config_o_mat/configurator/memory.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/memory'
+
+module ConfigOMat
+  module Configurator
+    class Memory < LifecycleVM::Memory
+      attr_accessor :argv, :env, :early_exit, :configuration_directory,
+                    :runtime_directory, :logs_directory, :run_count, :profile_defs,
+                    :template_defs, :service_defs, :dependencies, :refresh_interval,
+                    :client_id, :compiled_templates, :applied_profiles, :applying_profile,
+                    :generated_templates, :services_to_reload, :profiles_to_apply,
+                    :last_refresh_time, :next_state, :retry_count, :retries_left, :retry_wait,
+                    :region, :appconfig_client, :systemd_interface
+
+      def initialize(
+        argv: [],
+        env: {},
+        early_exit: false,
+        configuration_directory: nil,
+        runtime_directory: nil,
+        logs_directory: nil,
+        run_count: 0,
+        profile_defs: {},
+        template_defs: {},
+        service_defs: {},
+        dependencies: {},
+        refresh_interval: 5,
+        client_id: '',
+        logger: nil,
+        compiled_templates: {},
+        applied_profiles: {},
+        profiles_to_apply: [],
+        applying_profile: nil,
+        generated_templates: {},
+        services_to_reload: Set.new,
+        last_refresh_time: 0,
+        next_state: :running,
+        retry_count: 3,
+        retries_left: 3,
+        retry_wait: 2,
+        region: nil,
+        appconfig_client: nil,
+        systemd_interface: nil
+      )
+        super()
+
+        @argv = argv
+        @env = env
+        @early_exit = early_exit
+        @configuration_directory = configuration_directory
+        @runtime_directory = runtime_directory
+        @logs_directory = logs_directory
+        @run_count = run_count
+        @profile_defs = profile_defs
+        @template_defs = template_defs
+        @service_defs = service_defs
+        @dependencies = dependencies
+        @refresh_interval = refresh_interval
+        @client_id = client_id
+        @logger = logger
+        @compiled_templates = compiled_templates
+        @applied_profiles = applied_profiles
+        @profiles_to_apply = profiles_to_apply
+        @applying_profile = applying_profile
+        @generated_templates = generated_templates
+        @services_to_reload = services_to_reload
+        @last_refresh_time = last_refresh_time
+        @next_state = next_state
+        @retry_count = retry_count
+        @retries_left = retries_left
+        @retry_wait = retry_wait
+        @region = region
+        @appconfig_client = appconfig_client
+        @systemd_interface = systemd_interface
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/apply_all_profiles.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+module ConfigOMat
+  module Op
+    class ApplyAllProfiles < LifecycleVM::OpBase
+      reads :profiles_to_apply, :applied_profiles
+      writes :profiles_to_apply, :applied_profiles
+
+      def call
+        profiles_to_apply.each do |profile|
+          applied_profiles[profile.name] = profile
+          error profile.name, profile.errors if profile.errors?
+        end
+
+        self.profiles_to_apply = []
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/commit_staged_profile.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+module ConfigOMat
+  module Op
+    class CommitStagedProfile < LifecycleVM::OpBase
+      reads :applied_profiles, :applying_profile
+      writes :applied_profiles, :applying_profile
+
+      def call
+        # This happens in the first boot case, where we apply all profiles at once and so there's no
+        # individual applying_profile.
+        return if applying_profile.nil?
+
+        applied_profiles[applying_profile.name] = applying_profile
+        self.applying_profile = nil
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/compile_templates.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+require 'erb'
+
+module ConfigOMat
+  module Op
+    class CompileTemplates < LifecycleVM::OpBase
+      reads :template_defs, :configuration_directory
+      writes :compiled_templates
+
+      def call
+        self.compiled_templates = template_defs.each_with_object({}) do |(key, templ_def), hash|
+          filename = File.join(configuration_directory, 'templates', templ_def.src)
+          begin
+            templ = ERB.new(File.read(filename))
+            templ.filename = filename
+            hash[key] = templ.def_class(Object, 'render(profiles)').new
+          rescue SyntaxError, StandardError => e
+            error filename, e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/connect_to_appconfig.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+require 'aws-sdk-appconfig'
+
+module ConfigOMat
+  module Op
+    class ConnectToAppconfig < LifecycleVM::OpBase
+      reads :region
+      writes :appconfig_client
+
+      def call
+        client_opts = { logger: logger }
+        client_opts[:region] = region if region
+
+        self.appconfig_client = Aws::AppConfig::Client.new(client_opts)
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/generate_all_templates.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+module ConfigOMat
+  module Op
+    class GenerateAllTemplates < LifecycleVM::OpBase
+      reads :template_defs, :dependencies, :applied_profiles, :applying_profile, :generated_templates,
+            :compiled_templates, :runtime_directory
+
+      writes :generated_templates, :services_to_reload
+
+      def call
+        # By doing the error check here instead of in StageOneProfile we ensure that the applying_profile
+        # gets set in memory, even if errored, which simplifies the retry logic.
+        error applying_profile.name, applying_profile.errors if applying_profile&.errors?
+
+        return if errors?
+
+        profiles = applied_profiles
+        profiles[applying_profile.name] = applying_profile if applying_profile
+        self.services_to_reload = []
+
+        template_defs.each do |(key, templ_def)|
+          compiled_template = compiled_templates[key]
+
+          begin
+            rendered = compiled_template.render(profiles)
+          rescue StandardError => e
+            error key, e
+            next
+          end
+
+          generated = GeneratedTemplate.new(rendered)
+
+          if generated_templates[key] != generated
+            destination_file = File.join(runtime_directory, templ_def.dst)
+            logger&.notice(:template_update, template: key, file: destination_file, digest: generated.digest)
+
+            generated_templates[key] = generated
+            File.open(destination_file, 'w') { |f| f.write(rendered) }
+            dependencies[key]&.each { |service| services_to_reload << service }
+          end
+        end
+
+        services_to_reload.uniq!
+
+        logger&.notice(:scheduled_restarts, services: services_to_reload)
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/next_tick.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+require 'sd_notify'
+
+module ConfigOMat
+  module Op
+    class NextTick < LifecycleVM::OpBase
+      PAUSE_INTERVAL = 1
+
+      reads :last_refresh_time, :refresh_interval, :run_count, :retry_count
+      writes :next_state, :run_count, :retries_left
+
+      def call
+        self.run_count += 1
+
+        SdNotify.watchdog
+
+        # If we got here then our retry process has succeeded.
+        self.retries_left = retry_count
+
+        # I'm calling Kernel.sleep directly so that tests can easily stub it out.
+        Kernel.sleep PAUSE_INTERVAL
+
+        if Time.now.to_i - last_refresh_time > refresh_interval
+          self.next_state = :refreshing_profiles
+        else
+          self.next_state = :running
+        end
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/notify_systemd_start.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+
+require 'sd_notify'
+
+module ConfigOMat
+  module Op
+    class NotifySystemdStart < LifecycleVM::OpBase
+      def call
+        SdNotify.ready
+      end
+    end
+  end
+end

data/lib/config_o_mat/configurator/op/parse_cli.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+require 'optparse'
+
+module ConfigOMat
+  module Op
+    class ParseCli < LifecycleVM::OpBase
+      reads :argv, :env
+      writes :configuration_directory, :runtime_directory, :logs_directory, :early_exit
+
+      # Add expand_path to string because the safe accessor is nice to use.
+      module CoreExt
+        refine String do
+          def expand_path
+            File.expand_path(self)
+          end
+        end
+      end
+
+      using CoreExt
+
+      def call
+        self.early_exit = false
+
+        parser = OptionParser.new do |opts|
+          opts.on('-v', '--version', 'Display the version') do
+            self.early_exit = true
+          end
+
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            self.early_exit = true
+          end
+
+          opts.on(
+            '-c', '--configuration-directory directory',
+            'Read configuration from the given directory instead of from $CONFIGURATION_DIRECTORY'
+          ) do |dir|
+            self.configuration_directory = dir
+          end
+
+          opts.on(
+            '-r', '--runtime-directory directory',
+            'Use the given directory for writing templates instead of $RUNTIME_DIRECTORY'
+          ) do |dir|
+            self.runtime_directory = dir
+          end
+
+          opts.on(
+            '-l', '--logs-directory directory',
+            'Use the given directory for writing log files instead of $LOGS_DIRECTORY'
+          ) do |dir|
+            self.logs_directory = dir
+          end
+        end
+
+        parser.parse(argv)
+
+        self.configuration_directory ||= env['CONFIGURATION_DIRECTORY']
+        self.runtime_directory ||= env['RUNTIME_DIRECTORY']
+        self.logs_directory ||= env['LOGS_DIRECTORY']
+
+        self.configuration_directory = configuration_directory&.expand_path
+        self.runtime_directory = runtime_directory&.expand_path
+      end
+
+      def validate
+        return if early_exit
+
+        if configuration_directory.nil? || configuration_directory.empty?
+          error :configuration_directory, 'must be present'
+        end
+
+        error :runtime_directory, 'must be present' if runtime_directory.nil? || runtime_directory.empty?
+      end
+    end
+  end
+end