config_o_mat 0.1.0

data/lib/config_o_mat/shared/op/load_meta_config.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lifecycle_vm/op_base'
+require 'config_o_mat/shared/types'
+require 'config_o_mat/shared/systemd_interface'
+
+require 'logsformyfamily'
+
+require 'securerandom'
+require 'yaml'
+
+module ConfigOMat
+  module Op
+    class LoadMetaConfig < LifecycleVM::OpBase
+      module DeepMerge
+        refine Hash do
+          def deep_merge(other)
+            dup.deep_merge!(other)
+          end
+
+          def deep_merge!(other)
+            merge!(other) do |key, this_val, other_val|
+              if this_val.respond_to?(:deep_merge) &&
+                 this_val.is_a?(other_val.class)
+                this_val.deep_merge(other_val)
+              else
+                other_val
+              end
+            end
+          end
+        end
+
+        refine Array do
+          def deep_merge(other)
+            dup.deep_merge!(other)
+          end
+
+          def deep_merge!(other)
+            concat(other)
+          end
+        end
+      end
+
+      using DeepMerge
+
+      LOG_TYPES = %i[stdout file].freeze
+      LOG_CONFIG_KEYS = %i[log_level log_type log_file].freeze
+
+      reads :configuration_directory, :logs_directory, :env
+      writes :profile_defs, :template_defs, :service_defs, :dependencies,
+             :refresh_interval, :client_id, :logger, :retry_count, :retries_left,
+             :retry_wait, :region, :systemd_interface
+
+      def call
+        default_config = {
+          refresh_interval: 5,
+          client_id: env.fetch('INVOCATION_ID') { SecureRandom.uuid },
+          retry_count: 3,
+          retry_wait: 2,
+          services: [],
+          templates: [],
+          profiles: [],
+          region: nil
+        }
+
+        # TODO: I would like to make this configurable. I think the trick
+        # Sequel uses for its model classes (< Sequel::Model(source_dataset))
+        # might be appropriate for how this system works?
+        # do: Op::LoadMetaConfig(parser: Yaml.method(&:safe_load))
+        parser = proc { |file| YAML.safe_load(file, symbolize_names: true) }
+        file_ending = '.conf'
+
+        files =
+          Dir.children(configuration_directory)
+             .lazy
+             .select { |f| f.end_with?(file_ending) }
+             .map { |f| File.join(configuration_directory, f) }
+             .select { |f| File.file?(f) }
+             .to_a
+             .sort!
+
+        loaded_files =
+          files.each_with_object({}) do |file, hash|
+            hash[file] = parser.call(File.read(file))
+          rescue StandardError => e
+            error file, e
+          end
+
+        # If we couldn't load a configuration file then it's probably not worth
+        # exploding on any objects we fail to initialize -- that's an obvious
+        # consequence and the additional noise might mask the root cause.
+        return if errors?
+
+        merged_config =
+          loaded_files.each_with_object(default_config) do |(_filename, config), memo|
+            memo.deep_merge!(config)
+          end
+
+        instantiate = proc do |key, klass|
+          merged_config[key].each_with_object({}) do |(name, obj), defs|
+            definition = klass.new(obj)
+            definition.validate!
+            defs[name] = definition
+          rescue StandardError => e
+            error key, { name => e }
+          end
+        end
+
+        logger&.info(:log_config, configuration: merged_config.slice(*LOG_CONFIG_KEYS))
+
+        self.service_defs = instantiate.call(:services, Service)
+        self.template_defs = instantiate.call(:templates, Template)
+        self.profile_defs = instantiate.call(:profiles, Profile)
+
+        self.logger = LogsForMyFamily::Logger.new if !logger
+
+        log_type = merged_config[:log_type]&.to_sym || :stdout
+        error :log_type, "must be one of #{LOG_TYPES.map(&:to_s)}" if log_type && !LOG_TYPES.include?(log_type)
+
+        log_level = merged_config[:log_level]&.to_sym
+        if log_level && !LogsForMyFamily::Logger::LEVELS.include?(log_level)
+          error :log_level, "must be one of #{LogsForMyFamily::Logger::LEVELS}"
+        end
+
+        backend =
+          if log_type == :file
+            log_file = merged_config[:log_file] || 'configurator.log'
+            if logs_directory.nil?
+              error :log_type, 'must set logs directory with -l or $LOGS_DIRECTORY to set log_type to file'
+            else
+              FileLogWriter.new(File.join(logs_directory, log_file))
+            end
+          else
+            StdoutLogWriter.new
+          end
+
+        # If we couldn't initialize our logger (or anything else) then bail here before
+        # we try to use it.
+        return if errors?
+
+        logger.filter_level(log_level) if log_level
+        logger.backends = [backend]
+
+        # Re-log our merged config with our configured logger.
+        logger.info(:parsed_config, configuration: merged_config)
+
+        self.refresh_interval = merged_config[:refresh_interval]
+        self.client_id = merged_config[:client_id]
+        self.retry_count = merged_config[:retry_count]
+        self.retries_left = retry_count
+        self.retry_wait = merged_config[:retry_wait]
+        self.region = merged_config[:region]
+
+        self.dependencies = service_defs.each_with_object({}) do |(name, service), template_to_services|
+          service.templates.each do |template|
+            template = template.to_sym
+            if !template_defs.key?(template)
+              error :services, { name => "references undefined template #{template}" }
+            else
+              # Listing the same template multiple times is acceptable. Since we allow
+              # merging config files, and this deep merges the service dependency list,
+              # it's quite possible that a service could inadvertently declare the same
+              # dependency twice in a way that's not easy to untangle.
+              template_to_services[template] ||= Set.new
+              template_to_services[template] << name
+            end
+          end
+        end
+
+        self.systemd_interface = SystemdInterface.new(DBus.system_bus)
+      end
+    end
+  end
+end

data/lib/config_o_mat/shared/op.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Dir[File.join(__dir__, 'op', '*')].sort.each { |file| require file }

data/lib/config_o_mat/shared/systemd_interface.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dbus'
+
+module ConfigOMat
+  class SystemdInterface
+    SERVICE_NAME = 'org.freedesktop.systemd1'
+    OBJECT_PATH = '/org/freedesktop/systemd1'
+    MANAGER_INTERFACE = 'org.freedesktop.systemd1.Manager'
+
+    UNIT_INTERFACE = 'org.freedesktop.systemd1.Unit'
+    UNIT_STATE = 'ActiveState'
+
+    def initialize(sysbus = nil)
+      if sysbus
+        @sysd_service = sysbus[SERVICE_NAME]
+        obj = @sysd_service[OBJECT_PATH]
+        @sysd_iface = obj[MANAGER_INTERFACE]
+      end
+    end
+
+    def service_interface(unit_name)
+      unit_path = @sysd_iface.GetUnit("#{unit_name}.service")
+      unit_obj = @sysd_service[unit_path]
+      unit_obj[UNIT_INTERFACE]
+    rescue DBus::Error
+      nil
+    end
+
+    def service_status(unit_name)
+      iface = service_interface(unit_name)
+      if iface
+        iface[UNIT_STATE]
+      else
+        'inactive'
+      end
+    end
+
+    def enable_restart_paths(units)
+      path_units = units.map { |unit| "teak-configurator-restart-service@#{unit}.path" }
+      enable_and_start(path_units)
+    end
+
+    def enable_start_stop_paths(units)
+      path_units = units.flat_map do |unit|
+        ["teak-configurator-start-service@#{unit}.path", "teak-configurator-stop-service@#{unit}.path"]
+      end
+      enable_and_start(path_units)
+    end
+
+    def daemon_reload
+      @sysd_iface.Reload()
+    end
+
+    def ==(other)
+      eql?(other)
+    end
+
+    def eql?(other)
+      return false if !other.is_a?(self.class)
+      return false if other.instance_variable_get(:@sysd_service) != @sysd_service
+      true
+    end
+
+  private
+
+    def enable_and_start(units)
+      @sysd_iface.EnableUnitFiles(
+        units,
+        true, # Runtime
+        false # force
+      )
+      daemon_reload
+      units.each do |unit|
+        @sysd_iface.StartUnit(unit, 'replace')
+      end
+    end
+  end
+end

data/lib/config_o_mat/shared/types.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'json'
+require 'yaml'
+require 'digest'
+
+module ConfigOMat
+  class LogWriter
+    def call(level_name, event_type, merged_data)
+      merged_data[:level] = level_name
+      merged_data[:event_type] = event_type
+
+      write { "#{JSON.generate(merged_data)}\n" }
+    end
+  end
+
+  class StdoutLogWriter < LogWriter
+    def write
+      $stdout.write(yield)
+      $stdout.flush
+    end
+  end
+
+  class FileLogWriter < LogWriter
+    attr_reader :file_path
+
+    def initialize(file_path)
+      @file_path = file_path
+    end
+
+    def write
+      File.open(@file_path, 'a') { |f| f.write(yield) }
+    end
+  end
+
+  class ConfigItem
+    class ValidationError < RuntimeError
+      attr_reader :errors
+
+      def initialize(errors)
+        @errors = errors
+        super(errors.to_s)
+      end
+    end
+
+    attr_reader :errors
+
+    def errors?
+      instance_variable_defined?(:@errors) && !(@errors.nil? || @errors.empty?)
+    end
+
+    def error(field, message)
+      @errors ||= {}
+      @errors[field] ||= []
+      @errors[field] << message
+    end
+
+    def validate!
+      validate
+      raise ValidationError.new(errors) if errors?
+    end
+
+    def eql?(other)
+      return false if !other.is_a?(self.class)
+      true
+    end
+
+    def ==(other)
+      eql?(other)
+    end
+  end
+
+  class Service < ConfigItem
+    RESTART_MODES = %i[restart flip_flop].freeze
+
+    attr_reader :systemd_unit, :restart_mode, :templates
+
+    def initialize(opts)
+      @systemd_unit = (opts[:systemd_unit] || '')
+      @restart_mode = opts[:restart_mode]&.to_sym
+      @templates = opts[:templates]
+
+      if @restart_mode == :flip_flop && !@systemd_unit.include?('@')
+        @systemd_unit = "#{@systemd_unit}@"
+      end
+    end
+
+    def validate
+      error :templates, 'must be present' if @templates.nil? || @templates.empty?
+      unless @templates.is_a?(Array) && @templates.all? { |v| v.is_a?(String) }
+        error :templates, 'must be an array of strings'
+      end
+      error :systemd_unit, 'must be present' if @systemd_unit.nil? || @systemd_unit.empty? || @systemd_unit == '@'
+      error :restart_mode, "must be one of #{RESTART_MODES}" unless RESTART_MODES.include?(@restart_mode)
+
+      if @restart_mode == :flip_flop && !@systemd_unit.end_with?('@')
+        error :systemd_unit, 'must not contain an instance (anything after a @)'
+      end
+
+      if restart_mode == :restart && @systemd_unit.end_with?('@')
+        error :systemd_unit, 'must not be a naked instantiated unit when restart_mode=restart'
+      end
+    end
+
+    def hash
+      systemd_unit.hash ^ restart_mode.hash ^ templates.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.systemd_unit != systemd_unit || other.restart_mode != restart_mode || other.templates != templates
+      true
+    end
+  end
+
+  class Template < ConfigItem
+    attr_reader :src, :dst
+
+    def initialize(opts)
+      @src = opts[:src]
+      @dst = opts[:dst]
+    end
+
+    def validate
+      error :src, 'must be present' if @src.nil? || @src.empty?
+      error :dst, 'must be present' if @dst.nil? || @dst.empty?
+    end
+
+    def hash
+      src.hash ^ dst.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.src != src || other.dst != dst
+      true
+    end
+  end
+
+  class Profile < ConfigItem
+    attr_reader :application, :environment, :profile
+
+    def initialize(opts)
+      @application = opts[:application]
+      @environment = opts[:environment]
+      @profile = opts[:profile]
+    end
+
+    def validate
+      error :application, 'must be present' if @application.nil? || @application.empty?
+      error :environment, 'must be present' if @environment.nil? || @environment.empty?
+      error :profile, 'must be present' if @profile.nil? || @profile.empty?
+    end
+
+    def hash
+      application.hash ^ environment.hash ^ profile.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.application != application || other.environment != environment || other.profile != profile
+      true
+    end
+  end
+
+  class LoadedProfile < ConfigItem
+    attr_reader :name, :version, :contents
+
+    PARSERS = {
+      'text/plain' => proc { |str| str },
+      'application/json' => proc { |str| JSON.parse(str, symbolize_names: true) },
+      'application/x-yaml' => proc { |str| YAML.safe_load(str, symbolize_names: true) }
+    }.freeze
+
+    def initialize(name, version, contents, content_type)
+      @name = name
+      @version = version
+
+      parser = PARSERS[content_type]
+
+      if parser
+        begin
+          @contents = parser.call(contents)
+        rescue StandardError => e
+          error :contents, e
+        end
+      else
+        error :content_type, "must be one of #{PARSERS.keys}"
+      end
+    end
+
+    def validate
+      error :name, 'must be present' if @name.nil? || @name.empty?
+      error :name, 'must be a Symbol' unless @name.is_a?(Symbol)
+      error :version, 'must be present' if @version.nil? || @version.empty?
+      error :contents, 'must be present' if @contents.nil? || @contents.empty?
+    end
+
+    def hash
+      @name.hash ^ @version.hash ^ @contents.hash
+    end
+
+    def to_h
+      @contents
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.version != version || other.contents != contents || other.name != name
+      true
+    end
+  end
+
+  class GeneratedTemplate < ConfigItem
+    attr_reader :digest
+
+    def initialize(contents)
+      @digest = Digest::SHA256.hexdigest(contents)
+    end
+
+    def validate
+    end
+
+    def hash
+      @digest.hash
+    end
+
+    def eql?(other)
+      return false if !super(other)
+      return false if other.digest != digest
+      true
+    end
+  end
+end

data/lib/config_o_mat/version.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module ConfigOMat
+  VERSION = "0.1.0"
+end

data/lib/version.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module ConfigOMat
+  VERSION = "0.1.0"
+end

data/systemd/teak-configurator-restart-service@.path

@@ -0,0 +1,27 @@
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# By enabling this unit for a service, we ensure that after the service starts systemd
+# listens for touches on the configurator's restart file, and will run teak-configurator-restart-service@%i.service
+# when the file is touched.
+#
+# This allows the configurator to restart systemd services without privileged access.
+[Unit]
+Description=Touch a file to restart %i.service
+
+[Path]
+PathModified=/run/teak-configurator/%i.restart
+
+[Install]
+WantedBy=%i.service

data/systemd/teak-configurator-restart-service@.service

@@ -0,0 +1,21 @@
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=Restart %i.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/systemctl restart %i.service
+RemainAfterExit=no

data/systemd/teak-configurator-start-service@.path

@@ -0,0 +1,27 @@
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# By enabling this unit for a service, we ensure that after the service starts systemd
+# listens for touches on the configurator's start file, and will run teak-configurator-start-service@%i.service
+# when the file is touched.
+#
+# This allows the configurator to start systemd services without privileged access.
+[Unit]
+Description=Touch a file to start %i.service
+
+[Path]
+PathModified=/run/teak-configurator/%i.start
+
+[Install]
+WantedBy=%i.service

data/systemd/teak-configurator-start-service@.service

@@ -0,0 +1,21 @@
+# Copyright 2021 Teak.io, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=Start %i.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/systemctl start %i.service
+RemainAfterExit=no