cem_acpt 0.1.0

data/lib/cem_acpt/bootstrap/operating_system/rhel_family.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+# Currently unused in this implementation.
+module CemAcpt::Bootstrap::OperatingSystem
+  # This module holds methods used by Bootstrap for RHEL-family operating systems
+  module RhelFamily
+    def puppet_agent_repo
+      "https://yum.puppet.com/#{@collection}-release-el-#{@major_version}.noarch.rpm"
+    end
+
+    def package_manager
+      return 'dnf' if @major_version.to_i >= 8
+
+      'yum'
+    end
+
+    def rvm_deps
+      [
+        'kernel-devel',
+        'gcc',
+        'gcc-c++',
+        'make',
+        'augeas',
+        'augeas-devel',
+        'patch',
+        'readline',
+        'readline-devel',
+        'zlib',
+        'zlib-devel',
+        'libffi-devel',
+        'openssl-devel',
+        'bzip2',
+        'autoconf',
+        'automake',
+        'libtool',
+        'bison',
+        'sqlite-devel',
+      ]
+    end
+
+    def repo_installs(repos, sudo: true)
+      return if repos.nil? || repos.empty?
+
+      cmd = [
+        repo_install_cmd(repos, sudo: sudo),
+        repo_install_verify_cmd(repos, sudo: sudo),
+      ].join(' && ')
+      "#{cmd} | sudo tee -ai #{@log_file}"
+    end
+
+    def package_installs(packages, sudo: true)
+      return if packages.nil? || packages.empty?
+
+      packages += rvm_deps
+      cmd = [
+        package_install_cmd(packages, sudo: sudo),
+        package_install_verify_cmd(packages, sudo: sudo),
+      ].join(' && ')
+      "#{cmd} | sudo tee -ai #{@log_file}"
+    end
+
+    def service_starts(services, sudo: true)
+      return if services.nil? || services.empty?
+
+      cmd = [
+        #service_start_verify_existance_cmd(services, sudo: sudo),
+        service_start_cmd(services, sudo: sudo),
+      ].join(' && ')
+      "#{cmd} | sudo tee -ai #{@log_file}"
+    end
+
+    def repo_install_cmd(repos, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << 'rpm -Uv'
+      repos.each { |r| cmd << r }
+      cmd.join(' ')
+    end
+
+    def repo_install_verify_cmd(repos, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << 'rpm -q'
+      repos.each { |r| cmd << r }
+      cmd.join(' ')
+    end
+
+    def package_install_cmd(packages, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << "#{package_manager} install #{package_install_cmd_opts}"
+      packages.each { |p| cmd << p }
+      cmd << package_install_cmd_output_format
+      cmd.join(' ')
+    end
+
+    def package_install_verify_cmd(packages, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << "#{package_manager} list installed"
+      packages.each { |p| cmd << p }
+      cmd.join(' ')
+    end
+
+    def package_install_cmd_opts
+      '-y'
+    end
+
+    def package_install_cmd_output_format
+      '| tr "\n" "#" | sed -e \'s/# / /g\' | tr "#" "\n"'
+    end
+
+    def service_start_cmd(services, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << 'systemctl start'
+      services.each { |s| cmd << s }
+      cmd.join(' ')
+    end
+
+    def service_start_verify_existance_cmd(services, sudo: true)
+      cmd = []
+      cmd << 'sudo' if sudo
+      cmd << 'systemctl list-units --type=service'
+      services.each { |s| cmd << s }
+      cmd.join(' ')
+    end
+  end
+end

data/lib/cem_acpt/bootstrap/operating_system.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Currently unused in this implementation.
+module CemAcpt::Bootstrap::OperatingSystem
+  class Error < StandardError; end
+
+  # Currently unused in this implementation.
+  def use_os(os)
+    case os
+    when %r{^(centos|rhel)$}
+      require_relative 'operating_system/rhel_family'
+      self.class.include CemAcpt::Bootstrap::OperatingSystem::RhelFamily
+    else
+      raise Error, "Operating system #{os} is not supported"
+    end
+  end
+end

data/lib/cem_acpt/bootstrap.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Bootstrap provides a method for bootstrapping test nodes.
+# Currently unused in this implementation.
+module CemAcpt::Bootstrap
+  require_relative 'bootstrap/bootstrapper'
+
+  def self.run(instance_name, instance_image, cmd_provider, collection: 'puppet7', repos: [], packages: [], services: [], commands: [], cmd_provider_args: {})
+    bootstrapper = CemAcpt::Bootstrap::Bootstrapper.new(instance_name, instance_image, cmd_provider, collection: 'puppet7', repos: [], packages: [], services: [], commands: [], cmd_provider_args: {})
+    bootstrapper.run
+  end
+end

data/lib/cem_acpt/context.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+module CemAcpt
+  # Context provides the context in which the RunHandler creates and starts Runners.
+  module Context
+    require_relative 'platform'
+    require_relative 'shared_objects'
+    require_relative 'test_data'
+    require_relative 'logging'
+
+    class << self
+      include CemAcpt::Logging
+    end
+
+    # Creates a context for the RunHandler to create and start Runners.
+    # Provides the following objects for the Runners: a config object,
+    # the test data hash, the node inventory, and the local port allocator.
+    # Additionally, it creates the platform-specific node objects for each
+    # test suite in the test data. It then calls the provided block with
+    # the context objects nodes, config, test_data, and node_inventory.
+    # @param config_opts [Hash] the config options
+    # @param config_file [String] the config file
+    def self.with(config_opts: nil, config_file: nil, &block)
+      logger.debug('In context with:')
+      logger.debug("config_opts: #{config_opts}")
+      logger.debug("config_file: #{config_file}")
+
+      config = CemAcpt::Config.new.load(opts: config_opts, config_file: config_file)
+      logger.debug("Loaded config: #{config.to_h}")
+
+      test_data = CemAcpt::TestData.acceptance_test_data(config)
+      logger.debug("Loaded test data: #{test_data}")
+
+      node_inventory = CemAcpt::NodeInventory.new
+      logger.debug("Initialized new node inventory: #{node_inventory}")
+
+      local_port_allocator = CemAcpt::LocalPortAllocator.new
+      logger.debug("Initialized new local port allocator: #{local_port_allocator}")
+
+      if config.has?('platform')
+        logger.debug("Using platform: #{config.get('platform')}")
+        nodes = CemAcpt::Platform.use(config.get('platform'), config, test_data, local_port_allocator)
+        block.call(nodes, config, test_data, node_inventory)
+      elsif config.has?('platforms')
+        config.get('platforms').each do |platform|
+          logger.debug("Using platform: #{platform}")
+          nodes = CemAcpt::Platform.use(platform, config, test_data, local_port_allocator)
+          block.call(nodes, config, test_data, node_inventory)
+        end
+      else
+        raise CemAcpt::Error, 'No platform(s) specified'
+      end
+    rescue StandardError => e
+      logger.fatal("Fatal error: #{e.message}")
+      logger.debug(e.backtrace.join('; '))
+      exit(1)
+    end
+  end
+end

data/lib/cem_acpt/core_extensions.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# This module holds extensions to Ruby and the Ruby stdlib
+# Extensions related to deep_freeze were pulled from: https://gist.github.com/steakknife/1a37057b3b8539f4aca3
+module CemAcpt::CoreExtensions
+  # DeepFreeze recursively freezes all keys and values in a hash
+  # Currently unused, but was used at one point and may be useful again
+  module DeepFreeze
+    # Holds deep_freeze extensions to Kernel
+    module Kernel
+      alias deep_freeze freeze
+      alias deep_frozen? frozen?
+    end
+
+    # Holds deep_freeze extensions to Enumerable
+    module Enumerable
+      def deep_freeze
+        unless @deep_frozen
+          each(&:deep_freeze)
+          @deep_frozen = true
+        end
+        freeze
+      end
+
+      def deep_frozen?
+        !!@deep_frozen
+      end
+    end
+
+    # Holds deep_freeze extensions to Hash
+    module Hash
+      def deep_freeze
+        transform_values! do |value|
+          value.respond_to?(:deep_freeze) ? value.deep_freeze : value.freeze
+        end
+        freeze
+        @deep_frozen = true
+      end
+
+      def deep_frozen?
+        !!@deep_frozen
+      end
+    end
+
+    # Holds deep_freeze extensions to OpenStruct
+    module OpenStruct
+      def deep_freeze
+        return if deep_frozen?
+
+        @table.reduce({}) do |h, (key, value)|
+          fkey = key.respond_to?(:deep_freeze) ? key.deep_freeze : key
+          fval = value.respond_to?(:deep_freeze) ? value.deep_freeze : value
+          h.merge(fkey => fval)
+        end.freeze
+        @deep_frozen = true
+      end
+
+      def deep_frozen?
+        !!@deep_frozen
+      end
+    end
+  end
+
+  # Refines the Hash class with some convenience methods.
+  # Must call `using CemAcpt::CoreExtensions::HashExtensions`
+  # before these methods will be available.
+  module ExtendedHash
+    refine Hash do
+      # Formats a hash by converting all keys to symbols.
+      # If any value is a hash, it will be recursively
+      # extend and formatted.
+      def format!
+        transform_keys!(&:to_sym)
+        transform_values! do |value|
+          if value.is_a?(Hash)
+            value.format!
+          else
+            value
+          end
+        end
+      end
+
+      def has?(path)
+        !!dot_dig(path)
+      end
+
+      # Digs into a Hash using a dot-separated path.
+      # If the path is not found, returns nil.
+      # Example:
+      #   hash = {a: {b: {c: 1}}}
+      #   hash.dot_dig('a.b.c') # => 1
+      def dot_dig(path)
+        result = dig(*path.split('.').map(&:to_sym))
+        return result unless result.nil?
+
+        dig(*path.split('.'))
+      end
+
+      # Stores a value in a nested Hash using a dot-separated path
+      # to dig through keys.
+      # Example:
+      #   hash = {a: {b: {c: 1}}}
+      #   hash.dot_store('a.b.c', 2)
+      #   hash #=> {a: {b: {c: 2}}}
+      def dot_store(path, value)
+        *key, last = path.split('.').map(&:to_sym)
+        key.inject(self, :fetch)[last] = value
+      end
+    end
+  end
+end

data/lib/cem_acpt/image_name_builder.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  require_relative 'core_extensions'
+  require_relative 'logging'
+
+  # Dynamically builds an image name based on parameters specified in the
+  # config. The config is expected to have a key 'image_name_builder' with
+  # the following options:
+  #   - 'parts'                   - (Required) An array of strings to be joined together to form the image name.
+  #                                   If the strings begin with a '$', they will be replaced with the corresponding
+  #                                   value from current test data. To specify nested keys, use '.' to separate
+  #                                   the keys. Example: '$name.pattern.framework' will be replaced with the
+  #                                   value of the name pattern key framework in the current test data.
+  #   - 'join_with'               - (Optional) The string to join the parts with. Defaults to ''.
+  #   - 'character_substitutions' - (Optional) An array of of 2-item arrays. The first item
+  #                                   is the character to replace, the second is the replacement.
+  #                                   Example: [[' ', '-'], ['_', '-']] will replace all spaces
+  #                                   and underscores with dashes.
+  #   - 'validation_pattern'      - (Optional) A regex pattern to validate the image name against.
+  class ImageNameBuilder
+    include CemAcpt::Logging
+    using CemAcpt::CoreExtensions::ExtendedHash
+
+    # Initializes the ImageNameBuilder.
+    # @param config [CemAcpt::Config] The config to use.
+    def initialize(config)
+      unless config.has?('image_name_builder')
+        raise ArgumentError, 'Configuration does not have an image_name_builder key'
+      end
+
+      @config = config.get('image_name_builder')
+    end
+
+    # Builds an image name based on the config. It does so in three steps:
+    #   1. Resolve variables in the parts array.
+    #   2. Join the parts together with the join_with string if specified
+    #      and validate the image name against the validation_pattern if
+    #      specified.
+    #   3. Perform any specified character substitutions on the image name.
+    # @param test_data [Hash] The test data to use to build the image name.
+    # @return [String] The image name.
+    def build(test_data)
+      logger.debug 'Building image name...'
+      logger.debug "Using config: #{@config.to_h}"
+      logger.debug "Test data: #{test_data}"
+      parts = resolve_parts(test_data)
+      logger.debug "Resolved parts: #{parts}"
+      image_name = create_image_name(parts)
+      logger.debug "Created image name: #{image_name}"
+      final_image_name = character_substitutions(image_name)
+      logger.debug "Final image name: #{final_image_name}"
+      final_image_name
+    end
+
+    private
+
+    # Resolves variables in the parts array by replacing them with the
+    # corresponding value from the test data.
+    # @param test_data [Hash] The test data to use to build the image name.
+    # @return [Array] The parts array with variables resolved.
+    def resolve_parts(test_data)
+      @config[:parts].each_with_object([]) do |part, ary|
+        logger.debug "Resolving part: #{part}"
+        if part.start_with?('$')
+          var_path = part[1..-1]
+          logger.debug "Resolving variable path: #{var_path}"
+          ary << test_data.dot_dig(var_path)
+        else
+          ary << part
+        end
+      end
+    end
+
+    # Creates an image name based on the parts array.
+    # @param parts [Array] The parts array to use to build the image name.
+    # @return [String] The image name.
+    def create_image_name(parts)
+      image_name = @config[:join_with] ? parts.join(@config[:join_with]) : parts.join
+      logger.debug("Final image name: #{image_name}")
+
+      if @config[:validation_pattern]
+        logger.debug "Validating image name: #{image_name}..."
+        return image_name if image_name.match?(@config[:validation_pattern])
+
+        raise "Image name #{image_name} does not match validation pattern #{@config[:validation_pattern]}"
+      end
+      image_name
+    end
+
+    # Performs character substitutions on the image name.
+    # @param name [String] The image name to perform substitutions on.
+    # @return [String] The image name with substitutions performed.
+    def character_substitutions(name)
+      return name unless @config[:character_substitutions]
+
+      subbed_name = name
+      @config[:character_substitutions].each do |char_sub|
+        subbed_name.gsub!(char_sub[0], char_sub[1])
+      end
+      subbed_name
+    end
+  end
+end

data/lib/cem_acpt/logging.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module CemAcpt
+  # Logging for CemAcpt
+  module Logging
+    class << self
+      attr_writer :logger
+
+      # Exposes a logger instance. Will either use the currently set logger or
+      # create a new one.
+      # @return [Logger]
+      def logger
+        @logger ||= Logger.new(
+          current_log_config[:logdev],
+          current_log_config[:shift_age],
+          current_log_config[:shift_size],
+          **current_log_config.reject { |k, _| %i[logdev shift_age shift_size].include?(k) }
+        )
+      end
+
+      # Shortcut method for logger.level
+      # @return [Logger::Severity]
+      def current_log_level
+        logger.level
+      end
+
+      # Shortcut method to set logger.level
+      # @param level [String] the log level to set
+      def new_log_level(level)
+        case level.downcase
+        when 'debug'
+          @logger.level = Logger::DEBUG
+        when 'info'
+          @logger.level = Logger::INFO
+        when 'warn'
+          @logger.level = Logger::WARN
+        when 'error'
+          @logger.level = Logger::ERROR
+        when 'fatal'
+          @logger.level = Logger::FATAL
+        when 'unknown'
+          @logger.level = Logger::UNKNOWN
+        else
+          raise "Cannot set log level #{level}: invalid level"
+        end
+      end
+
+      # Shows the current log format style if set, or the default if not.
+      # @return [Symbol] the current log format style
+      def current_log_format
+        @current_log_format ||= :text
+      end
+
+      # Sets the current log format style and returns a proc to be passed to
+      # Logger#formatter=
+      # @param f [Symbol] the log format style to set
+      # @return [Proc] the proc to be passed to Logger#formatter=
+      def new_log_formatter(f)
+        case f.downcase.to_sym
+        when :json
+          require 'json'
+          @current_log_format = :json
+          proc do |severity, datetime, progname, msg|
+            {
+              timestamp: datetime.iso8601(3),
+              progname: progname,
+              severity: severity,
+              message: msg,
+            }.to_json + "\n"
+          end
+        when :text
+          @current_log_format = :text
+          proc do |severity, _datetime, _progname, msg|
+            "#{severity} - #{msg}\n"
+          end
+        else
+          @current_log_format = :file
+          proc do |severity, datetime, _progname, msg|
+            "[#{datetime.iso8601(3)}] #{severity}: #{msg}\n"
+          end
+        end
+      end
+
+      # Returns the current log config if set, or the default if not.
+      # @return [Hash] the current log config
+      def current_log_config
+        return @log_config if @log_config
+
+        formatter = new_log_formatter(current_log_format)
+        @log_config = {
+          logdev: $stdout,
+          shift_age: 'o',
+          shift_size: 1_048_576,
+          level: Logger::INFO,
+          progname: 'CemAcpt',
+          datetime_format: '%Y%m%dT%H%M%S%z',
+          formatter: formatter,
+        }
+        @log_config
+      end
+
+      # Creates a new log config hash and a new Logger instance using that config and sets
+      # the new Logger instance as the current logger. NO DEFAULT VALUES ARE SET.
+      # @param logdev [String, IO] the log device to use. If 'stdout' or 'stderr' are passed,
+      #   the respective IO object will be used. Otherwise, Strings will be treated as file paths.
+      #   If an IO object is passed, it will be used directly.
+      #   If no logdev is passed, or an invalid logdev is passed, the default is $stdout.
+      # @param shift_age [String] the log rotation age
+      # @param shift_size [Integer] the log rotation size
+      # @param level [Logger::Severity] the log level
+      # @param formatter [Proc] the log formatter
+      # @param datetime_format [String] the datetime format
+      def new_log_config(logdev: nil, shift_age: nil, shift_size: nil, level: nil, formatter: nil, datetime_format: nil)
+        @log_config[:shift_age] = shift_age if shift_age
+        @log_config[:shift_size] = shift_size if shift_size
+        @log_config[:level] = level if level
+        @log_config[:formatter] = formatter if formatter
+        @log_config[:datetime_format] = datetime_format if datetime_format
+        case logdev
+        when 'stdout'
+          @log_config[:logdev] = $stdout
+        when 'stderr'
+          @log_config[:logdev] = $stderr
+        when String
+          @log_config[:logdev] = target
+        when IO
+          @log_config[:logdev] = logdev
+        else
+          @log_config[:logdev] = $stdout
+          logger.warn("Unknown log target: #{logdev.inspect}, using STDOUT")
+        end
+        @logger = Logger.new(
+          @log_config[:logdev],
+          @log_config[:shift_age],
+          @log_config[:shift_size],
+          **@log_config.reject { |k, _| %i[logdev shift_age shift_size].include?(k) },
+        )
+      end
+    end
+
+    # Provides class method wrappers for logging methods
+    def self.included(base)
+      class << base
+        def logger
+          CemAcpt::Logging.logger
+        end
+
+        def current_log_level
+          CemAcpt::Logging.current_log_level
+        end
+
+        def new_log_level(level)
+          CemAcpt::Logging.new_log_level(level)
+        end
+
+        def current_log_config
+          CemAcpt::Logging.current_log_config
+        end
+
+        def new_log_config(logdev: nil, shift_age: nil, shift_size: nil, level: nil, formatter: nil, datetime_format: nil)
+          CemAcpt::Logging.new_log_config(logdev: logdev, shift_age: shift_age, shift_size: shift_size, level: level, formatter: formatter, datetime_format: datetime_format)
+        end
+      end
+    end
+
+    # Exposes the logger instance
+    def logger
+      CemAcpt::Logging.logger
+    end
+
+    # Exposes the current log level
+    def current_log_level
+      CemAcpt::Logging.current_log_level
+    end
+
+    # Exposes setting the log level
+    def new_log_level(level)
+      CemAcpt::Logging.new_log_level(level)
+    end
+
+    # Exposes the current log config
+    def current_log_config
+      CemAcpt::Logging.current_log_config
+    end
+
+    # Exposes setting a new log config
+    def new_log_config(logdev: nil, shift_age: nil, shift_size: nil, level: nil, formatter: nil, datetime_format: nil)
+      CemAcpt::Logging.new_log_config(logdev: logdev, shift_age: shift_age, shift_size: shift_size, level: level, formatter: formatter, datetime_format: datetime_format)
+    end
+  end
+end

data/lib/cem_acpt/platform/base/cmd.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module CemAcpt::Platform
+  require_relative File.join(__dir__, '..', '..', 'logging.rb')
+
+  # Base class for command providers. Provides an API for subclasses to implement.
+  class CmdBase
+    include CemAcpt::Logging
+
+    def initialize(*args, **kwargs); end
+
+    def local_exec(*_args, **_kwargs)
+      raise NotImplementedError, '#local_exec must be implemented by a subclass'
+    end
+
+    def ssh(_instance_name, _command, _ignore_command_errors: false, _opts: {})
+      raise NotImplementedError, '#ssh must be implemented by a subclass'
+    end
+
+    def scp_upload(_instance_name, _local, _remote, _scp_opts: {}, _opts: {})
+      raise NotImplementedError, '#scp_upload must be implemented by a subclass'
+    end
+
+    def scp_download(_instance_name, _local, _remote, _scp_opts: {}, _opts: {})
+      raise NotImplementedError, '#scp_download must be implemented by a subclass'
+    end
+
+    def ssh_ready?(_instance_name, _timeout = 300, _opts: {})
+      raise NotImplementedError, '#ssh_ready? must be implemented by a subclass'
+    end
+
+    def apply_manifest(_instance_name, _manifest, _opts: {})
+      raise NotImplementedError, '#create_manifest_on_node must be implemented by a subclass'
+    end
+
+    def run_shell(_instance_name, _command, _opts: {})
+      raise NotImplementedError, '#run_shell must be implemented by a subclass'
+    end
+
+    def trim_output(output)
+      if output.include?("\n")
+        output.split("\n").map(&:strip).reject(&:empty?).join("\n")
+      else
+        output.strip
+      end
+    end
+
+    def with_timed_retry(timeout = 300)
+      return unless block_given?
+
+      last_error = nil
+      start_time = Time.now
+      while Time.now - start_time < timeout
+        begin
+          output = yield
+          return output
+        rescue StandardError => e
+          last_error = e
+          sleep(5)
+        end
+      end
+      raise last_error
+    end
+  end
+end