cem_acpt 0.2.5 → 0.6.1

data/lib/cem_acpt/config.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+require_relative 'core_extensions'
+
+module CemAcpt
+  using CemAcpt::CoreExtensions::ExtendedHash
+
+  # Holds the configuration for cem_acpt
+  class Config
+    KEYS = %i[
+      actions
+      ci_mode
+      config_file
+      image_name_builder
+      log_level
+      log_file
+      log_format
+      module_dir
+      node_data
+      no_ephemeral_ssh_key
+      platform
+      provisioner
+      quiet
+      terraform
+      test_data
+      tests
+      user_config
+      verbose
+    ].freeze
+
+    attr_reader :config, :env_vars
+
+    def initialize(opts: {}, config_file: nil, load_user_config: true)
+      @load_user_config = load_user_config
+      load(opts: opts, config_file: config_file)
+    end
+
+    # The default configuration
+    def defaults
+      {
+        actions: {},
+        ci_mode: false,
+        config_file: nil,
+        image_name_builder: {
+          character_substitutions: ['_', '-'],
+          parts: ['cem-acpt', '$image_fam', '$collection', '$firewall'],
+          join_with: '-',
+        },
+        log_level: 'info',
+        log_file: nil,
+        log_format: 'text',
+        module_dir: Dir.pwd,
+        node_data: {},
+        no_ephemeral_ssh_key: false,
+        platform: {
+          name: 'gcp',
+        },
+        quiet: false,
+        test_data: {
+          for_each: {
+            collection: %w[puppet7],
+          },
+          vars: {},
+          name_pattern_vars: %r{^(?<framework>[a-z]+)_(?<image_fam>[a-z0-9-]+)_(?<firewall>[a-z]+)_(?<framework_vars>[-_a-z0-9]+)$},
+          vars_post_processing: {
+            new_vars: [
+              {
+                name: 'profile',
+                string_split: {
+                  from: 'framework_vars',
+                  using: '_',
+                  part: 0,
+                },
+              },
+              {
+                name: 'level',
+                string_split: {
+                  from: 'framework_vars',
+                  using: '_',
+                  part: 1,
+                },
+              },
+            ],
+            delete_vars: %w[framework_vars],
+          },
+        },
+        tests: [],
+        verbose: false,
+      }
+    end
+
+    # Load the configuration from the environment variables, config file, and opts
+    # The order of precedence is:
+    #   1. environment variables
+    #   2. user config file (config.yaml in user_config_dir)
+    #   3. specified config file (if it exists)
+    #   4. opts
+    #   5. static options (set in this class)
+    # @param opts [Hash] The options to load
+    # @param config_file [String] The config file to load
+    # @return [self] This object with the config loaded
+    def load(opts: {}, config_file: nil)
+      create_config_dirs!
+      init_config!(opts: opts, config_file: config_file)
+      add_env_vars!(@config)
+      @config.merge!(user_config) if user_config && @load_user_config
+      @config.merge!(config_from_file) if config_from_file
+      @config.merge!(@options) if @options
+      add_static_options!(@config)
+      @config.format! # Symbolize keys of all hashes
+      validate_config!
+      # Freeze the config so it can't be modified
+      # This helps with thread safety and deterministic behavior
+      @config.freeze
+      self
+    end
+    alias to_h config
+
+    def explain
+      explanation = {}
+      %i[defaults env_vars user_config config_from_file options].each do |source|
+        source_vals = send(source).dup
+        next if source_vals.nil? || source_vals.empty?
+
+        # The loop below will overwrite the value of explanation[key] if the same key is found in multiple sources
+        # This is intentional, as the last source to set the value is the one that should be used
+        source_vals.each do |key, value|
+          explanation[key] = source if @config.dget(key.to_s) == value
+        end
+      end
+      explained = explanation.each_with_object([]) do |(key, value), ary|
+        ary << "Key '#{key}' from source '#{value}'"
+      end
+      explained.join("\n")
+    end
+
+    def [](key)
+      if key.is_a?(Symbol)
+        @config[key].dup
+      elsif key.is_a?(String)
+        @config.dget(key).dup
+      else
+        raise ArgumentError, "Invalid key type '#{key.class}'"
+      end
+    end
+
+    def get(dot_key)
+      @config.dget(dot_key).dup
+    end
+    alias dget get
+
+    def has?(dot_key)
+      !!get(dot_key)
+    end
+
+    def empty?
+      @config.empty?
+    end
+
+    def ci_mode?
+      !!get('ci_mode') || !!(ENV['GITHUB_ACTIONS'] || ENV['CI'])
+    end
+    alias ci? ci_mode?
+
+    def debug_mode?
+      get('log_level') == 'debug'
+    end
+    alias debug? debug_mode?
+
+    def verbose_mode?
+      !!get('verbose')
+    end
+    alias verbose? verbose_mode?
+
+    def quiet_mode?
+      !!get('quiet')
+    end
+    alias quiet? quiet_mode?
+
+    def to_yaml
+      @config.to_yaml
+    end
+
+    def to_json(*args)
+      @config.to_json(*args)
+    end
+
+    private
+
+    attr_reader :options
+
+    def user_config_dir
+      @user_config_dir ||= File.join(Dir.home, '.cem_acpt')
+    end
+
+    def user_config_file
+      @user_config_file ||= File.join(user_config_dir, 'config.yaml')
+    end
+
+    def terraform_dir
+      @terraform_dir ||= File.join(user_config_dir, 'terraform')
+    end
+
+    def valid_env_var?(env_var)
+      env_var.start_with?('CEM_ACPT_') && ENV[env_var]
+    end
+
+    def env_var_to_dot_key(env_var)
+      env_var.sub('CEM_ACPT_', '').gsub(%r{__}, '.').downcase
+    end
+
+    def add_static_options!(config)
+      config.dset('user_config.dir', user_config_dir)
+      config.dset('user_config.file', user_config_file)
+      config.dset('provisioner', 'terraform')
+      config.dset('terraform.dir', terraform_dir)
+      set_third_party_env_vars!(config)
+    end
+
+    # Certain environment variables are used by other tools like GitHub Actions
+    # This method sets the relative config values for those environment variables.
+    # This is the last step in composing the config, so it will override any
+    # values set by the user.
+    def set_third_party_env_vars!(config)
+      if ENV['RUNNER_DEBUG'] == '1'
+        config.dset('log_level', 'debug')
+        config.dset('verbose', true)
+      end
+      if ENV['GITHUB_ACTIONS'] == 'true' || ENV['CI'] == 'true'
+        config.dset('ci_mode', true)
+      end
+    end
+
+    # Used to source the config during loading of config files.
+    # Because config may not be fully loaded yet, this method
+    # checks the options hash first, then the current config,
+    # then the environment variables for the given key
+    # @param key [String] The key to find in dot notation
+    # @return [Any] The value of the key
+    def find_option(key)
+      return @options.dget(key) if @options.dget(key)
+      return @config.dget(key) if @config.dget(key)
+      ENV.each do |k, v|
+        next unless valid_env_var?(k)
+
+        return v if env_var_to_dot_key(k) == key
+      end
+      nil
+    end
+
+    def init_config!(opts: {}, config_file: nil)
+      # Blank out the config
+      @user_config = {}
+      @config_from_file = {}
+      @options = {}
+      @config = defaults.dup
+      # Set the parameterized defaults
+      config_file = ENV['CEM_ACPT_CONFIG_FILE'] if config_file.nil?
+      @config.dset('config_file', config_file) if config_file
+      @options = opts || {}
+    end
+
+    def add_env_vars!(config)
+      @env_vars = {}
+      # First load known environment variables into their respective config keys
+      # Then load any environment variables that start with CEM_ACPT_<known key> into their respective config keys
+      ENV.each do |env_var, value|
+        next unless valid_env_var?(env_var)
+
+        key = env_var_to_dot_key(env_var) # Convert CEM_ACPT_<key> to <dotkey>
+        next unless KEYS.include?(key.split('.').first.to_sym) # Skip if the key is not a known config key
+        @env_vars[key] = value
+        config.dset(key, value)
+      end
+    end
+
+    def user_config
+      return @user_config unless @user_config.nil? || @user_config.empty?
+
+      @user_config = if user_config_file && File.exist?(user_config_file)
+                       load_config_file(user_config_file)
+                     else
+                       {}
+                     end
+
+      @user_config
+    end
+
+    def config_from_file
+      return @config_from_file unless @config_from_file.nil? || @config_from_file.empty?
+
+      conf_file = find_option('config_file')
+      return {} if conf_file.nil? || conf_file.empty?
+
+      unless conf_file
+        warn "Invalid config_file type '#{conf_file.class}'. Must be a String."
+        return {}
+      end
+
+      mod_dir = find_option('module_dir')
+      if mod_dir && File.exist?(File.join(mod_dir, conf_file))
+        @config_from_file = load_config_file(File.join(mod_dir, conf_file))
+      elsif File.exist?(File.expand_path(conf_file))
+        @config_from_file = load_config_file(File.expand_path(conf_file))
+      else
+        err_msg = [
+          "Config file '#{File.expand_path(conf_file)}' does not exist.",
+        ]
+        err_msg << "Config file '#{File.join(mod_dir, conf_file)}' does not exist." if mod_dir
+        raise err_msg.join("\n")
+      end
+
+      @config_from_file
+    end
+
+    def load_config_file(config_file)
+      return {} if config_file.nil? || config_file.empty? || !File.exist?(File.expand_path(config_file))
+
+      loaded = load_yaml(config_file)
+      loaded.format!
+      loaded
+    end
+
+    def load_yaml(config_file)
+      if YAML.respond_to?(:safe_load_file) # Ruby 3.0+
+        YAML.safe_load_file(File.expand_path(config_file), permitted_classes: [Regexp])
+      else
+        YAML.safe_load(File.read(File.expand_path(config_file)), permitted_classes: [Regexp])
+      end
+    end
+
+    def validate_config!
+      @config.each do |key, _value|
+        warn "Unknown config key: #{key}" unless KEYS.include?(key)
+      end
+    end
+
+    def create_config_dirs!
+      FileUtils.mkdir_p(user_config_dir) unless Dir.exist?(user_config_dir)
+      FileUtils.cp_r(File.expand_path(File.join(__dir__, '..', 'terraform')), user_config_dir)
+    end
+  end
+end

data/lib/cem_acpt/core_extensions.rb

@@ -1,66 +1,7 @@
 # frozen_string_literal: true
 
-# This module holds extensions to Ruby and the Ruby stdlib
-# Extensions related to deep_freeze were pulled from: https://gist.github.com/steakknife/1a37057b3b8539f4aca3
+# This module holds extensions and refinements to Ruby and the Ruby stdlib
 module CemAcpt::CoreExtensions
-  # DeepFreeze recursively freezes all keys and values in a hash
-  # Currently unused, but was used at one point and may be useful again
-  module DeepFreeze
-    # Holds deep_freeze extensions to Kernel
-    module Kernel
-      alias deep_freeze freeze
-      alias deep_frozen? frozen?
-    end
-
-    # Holds deep_freeze extensions to Enumerable
-    module Enumerable
-      def deep_freeze
-        unless @deep_frozen
-          each(&:deep_freeze)
-          @deep_frozen = true
-        end
-        freeze
-      end
-
-      def deep_frozen?
-        !!@deep_frozen
-      end
-    end
-
-    # Holds deep_freeze extensions to Hash
-    module Hash
-      def deep_freeze
-        transform_values! do |value|
-          value.respond_to?(:deep_freeze) ? value.deep_freeze : value.freeze
-        end
-        freeze
-        @deep_frozen = true
-      end
-
-      def deep_frozen?
-        !!@deep_frozen
-      end
-    end
-
-    # Holds deep_freeze extensions to OpenStruct
-    module OpenStruct
-      def deep_freeze
-        unless deep_frozen?
-          @table.reduce({}) do |h, (key, value)|
-            fkey = key.respond_to?(:deep_freeze) ? key.deep_freeze : key
-            fval = value.respond_to?(:deep_freeze) ? value.deep_freeze : value
-            h.merge(fkey => fval)
-          end.freeze
-          @deep_frozen = true
-        end
-      end
-
-      def deep_frozen?
-        !!@deep_frozen
-      end
-    end
-  end
-
   # Refines the Hash class with some convenience methods.
   # Must call `using CemAcpt::CoreExtensions::HashExtensions`
   # before these methods will be available.
@@ -83,6 +24,7 @@ module CemAcpt::CoreExtensions
       def has?(path)
         !!dot_dig(path)
       end
+      alias dhas? has?
 
       # Digs into a Hash using a dot-separated path.
       # If the path is not found, returns nil.
@@ -91,7 +33,16 @@ module CemAcpt::CoreExtensions
       #   hash.dot_dig('a.b.c') # => 1
       def dot_dig(path)
         dig(*path.split('.').map(&:to_sym)) || dig(*path.split('.'))
+      rescue TypeError
+        # TypeError is raised if parts of the path don't have the #dig method
+        # This can happen if you do something like:
+        #   hash = {a: {b: {c: 1}}}
+        #   hash.dot_dig('a.b.c.d')
+        # The integer 1 doesn't have the #dig method, so we get a TypeError
+        # Since this means the path is invalid, we return nil
+        nil
       end
+      alias dget dot_dig
 
       # Stores a value in a nested Hash using a dot-separated path
       # to dig through keys.
@@ -99,10 +50,15 @@ module CemAcpt::CoreExtensions
       #   hash = {a: {b: {c: 1}}}
       #   hash.dot_store('a.b.c', 2)
       #   hash #=> {a: {b: {c: 2}}}
+      #   hash.dot_store('a.b.d', 3)
+      #   hash #=> {a: {b: {c: 2, d: 3}}}
       def dot_store(path, value)
         *key, last = path.split('.').map(&:to_sym)
-        key.inject(self, :fetch)[last] = value
+        key.inject(self) do |memo, k|
+          memo[k] ||= {}
+        end[last] = value
       end
+      alias dset dot_store
     end
   end
 end

data/lib/cem_acpt/goss/api/action_response.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  module Goss
+    module Api
+      class ActionResponse
+        attr_reader :host, :action, :body
+
+        def initialize(host, action, status, body)
+          @host = host
+          @action = action
+          @status = status
+          @body = body
+        end
+
+        def to_s
+          "#<#{self.class.name}:0x#{object_id.to_s(16)}>"
+        end
+
+        def inspect
+          to_s
+        end
+
+        def to_h
+          {
+            host: host,
+            action: action,
+            status: @status,
+            body: @body,
+          }
+        end
+
+        def status
+          @status.to_i
+        end
+        alias http_status status
+
+        def success?
+          status == 200
+        end
+
+        def results
+          @results ||= @body['results'].map { |r| ActionResponseResult.new(r) }
+        end
+
+        def results?
+          !results.nil? && !results.empty?
+        end
+
+        def summary
+          @summary ||= ActionResponseSummary.new(@body['summary'])
+        end
+
+        def summary?
+          !summary.nil? && !summary.empty?
+        end
+      end
+
+      module DurationHandler
+        DURATION_UNITS = %i[nanoseconds milliseconds seconds].freeze
+
+        Duration = Struct.new(:duration, :unit, :round) do
+          def to_f
+            return @to_f if defined?(@to_f)
+            return 0.0 if duration.nil?
+
+            case unit
+            when :nanoseconds
+              @to_f = duration.to_f.round(round)
+            when :milliseconds
+              @to_f = (duration.to_f / 1_000_000).round(round)
+            when :seconds
+              @to_f = (duration.to_f / 1_000_000_000).round(round)
+            else
+              raise ArgumentError, "Invalid unit #{unit}, must be one of #{DURATION_UNITS}"
+            end
+            @to_f
+          end
+
+          def to_s
+            return @to_s if defined?(@to_s)
+
+            case unit
+            when :nanoseconds
+              @to_s = "#{to_f}ns"
+            when :milliseconds
+              @to_s = "#{to_f}ms"
+            when :seconds
+              @to_s = "#{to_f}s"
+            else
+              raise ArgumentError, "Invalid unit #{unit}, must be one of #{DURATION_UNITS}"
+            end
+            @to_s
+          end
+        end
+
+        # @param unit [Symbol] The unit to return the duration in
+        # @param round [Integer] The number of decimal places to round to
+        # @return [Duration] The Duration object
+        def duration(unit: :seconds, round: 3)
+          @all_durations ||= {}
+          @all_durations[unit] ||= {}
+          @all_durations[unit][round] ||= Duration.new(@duration, unit, round)
+        end
+      end
+
+      class ActionResponseSummary
+        include DurationHandler
+
+        attr_reader :failed_count, :summary_line, :test_count
+
+        def initialize(summary)
+          @summary = summary
+          @duration = @summary['total-duration']
+          @failed_count = @summary['failed-count']
+          @summary_line = @summary['summary-line']
+          @test_count = @summary['test-count']
+        end
+        alias to_s summary_line
+        alias total_duration duration
+
+        def to_h
+          @summary
+        end
+
+        def failed_percentage
+          @failed_percentage ||= (test_count.zero? ? 0.00 : (failed_count.to_f / test_count.to_f) * 100).round(2)
+        end
+
+        def passed_count
+          @passed_count ||= test_count - failed_count
+        end
+      end
+
+      class ActionResponseResult
+        include DurationHandler
+
+        attr_reader(:duration, :err, :expected, :found, :human, :meta, :property,
+                    :resource_id, :resource_type, :result, :skipped, :successful,
+                    :summary_line, :test_type, :title)
+
+        def initialize(raw_result)
+          @raw_result = raw_result
+          @duration = @raw_result['duration']
+          @err = @raw_result['err']
+          @expected = @raw_result['expected']
+          @found = @raw_result['found']
+          @human = @raw_result['human']
+          @meta = @raw_result['meta']
+          @property = @raw_result['property']
+          @resource_id = @raw_result['resource-id']
+          @resource_type = @raw_result['resource-type']
+          @result = @raw_result['result']
+          @skipped = @raw_result['skipped']
+          @successful = @raw_result['successful']
+          @summary_line = @raw_result['summary-line']
+          @test_type = @raw_result['test-type']
+          @title = @raw_result['title']
+        end
+        alias error err
+        alias to_s summary_line
+        alias skipped? skipped
+        alias success? successful
+
+        def to_h
+          @raw_result
+        end
+
+        def error?
+          !err.nil?
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/goss/api.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'async/barrier'
+require 'async/http/internet'
+require 'json'
+require_relative 'api/action_response'
+
+module CemAcpt
+  module Goss
+    # Holds methods for interacting with the Goss API running on a test node.
+    module Api
+      class << self
+        # The actions that can be run against the Goss API. The key is the action
+        # name and the value is the port/endpoint of the action.
+        ACTIONS = {
+          acpt: '8080/acpt',
+          idempotent: '8081/idempotent',
+          noop: '8082/noop',
+        }.freeze
+
+        # Create a URI for the specified action against the specified host.
+        # @param host [String] The host to run the action against. This should be
+        #   a public IP address or a DNS-resolvable name.
+        # @param action [Symbol] The action to run.
+        # @return [URI] The URI for the action.
+        def action_uri(host, action)
+          URI("http://#{host}:#{ACTIONS[action.to_sym]}")
+        end
+
+        # Run the specified actions against the specified hosts asynchronously.
+        # @param hosts [Array<String>] The hosts to run the actions against. Each
+        #   host should be a public IP address or a DNS-resolvable name.
+        # @param results [Queue] The queue to push the results to.
+        # @param only [Array<Symbol>] The actions to run.
+        # @param except [Array<Symbol>] The actions to skip.
+        # @return [Queue] The queue of results.
+        def run_actions_async(hosts, results: Queue.new, only: [], except: [])
+          raise ArgumentError, 'hosts must be an Array' unless hosts.is_a?(Array)
+          raise ArgumentError, 'results must be a Queue-like object implementing #<<' unless results.respond_to?(:<<)
+          raise ArgumentError, 'only must be an Array' unless except.is_a?(Array)
+          raise ArgumentError, 'except must be an Array' unless except.is_a?(Array)
+          only.map!(&:to_sym)
+          except.map!(&:to_sym)
+          only_specified = !only.empty?
+          except_specified = !except.empty?
+          Async do
+            internet = Async::HTTP::Internet.new
+            barrier = Async::Barrier.new
+            barrier.async do
+              hosts.each do |host|
+                ACTIONS.keys.each do |action, _|
+                  next if only_specified && !only.include?(action)
+                  next if except_specified && except.include?(action)
+
+                  results << run_action(internet, host, action)
+                end
+              end
+            end
+            barrier.wait
+          ensure
+            internet&.close
+            results.close if results.respond_to?(:close)
+          end
+          results
+        end
+
+        # Run the specified action against the specified host.
+        # @param internet [Async::HTTP::Internet] The Async::HTTP::Internet object to use for the request.
+        # @param host [String] The host to run the action against. This should be
+        #   a public IP address or a DNS-resolvable name.
+        # @param action [Symbol] The action to run.
+        # @return [ActionResponse] The response from the action.
+        def run_action(internet, host, action)
+          uri = action_uri(host, action)
+          response = internet.get(uri.to_s)
+          body = JSON.parse(response.read)
+          ActionResponse.new(host, action, response.status, body)
+        end
+      end
+    end
+  end
+end