cem_acpt 0.7.3 → 0.8.0

data/lib/cem_acpt/provision/terraform/terraform_cmd.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'stringio'
+require_relative '../../logging'
+
+module CemAcpt
+  module Provision
+    # Stand-in for ruby-terraform because ruby-terraform doesn't work with Ruby 3
+    class TerraformCmd
+      include CemAcpt::Logging
+
+      attr_accessor :working_dir
+      attr_reader :bin_path
+
+      def initialize(working_dir = nil, environment = {})
+        @working_dir = working_dir
+        @environment = environment
+        @bin_path = which_terraform
+      end
+
+      def inspect
+        to_s
+      end
+
+      def to_s
+        "#<#{self.class.name}:0x#{object_id.to_s(16)}>"
+      end
+
+      def environment=(env)
+        raise ArgumentError, 'environment must be a Hash' unless env.is_a?(Hash)
+
+        @environment = env
+      end
+
+      def environment(env = {})
+        raise ArgumentError, 'additional environment must be a Hash' unless env.is_a?(Hash)
+
+        if env.key?(:environment) && env[:environment].is_a?(Hash)
+          env = env[:environment]
+        end
+        @environment.merge(env)
+      end
+
+      def init(opts = {}, env = {})
+        run_cmd('init', opts, env)
+      end
+
+      def plan(opts = {}, env = {})
+        plan = extract_arg!(opts, :plan, required: true)
+        opts[:out] = plan
+        run_cmd('plan', opts, env)
+      end
+
+      def apply(opts = {}, env = {})
+        plan = extract_arg!(opts, :plan, required: true)
+        run_cmd('apply', opts, env, suffix: plan)
+      end
+
+      def destroy(opts = {}, env = {})
+        run_cmd('destroy', opts, env)
+      end
+
+      def show(opts = {}, env = {})
+        run_cmd('show', opts, env)
+      end
+
+      def output(opts = {}, env = {})
+        name = extract_arg!(opts, :name, required: true)
+        run_cmd('output', opts, env, suffix: name)
+      end
+
+      private
+
+      def extract_arg!(opts, key, required: false)
+        key = key.to_sym
+        raise ArgumentError, "option #{key} is required" if required && !opts.key?(key) && (opts[key].nil? || opts[key].empty?)
+
+        opts.delete(key)
+      end
+
+      def run_cmd(cmd, opts = {}, env = {}, suffix: '')
+        logger.debug('CemAcpt::Provision::TerraformCmd') { "Running command with args: cmd = \"#{cmd}\", opts = \"#{opts}\", env = \"#{env}\", suffix = \"#{suffix}\"" }
+        val, outerr = execute(format_cmd(cmd, opts, suffix), environment(env))
+        raise "Error running command: #{cmd}\n#{outerr}" unless val.success?
+
+        outerr
+      end
+
+      def execute(cmd, env)
+        logger.debug('CemAcpt::Provision::TerraformCmd') { "Executing command: #{cmd}" }
+        io_outerr = StringIO.new
+        val = Open3.popen2e(env, cmd) do |stdin, outerr, wait_thr|
+          stdin.close
+          outerr.sync = true
+          output_thread = Thread.new do
+            while (line = outerr.readline_nonblock)
+              logger << line
+              io_outerr.write(line) unless line.chomp.empty?
+            end
+          rescue IO::WaitReadable
+            retry
+          rescue EOFError
+            # Do nothing
+          end
+          wait_thr.join
+          output_thread.exit
+          wait_thr.value
+        end
+        [val, io_outerr.string]
+      end
+
+      def chdir(opts = {})
+        [extract_arg!(opts, :chdir), working_dir, Dir.pwd].each do |d|
+          next if d.nil? || d.empty?
+
+          d = File.expand_path(d)
+          return "-chdir=#{d}" if File.directory?(d)
+
+          logger.warn('CemAcpt::Provision::TerraformCmd') { "Directory #{d} does not exist, using next..." }
+        end
+      end
+
+      def which_terraform
+        exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+        ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+          exts.each do |ext|
+            exe = File.join(path, "terraform#{ext}")
+            return exe if File.executable?(exe) && !File.directory?(exe)
+          end
+        end
+        raise 'terraform not found in PATH, make sure it is installed'
+      end
+
+      def format_cmd(cmd, opts = {}, suffix = '')
+        formatted = [bin_path, chdir(opts), cmd]
+        opts_s = opts_string(opts)
+        formatted << opts_s unless opts_s.empty?
+        formatted << suffix unless suffix.empty?
+        formatted.join(' ')
+      end
+
+      def opts_string(opts = {})
+        formatted = opts.map do |k, v|
+          next if k.nil? || k.empty?
+
+          k = k.to_s
+          k.tr!('_', '-')
+          if k == 'vars'
+            v.map { |vk, vv| "-var '#{vk}=#{vv}'" }.join(' ')
+          elsif %w[input lock refresh].include?(k) # These are boolean flags with values
+            "-#{k}=#{v}"
+          elsif v.nil? || (v.respond_to?(:empty) && v.empty?) || v.is_a?(TrueClass)
+            "-#{k}"
+          else
+            case v
+            when Array, Hash
+              "-#{k}='#{v.to_json}'"
+            else
+              "-#{k}=#{v}"
+            end
+          end
+        end
+        formatted.join(' ')
+      end
+    end
+  end
+
+  # IO monkey patch for non-blocking readline
+  class ::IO
+    def readline_nonblock
+      rlnb = []
+      while (ch = read_nonblock(1))
+        rlnb << ch
+        break if ch == "\n"
+      end
+      rlnb.join
+    end
+  end
+end

data/lib/cem_acpt/provision/terraform/windows.rb

@@ -6,6 +6,7 @@ module CemAcpt
   module Provision
     # Class provides methods for gathering provision data for Windows nodes
     class Windows < OsData
+      # A name that will match with how image names are on GCP
       def self.valid_names
         %w[windows]
       end
@@ -17,6 +18,14 @@ module CemAcpt
       def puppet_bin_path
         'C:/Program Files/Puppet Labs/Puppet/bin/puppet.bat'
       end
+
+      def destination_provision_directory
+        'C:/cem_acpt'
+      end
+
+      def provision_commands
+        ['placeholder']
+      end
     end
   end
 end

data/lib/cem_acpt/provision/terraform.rb

@@ -2,10 +2,10 @@
 
 require 'fileutils'
 require 'json'
-require 'ruby-terraform'
 require_relative '../logging'
 require_relative 'terraform/linux'
 require_relative 'terraform/windows'
+require_relative 'terraform/terraform_cmd'
 
 module CemAcpt
   module Provision
@@ -27,13 +27,12 @@ module CemAcpt
         @public_key = nil
       end
 
+      # @return [Hash] A hash of instance names and IPs
       def provision(reuse_working_dir: false)
-        logger.info('Terraform') { 'Provisioning nodes...' }
+        logger.info('CemAcpt::Provision::Terraform') { 'Provisioning nodes...' }
         @working_dir = new_working_dir unless reuse_working_dir
         validate_working_dir!
         save_vars_to_file!(formatted_vars) # Easier to reuse nodes this way
-
-        terraform_configure_logging
         terraform_init
         terraform_plan(formatted_vars, DEFAULT_PLAN_NAME)
         terraform_apply(DEFAULT_PLAN_NAME)
@@ -42,7 +41,7 @@ module CemAcpt
 
       def destroy
         terraform_destroy(formatted_vars)
-        logger.verbose('Terraform') { "Deleting old working directory #{working_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Deleting old working directory #{working_dir}" }
         FileUtils.rm_rf(working_dir)
         @working_dir = nil
         @module_package_path = nil
@@ -57,25 +56,17 @@ module CemAcpt
       private
 
       def terraform
-        @terraform ||= RubyTerraform
-      end
-
-      def terraform_configure_logging
-        terraform.configure do |c|
-          c.logger = logger
-          c.stdout = c.logger
-          c.stderr = c.logger
-        end
+        @terraform ||= CemAcpt::Provision::TerraformCmd.new(working_dir, environment)
       end
 
       def terraform_init
-        logger.debug('Terraform') { 'Initializing Terraform' }
+        logger.info('CemAcpt::Provision::Terraform') { 'Initializing Terraform' }
         terraform.init({ chdir: working_dir, input: false, no_color: true }, { environment: environment })
       end
 
       def terraform_plan(vars, plan_name = DEFAULT_PLAN_NAME)
-        logger.debug('Terraform') { "Creating Terraform plan '#{plan_name}'" }
-        logger.verbose('Terraform') { "Using vars:\n#{JSON.pretty_generate(vars)}" }
+        logger.info('CemAcpt::Provision::Terraform') { "Creating Terraform plan '#{plan_name}'" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Using vars:\n#{JSON.pretty_generate(vars)}" }
         terraform.plan(
           {
             chdir: working_dir,
@@ -91,23 +82,24 @@ module CemAcpt
       end
 
       def terraform_apply(plan_name = DEFAULT_PLAN_NAME)
-        logger.debug('Terraform') { "Running Terraform apply with the plan #{plan_name}" }
+        logger.info('CemAcpt::Provision::Terraform') { "Running Terraform apply with the plan #{plan_name}" }
         terraform.apply({ chdir: working_dir, input: false, no_color: true, plan: plan_name }, { environment: environment })
       end
 
       def terraform_output(name, json: true)
-        logger.debug('Terraform') { "Getting Terraform output #{name}" }
+        logger.info('CemAcpt::Provision::Terraform') { "Getting Terraform output #{name}" }
         terraform.output({ chdir: working_dir, no_color: true, json: json, name: name }, { environment: environment })
       end
 
       def terraform_destroy(vars)
-        logger.debug('Terraform') { 'Destroying Terraform resources' }
+        logger.info('CemAcpt::Provision::Terraform') { 'Destroying Terraform resources...' }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Using vars: #{vars}" }
         terraform.destroy(
           {
             chdir: working_dir,
             auto_approve: true,
             input: false,
-            no_color: true,
+            no_color: nil,
             vars: vars,
           },
           {
@@ -117,18 +109,18 @@ module CemAcpt
       end
 
       def terraform_show
-        logger.debug('Terraform') { 'Showing Terraform state' }
+        logger.info('CemAcpt::Provision::Terraform') { 'Showing Terraform state' }
         terraform.show({ chdir: working_dir, no_color: true }, { environment: environment })
       end
 
       def new_backend(test_name)
         if CemAcpt::Provision::Linux.use_for?(test_name)
-          logger.info('Terraform') { 'Using Linux backend' }
-          logger.verbose('Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
+          logger.info('CemAcpt::Provision::Terraform') { 'Using Linux backend' }
+          logger.verbose('CemAcpt::Provision::Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
           CemAcpt::Provision::Linux.new(@config, @provision_data)
         elsif CemAcpt::Provision::Windows.use_for?(test_name)
-          logger.info('Terraform') { 'Using Windows backend' }
-          logger.verbose('Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
+          logger.info('CemAcpt::Provision::Terraform') { 'Using Windows backend' }
+          logger.verbose('CemAcpt::Provision::Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
           CemAcpt::Provision::Windows.new(@config, @provision_data)
         else
           err_msg = [
@@ -136,9 +128,8 @@ module CemAcpt
             "Known OSes are: #{CemAcpt::Provision::Linux.valid_names.join(', ')}",
             "and #{CemAcpt::Provision::Windows.valid_names.join(', ')}.",
             "Known versions are: #{CemAcpt::Provision::Linux.valid_versions.join(', ')}",
-            ", and #{CemAcpt::Provision::Windows.valid_versions.join(', ')}."
+            ", and #{CemAcpt::Provision::Windows.valid_versions.join(', ')}.",
           ].join(' ')
-          logger.error('Terraform') { err_msg }
           raise ArgumentError, err_msg
         end
       end
@@ -146,57 +137,55 @@ module CemAcpt
       def new_environment(config)
         env = (config.get('terraform.environment') || {})
         env['CLOUDSDK_PYTHON_SITEPACKAGES'] = '1' # This is needed for gcloud to use numpy
-        logger.verbose('Terraform') { "Using environment:\n#{JSON.pretty_generate(env)}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Using environment:\n#{JSON.pretty_generate(env)}" }
         env
       end
 
       def new_working_dir
-        logger.debug('Terraform') { 'Creating new working directory' }
+        logger.debug('CemAcpt::Provision::Terraform') { 'Creating new working directory' }
         base_dir = File.join(@config.get('terraform.dir'), @config.get('platform.name'))
-        logger.verbose('Terraform') { "Base directory defined as #{base_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Base directory defined as #{base_dir}" }
         @backend.base_provision_directory = base_dir
-        logger.verbose('Terraform') { 'Base directory set in backend' }
+        logger.verbose('CemAcpt::Provision::Terraform') { 'Base directory set in backend' }
         work_dir = File.join(@config.get('terraform.dir'), "test_#{Time.now.to_i}")
-        logger.verbose('Terraform') { "Working directory defined as #{work_dir}" }
-        logger.verbose('Terraform') { "Copying backend provision directory #{@backend.provision_directory} to working directory" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Working directory defined as #{work_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Copying backend provision directory #{@backend.provision_directory} to working directory" }
         FileUtils.cp_r(@backend.provision_directory, work_dir)
-        logger.verbose('Terraform') { "Copied provision directory #{@backend.provision_directory} to #{work_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Copied provision directory #{@backend.provision_directory} to #{work_dir}" }
         FileUtils.cp(@provision_data[:module_package_path], work_dir)
         @module_package_path = File.join(work_dir, File.basename(@provision_data[:module_package_path]))
-        logger.verbose('Terraform') { "Copied module package #{@provision_data[:module_package_path]} to #{work_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Copied module package #{@provision_data[:module_package_path]} to #{work_dir}" }
         if File.exist?(@provision_data[:private_key])
           FileUtils.cp(@provision_data[:private_key], work_dir)
           @private_key = File.join(work_dir, File.basename(@provision_data[:private_key]))
-          logger.verbose('Terraform') { "Copied private key #{@provision_data[:private_key]} to #{work_dir}" }
+          logger.verbose('CemAcpt::Provision::Terraform') { "Copied private key #{@provision_data[:private_key]} to #{work_dir}" }
         end
         if File.exist?(@provision_data[:public_key])
           FileUtils.cp(@provision_data[:public_key], work_dir)
           @public_key = File.join(work_dir, File.basename(@provision_data[:public_key]))
-          logger.verbose('Terraform') { "Copied public key #{@provision_data[:public_key]} to #{work_dir}" }
+          logger.verbose('CemAcpt::Provision::Terraform') { "Copied public key #{@provision_data[:public_key]} to #{work_dir}" }
         end
         work_dir
       rescue StandardError => e
-        logger.error('Terraform') { 'Error creating working directory' }
+        logger.error('CemAcpt::Provision::Terraform') { 'Error creating working directory' }
         raise e
       end
 
       def validate_working_dir!
-        logger.debug('Terraform') { "Validating working directory #{working_dir}" }
-        logger.verbose('Terraform') { "Content of #{working_dir}:\n#{Dir.glob(File.join(working_dir, '*')).join("\n")}" }
+        logger.debug('CemAcpt::Provision::Terraform') { "Validating working directory #{working_dir}" }
+        logger.verbose('CemAcpt::Provision::Terraform') { "Content of #{working_dir}:\n#{Dir.glob(File.join(working_dir, '*')).join("\n")}" }
         raise "Terraform working directory #{working_dir} does not exist" unless File.directory?(working_dir)
         raise "Terraform working directory #{working_dir} does not contain a Terraform file" unless Dir.glob(File.join(working_dir, '*.tf')).any?
 
-        logger.info('Terraform') { "Using working directory: #{working_dir}" }
+        logger.info('CemAcpt::Provision::Terraform') { "Using working directory: #{working_dir}" }
       rescue StandardError => e
-        logger.error('Terraform') { 'Error validating working directory' }
         raise e
       end
 
       def save_vars_to_file!(vars)
-        logger.debug('Terraform') { "Saving vars to file #{File.join(working_dir, DEFAULT_VARS_FILE)}" }
+        logger.debug('CemAcpt::Provision::Terraform') { "Saving vars to file #{File.join(working_dir, DEFAULT_VARS_FILE)}" }
         File.write(File.join(working_dir, DEFAULT_VARS_FILE), vars.to_json)
       rescue StandardError => e
-        logger.error('Terraform') { 'Error saving vars to file' }
         raise e
       end
 
@@ -214,7 +203,6 @@ module CemAcpt
         end
         node_data.to_json
       rescue StandardError => e
-        logger.error('Terraform') { 'Error creating node data' }
         raise e
       end
 
@@ -228,7 +216,6 @@ module CemAcpt
           }
         )
       rescue StandardError => e
-        logger.error('Terraform') { 'Error creating formatted vars' }
         raise e
       end
     end

data/lib/cem_acpt/provision.rb

@@ -10,7 +10,7 @@ module CemAcpt
     def self.new_provisioner(config, provision_data)
       case config.get('provisioner')
       when 'terraform'
-        logger.debug('Provision') { 'Using Terraform provisioner' }
+        logger.debug('CemAcpt::Provision') { 'Using Terraform provisioner' }
         CemAcpt::Provision::Terraform.new(config, provision_data)
       else
         raise ArgumentError, "Unknown provisioner #{config.get('provisioner')}"

data/lib/cem_acpt/puppet_helpers.rb

@@ -30,7 +30,7 @@ module CemAcpt
         _metadata = builder.metadata
 
         # Builds the module package
-        logger.info("Building module package for #{builder.release_name}")
+        logger.info('CemAcpt::PuppetHelpers') { "Building module package for #{builder.release_name}" }
         builder.build
       end
     end

data/lib/cem_acpt/test_data.rb

@@ -33,7 +33,7 @@ module CemAcpt
       # Extracts, formats, and returns a test data hash.
       # @return [Array<Hash>] an array of test data hashes
       def acceptance_test_data
-        logger.info 'Gathering acceptance test data...'
+        logger.info('CemAcpt::TestData') { 'Gathering acceptance test data...' }
         raise "No 'tests' entry found in config" unless @config.has?('tests')
 
         @config.get('tests').each_with_object([]) do |test_name, a|
@@ -45,7 +45,7 @@ module CemAcpt
           raise "Goss file not found for test #{test_name}" unless File.exist?(goss_file)
           raise "Puppet manifest not found for test #{test_name}" unless File.exist?(puppet_manifest)
 
-          logger.debug("Complete test directory found for test #{test_name}: #{test_dir}")
+          logger.debug('CemAcpt::TestData') { "Complete test directory found for test #{test_name}: #{test_dir}" }
           test_data = {
             test_name: test_name,
             test_dir: File.expand_path(test_dir),
@@ -71,7 +71,7 @@ module CemAcpt
         @acceptance_tests = acpt_test_dir.children.select { |f| f.directory? && File.exist?(File.join(f, 'goss.yaml')) }.map(&:to_s)
         raise 'No acceptance tests found' if @acceptance_tests.empty?
 
-        logger.info "Found #{@acceptance_tests.size} acceptance tests"
+        logger.info('CemAcpt') { "Found #{@acceptance_tests.size} acceptance tests" }
       end
 
       # Processes a for_each statement in the test data config.

data/lib/cem_acpt/test_runner/log_formatter/error_formatter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  module TestRunner
+    module LogFormatter
+      class ErrorFormatter
+        def inspect
+          to_s
+        end
+
+        def to_s
+          "#<#{self.class.name}:0x#{object_id.to_s(16)}>"
+        end
+
+        def summary(response)
+          "Error: #{response.summary}"
+        end
+
+        def results(response)
+          [response.summary, response.results.join("\n")]
+        end
+
+        def host_name(response)
+          "Error: #{response.error.class.name}"
+        end
+
+        def test_name(response)
+          "Error: #{response.error.class.name}"
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/log_formatter.rb

@@ -1,10 +1,19 @@
 # frozen_string_literal: true
 
 require_relative 'log_formatter/goss_action_response'
+require_relative 'log_formatter/error_formatter'
 
 module CemAcpt
   module TestRunner
     # Holds classes for formatting test runner results
-    module LogFormatter; end
+    module LogFormatter
+      def self.new_formatter(result, *args, **kwargs)
+        if result.error?
+          ErrorFormatter.new
+        else
+          GossActionResponse.new(*args)
+        end
+      end
+    end
   end
 end