cem_acpt 0.10.3 → 0.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2efdc497a91addf155de008bebfc7062fa1bac0336be8c932dfe660a6f9a2322
-  data.tar.gz: 7bcb94758bd21aa8f826405a6620d42529b621f4194bb2ed1ca067c159dfeefa
+  metadata.gz: 3f6d280d4ac5c56cd722e9a2a906647996cc18a5db7babc1f950b74a44306fc3
+  data.tar.gz: f62af838277ee9e25b2af58093d1183460bc9022a175399c5830d38e49d01620
 SHA512:
-  metadata.gz: 7c4c6c1f60167444e9f7b2735c6f12fb4258fa2c37b4d2552d5e4d2a3a1d25d555aa38a15592c22a9784fa7430789bb18796f50185e60248bdd7c48227be49d2
-  data.tar.gz: '09a2205577b3f8542dac2f849f769512978b64b044a97c2ec8a731e46171be0116f86e3b4857bc78f6e30f381a7fe7a2256991856b33d225f121e6bf95f09bb8'
+  metadata.gz: 0d573fb143effd9425d6b38f6e764c8bafb44acb583bf95b9f2a600ca787e975875e2004edcaedc78876c303342663376334fde8fff37c5fef1bed3b131c9442
+  data.tar.gz: a4e164fd64921e3bdde69e664f846ebe3fc307e1d919fff61df40634ac34794124d5e15539c3c657f874c6e9403228e3dca2593c4fe54358b0c588ba92e5e698

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cem_acpt (0.10.3)
+    cem_acpt (0.10.4)
       async-http (>= 0.60, < 0.70)
       bcrypt_pbkdf (>= 1.0, < 2.0)
       deep_merge (>= 1.2, < 2.0)

data/lib/cem_acpt/actions.rb

@@ -3,6 +3,7 @@
 require 'async'
 require 'async/barrier'
 require 'async/http/internet'
+require_relative 'logging'
 
 module CemAcpt
   # Provides a way to register actions to be executed if the registered
@@ -12,7 +13,7 @@ module CemAcpt
     class Action
       attr_reader :name, :order
 
-      def initialize(name, order = 0, &block)
+      def initialize(name, order: 0, &block)
         @name = name.to_sym
         @order = order
         @block = block
@@ -42,7 +43,7 @@ module CemAcpt
       #   order of the associated action groups' execution order. Defaults to 0.
       # @param block [Proc] the block to be executed
       def register_action(name, order: 0, &block)
-        new_action = Action.new(name, order, &block)
+        new_action = Action.new(name, order: order, &block)
         @actions << new_action
         sort!
         self # return self to allow chaining
@@ -119,6 +120,8 @@ module CemAcpt
     class << self
       attr_reader :config
 
+      include CemAcpt::Logging
+
       # Configures the Actions module.
       # @param world_config [CemAcpt::Config::Base] the current config for the "world"
       # @yield [CemAcpt::Actions::ActionConfig] the config object for the Actions module
@@ -134,12 +137,14 @@ module CemAcpt
       # @option opts [Hash] :context the context to be passed to the actions
       # @return [Array] the results of the actions
       def execute(**opts)
+        logger.debug('CemAcpt::Actions') { "Executing registered actions with opts: #{opts}" }
         context = opts[:context] || {}
         ordered_groups = config.groups.values.sort_by(&:order)
         ordered_groups.each_with_object([]) do |group, results|
           actions = group.filter_actions
           next if actions.empty?
 
+          logger.debug('CemAcpt::Actions') { "Executing action group: #{group.name}" }
           context[:group] = group.name
           context[:actions] = actions.map(&:name)
           if group.async

data/lib/cem_acpt/goss/api/action_response.rb

@@ -4,13 +4,14 @@ module CemAcpt
   module Goss
     module Api
       class ActionResponse
-        attr_reader :host, :action, :body
+        attr_reader :host, :action, :body, :metadata
 
         def initialize(host, action, status, body)
           @host = host
           @action = action
           @status = status
           @body = body
+          @metadata = {}
         end
 
         def to_s
@@ -27,6 +28,7 @@ module CemAcpt
             action: action,
             status: @status,
             body: @body,
+            metadata: @metadata,
           }
         end
 
@@ -40,11 +42,11 @@ module CemAcpt
         end
 
         def error
-          results.find(&:error) || StandardError.new('Unknown error')
+          results.find(&:error)
         end
 
         def error?
-          !success?
+          !success? && !error.nil?
         end
 
         def results

data/lib/cem_acpt/goss/api.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'async'
-require 'async/barrier'
-require 'async/http/internet'
 require 'json'
 require_relative 'api/action_response'
 require_relative '../logging'
@@ -31,64 +28,37 @@ module CemAcpt
           URI("http://#{host}:#{ACTIONS[action.to_sym]}")
         end
 
-        # Run the specified actions against the specified hosts asynchronously.
-        # @param context [Hash] The context for the test run.
-        # @option context [Array<String>] :hosts The hosts to run the actions against. Each
-        #   host should be a public IP address or a DNS-resolvable name.
-        # @option context [Queue] :results The queue to push the results to.
-        # @option [Array<String>] :actions The actions to run.
-        # @return [Queue] The queue of results.
-        def run_actions_async(context = {})
-          actions = context[:actions] || []
-          hosts = context[:hosts] || []
-          results = context[:results] || Queue.new
-          raise ArgumentError, 'hosts must be an Array' unless hosts.is_a?(Array)
-          raise ArgumentError, 'results must be a Queue-like object implementing #<<' unless results.respond_to?(:<<)
-          raise ArgumentError, 'actions must be an Array' unless actions.is_a?(Array)
-
-          actions.map!(&:to_sym)
-          actions.select! { |action| ACTIONS.key?(action) }
-          logger.info('CemAcpt::Goss::Api') do
-            "Running test actions #{actions.join(', ')} against #{hosts.size} host(s)..."
-          end
-          task = Async do
-            internet = Async::HTTP::Internet.new
-            barrier = Async::Barrier.new
-            barrier.async do
-              hosts.each do |host|
-                actions.each do |action|
-                  results << run_action(internet, host, action)
-                end
-              end
-            end
-            barrier.wait
-          ensure
-            internet&.close
+        # Get the JSON response from the specified URI.
+        # @param uri [URI] The URI to get the JSON response from.
+        # @param internet [nil, Async::HTTP::Internet] The object to use for the request. If nil,
+        #   Net::HTTP will be used.
+        # @return [Array] The status code and the parsed JSON response body.
+        def get(uri, internet = nil)
+          if internet.nil?
+            require 'net/http'
+            response = Net::HTTP.get_response(uri)
+            [response.code, JSON.parse(response.body)]
+          else
+            response = internet.get(uri.to_s)
+            [response.status, JSON.parse(response.read)]
           end
-          task.wait
-          logger.info('CemAcpt::Goss::Api') { 'Finished running test actions, returning results...' }
-          results
-        end
-
-        def run_action_async(context = {})
-          context[:results] ||= Queue.new
-          context[:hosts].each do |host|
-            context[:results] << run_action(context[:internet], host, context[:action])
-          end
-          context[:results]
         end
 
         # Run the specified action against the specified host.
-        # @param internet [Async::HTTP::Internet] The Async::HTTP::Internet object to use for the request.
         # @param host [String] The host to run the action against. This should be
         #   a public IP address or a DNS-resolvable name.
         # @param action [Symbol] The action to run.
+        # @param internet [nil, Async::HTTP::Internet] The object to use for the request.
         # @return [ActionResponse] The response from the action.
-        def run_action(internet, host, action)
+        def run_action(host, action, internet = nil)
           uri = action_uri(host, action)
-          response = internet.get(uri.to_s)
-          body = JSON.parse(response.read)
-          ActionResponse.new(host, action, response.status, body)
+          status_code, body = get(uri, internet)
+          ActionResponse.new(host, action, status_code, body)
+        end
+
+        def get_run_logs(host, internet = nil)
+          status_code, body = get(URI("http://#{host}:8083/run-logs"), internet)
+          ActionResponse.new(host, :run_logs, status_code, body)
         end
       end
     end

data/lib/cem_acpt/logging.rb

@@ -198,7 +198,7 @@ module CemAcpt
               else
                 severity
               end
-        block_message = block_given? ? block.call : nil
+        block_message = block ? yield : nil
         base = [progname, block_message].compact.join(': ')
         return if base.empty?
 
@@ -272,7 +272,7 @@ module CemAcpt
 
         @log_config = {
           logdev: $stdout,
-          shift_age: 'o',
+          shift_age: 0,
           shift_size: 1_048_576,
           level: ::Logger::INFO,
           progname: 'CemAcpt',

data/lib/cem_acpt/provision/terraform/linux.rb

@@ -30,6 +30,9 @@ module CemAcpt
         commands = [
           "sudo /opt/puppetlabs/puppet/bin/puppet module install #{destination_provision_directory}/#{remote_module_package_name}",
           'curl -fsSL https://goss.rocks/install | sudo sh',
+          'sudo /opt/puppetlabs/puppet/bin/gem install webrick',
+          'sudo chmod +x /opt/cem_acpt/log_service/log_service.rb',
+          'sudo /opt/cem_acpt/log_service/log_service.rb',
         ]
         unless systemd_files.empty?
           systemd_files.each do |file|
@@ -54,6 +57,10 @@ module CemAcpt
           ]
           commands = (commands << provision_commands).flatten
           commands
+        elsif image_name.include?('ubuntu')
+          commands = ['sudo apt purge -y unattended-upgrades', 'sudo apt-get update -y']
+          commands = (commands << provision_commands).flatten
+          commands
         else
           provision_commands
         end

data/lib/cem_acpt/test_runner/test_results.rb

@@ -28,14 +28,14 @@ module CemAcpt
         end
 
         def <<(result)
-          case result
-          when CemAcpt::Goss::Api::ActionResponse, CemAcpt::Bolt::SummaryResults
-            @results_queue << TestActionResult.new(result, new_formatter(result))
-          when StandardError
-            @results_queue << TestErrorActionResult.new(result, new_formatter(result))
-          else
-            raise ArgumentError, "result must be a CemAcpt::Goss::Api::ActionResponse, CemAcpt::Bolt::SummaryResults, or StandardError, got #{result.class}"
-          end
+          @results_queue << case result
+                            when TestActionResult, TestErrorActionResult
+                              result
+                            when StandardError
+                              TestErrorActionResult.new(result, new_formatter(result))
+                            else
+                              TestActionResult.new(result, new_formatter(result))
+                            end
         end
 
         def to_a

data/lib/cem_acpt/test_runner.rb

@@ -28,6 +28,7 @@ module CemAcpt
 
       def initialize(config)
         @config = config
+        @module_builder = CemAcpt::Utils::Puppet::ModulePackageBuilder.new(config.get('module_dir'))
         @run_data = {}
         @duration = 0
         @exit_code = 0
@@ -101,8 +102,7 @@ module CemAcpt
           destroy_test_nodes
         end
       rescue StandardError => e
-        logger.verbose('CemAcpt::TestRunner') { "Error cleaning up: #{e}" }
-        logger.verbose('CemAcpt::TestRunner') { e.backtrace.join("\n") }
+        logger.verbose('CemAcpt::TestRunner') { "Error cleaning up: #{e}\n#{e.backtrace.join("\n")}" }
       ensure
         logger.end_ci_group
       end
@@ -118,16 +118,20 @@ module CemAcpt
           c.register_group(:goss, order: 0, async: true)
           CemAcpt::Goss::Api::ACTIONS.each_key do |a|
             c[:goss].register_action(a) do |context|
-              run_goss_test(context)
+              context[:hosts] = @hosts
+              context[:results] = @results
+              context[:hosts].each do |host|
+                action_res = CemAcpt::Goss::Api.run_action(host, context[:action], context[:internet])
+                action_res.metadata[:run_logs] = CemAcpt::Goss::Api.get_run_logs(host, context[:internet]).body
+                context[:results] << action_res
+              end
+              context[:results]
             end
           end
           c.register_group(:bolt, order: 1).register_action(:bolt) do |context|
             run_bolt_tests(context)
           end
         end
-        logger.debug('CemAcpt::TestRunner') { "All actions #{CemAcpt::Actions.config.action_names}" }
-        logger.debug('CemAcpt::TestRunner') { "Only actions: #{CemAcpt::Actions.config.only}" }
-        logger.debug('CemAcpt::TestRunner') { "Except actions: #{CemAcpt::Actions.config.except}" }
         logger.info('CemAcpt::TestRunner') do
           "Configured and registered actions, will run actions: #{CemAcpt::Actions.config.action_names.join(', ')}"
         end
@@ -135,13 +139,7 @@ module CemAcpt
 
       # @return [String] The path to the module package
       def build_module_package
-        pkg_path = if config.get('tests').first.include? 'windows'
-                     CemAcpt::Utils.package_win_module(config.get('module_dir'))
-                   else
-                     CemAcpt::Utils::Puppet.build_module_package(config.get('module_dir'))
-                   end
-        logger.info('CemAcpt::TestRunner') { "Created module package: #{pkg_path}..." }
-        pkg_path
+        @module_builder.build
       end
 
       # @return [Array<String>] The paths to the ssh private key, public key, and known hosts file
@@ -261,20 +259,6 @@ module CemAcpt
         CemAcpt::Actions.execute
       end
 
-      def run_goss_tests(context = {})
-        logger.info('CemAcpt::TestRunner') { 'Running goss tests...' }
-        context[:hosts] = @hosts
-        context[:results] = @results
-        CemAcpt::Goss::Api.run_actions_async(context)
-      end
-
-      def run_goss_test(context = {})
-        logger.info('CemAcpt::TestRunner') { "Running Goss test for action #{context[:action]}..." }
-        context[:hosts] = @hosts
-        context[:results] = @results
-        CemAcpt::Goss::Api.run_action_async(context)
-      end
-
       def run_bolt_tests(_context = {})
         logger.info('CemAcpt::TestRunner') { 'Running Bolt tests...' }
         # If the Bolt config has tests:only or tests:ignore lists, we need to filter the hosts
@@ -364,6 +348,18 @@ module CemAcpt
             logger.info { r }
           else
             logger.error { r }
+            logs = if config.debug? && !config.get('puppet.no_debug')
+                     result.metadata[:run_logs]&.dup
+                   else
+                     filtered = result.metadata[:run_logs]&.dup
+                     filtered&.each { |_, v| v.reject! { |l| l.include?('(debug)') } }
+                     filtered
+                   end
+            unless logs.nil? || logs.empty?
+              logger.debug { "Logs for provision apply:\n#{logs['provision'].join}" } if logs['provision']
+              logger.error { "Logs for idempotent apply:\n#{logs['idempotent'].join}" } if logs['idempotent']
+              logger.error { "Logs for noop apply:\n#{logs['noop'].join}" } if logs['noop']
+            end
           end
         end
       end

data/lib/cem_acpt/utils/puppet.rb

@@ -2,27 +2,61 @@
 
 require 'puppet/modulebuilder'
 require 'fileutils'
+require_relative '../logging'
 
 module CemAcpt
   module Utils
     # Puppet-related utilities
     module Puppet
-      # Builds a Puppet module package.
-      # @param module_dir [String] Path to the module directory. If target_dir
-      #   is specified as a relative path, it will be relative to the module dir.
-      # @param target_dir [String] Path to the target directory where the package
-      #   will be built. This defaults to the relative path 'pkg/'.
-      # @param should_log [Boolean] Whether or not to log the build process.
-      # @return [String] Path to the built package.
-      def self.build_module_package(module_dir, target_dir = nil, should_log: false)
-        builder_logger = should_log ? logger : nil
-        builder = ::Puppet::Modulebuilder::Builder.new(::File.expand_path(module_dir), target_dir, builder_logger)
-
-        # Validates module metadata by raising exception if invalid
-        _metadata = builder.metadata
-
-        # Builds the module package
-        builder.build
+      class ModulePackageBuilder
+        include CemAcpt::Logging
+
+        attr_reader :module_dir, :target_dir, :should_log, :metadata, :package_path
+
+        def initialize(module_dir, target_dir: nil, should_log: false, builder: ::Puppet::Modulebuilder::Builder)
+          @module_dir = ::File.expand_path(module_dir)
+          @target_dir = target_dir
+          @should_log = should_log
+          @builder = new_builder(builder)
+          @metadata = @builder.metadata # Validates metadata
+          @package_built = false
+          @package_path = nil
+        end
+
+        # Builds a Puppet module package.
+        # @return [String] Path to the built package.
+        def build
+          @package_path = @metadata['name'].include?('windows') ? build_windows_package : @builder.build
+          @package_built = true
+          logger.info('CemAcpt::Utils::Puppet::ModulePackageBuilder') { "Built module package: #{@package_path}" }
+          @package_path
+        end
+
+        # Determines if a package has been built.
+        def package_built?
+          @package_built
+        end
+
+        private
+
+        # This method is used to create a new builder instance.
+        # It is used to allow for dependency injection in tests.
+        def new_builder(builder)
+          builder.new(@module_dir, @target_dir, @should_log ? logger : nil)
+        end
+
+        # This method is used to build a Windows module package.
+        # It's unknown exactly why we need to package the Windows module differently from the Linux module.
+        # When we find out, we should add an explanation here.
+        def build_windows_module
+          package_name = "#{@metadata['name']}.tar.gz"
+          package_file = ::File.join(@module_dir, package_name)
+          ::FileUtils.rm_f(package_file) # Remove the old package file if it exists
+          Dir.chdir(@module_dir) do
+            `touch #{package_name} && tar -czf #{package_name} --exclude=#{package_name} *`
+          end
+          package_file
+        end
       end
     end
   end

data/lib/cem_acpt/utils.rb

@@ -13,6 +13,7 @@ module CemAcpt
     class << self
       include CemAcpt::Logging
 
+      # This is method currently unused, see lib/cem_acpt/utils/puppet.rb for details.
       def package_win_module(module_dir)
         # Path to the package file
         package_file = File.join(module_dir, 'puppetlabs-cem_windows.tar.gz')
@@ -53,7 +54,7 @@ module CemAcpt
       def get_windows_login_info(instance_name, hash_of_instance)
         password_and_username = {}
         password_and_username[instance_name] = {}
-        info = reset_password_readiness_polling(instance_name).split(/\r?\n/)[1..2]
+        info = reset_password_readiness_polling(instance_name).split(%r{\r?\n})[1..2]
         info.each do |line|
           key_val = line.split(' ')
           password_and_username[instance_name][key_val[0].strip.delete(':')] = key_val[1].strip

data/lib/cem_acpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CemAcpt
-  VERSION = '0.10.3'
+  VERSION = '0.10.4'
 end

data/lib/terraform/gcp/linux/log_service/log_service.rb

@@ -0,0 +1,62 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+# frozen_string_literal: true
+
+require 'json'
+require 'webrick'
+
+# Implements a simple HTTP server that serves the various run logs
+class RunLogs < WEBrick::HTTPServlet::AbstractServlet
+  # rubocop:disable Naming/MethodName
+  def do_GET(_request, response)
+    run_logs = {}
+    p_logs = read_log_file('/opt/cem_acpt/provision_apply.log')
+    i_logs = read_log_file('/opt/cem_acpt/idempotent_apply.log')
+    n_logs = read_log_file('/opt/cem_acpt/noop_apply.log')
+    run_logs['provision'] = p_logs unless p_logs.empty?
+    run_logs['idempotent'] = i_logs unless i_logs.empty?
+    run_logs['noop'] = n_logs unless n_logs.empty?
+    response.status = run_logs.empty? ? 404 : 200
+    response['Content-Type'] = 'application/json'
+    response.body = run_logs.to_json
+  end
+  # rubocop:enable Naming/MethodName
+
+  private
+
+  def read_log_file(file)
+    if File.exist?(file)
+      format_log(File.readlines(file))
+    else
+      []
+    end
+  rescue StandardError => e
+    ['Error reading log file', e.message, e.backtrace.join("\n")]
+  end
+
+  def format_log(log)
+    # Force UTF-8 on each line so they aren't mangled by the UTF to ASCII conversion
+    encoded = log.map { |l| l.force_encoding(Encoding::UTF_8) }
+    # Now we need to condense the lines into logical groups based on log entry. This
+    # allows us to filter out debug logs that span multiple lines.
+    memo = nil
+    encoded.each_with_object([]) do |line, formatted|
+      if memo.nil?
+        memo = line
+      elsif line.match?(%r{^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \+\d{4}.*$})
+        formatted << memo
+        memo = line
+      else
+        memo += line
+      end
+    end
+  end
+end
+
+server = WEBrick::HTTPServer.new(Port: 8083)
+server.mount '/run-logs', RunLogs
+
+trap 'INT' do
+  server.shutdown
+end
+WEBrick::Daemon.start
+server.start

data/lib/terraform/gcp/linux/systemd/goss-acpt.service

@@ -2,7 +2,7 @@
 Description=unit file for goss server acpt endpoint service
 
 [Service]
-ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss.yaml serve -f json --endpoint "/acpt" --cache "10m"
+ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss.yaml serve -f json -o pretty --endpoint "/acpt" --cache '0s'
 
 [Install]
 WantedBy=multi-user.target

data/lib/terraform/gcp/linux/systemd/goss-idempotent.service

@@ -2,7 +2,7 @@
 Description=unit file for goss server acpt endpoint service
 
 [Service]
-ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss/puppet_idempotent.yaml serve -f json -l ":8081" --endpoint "/idempotent" --cache "10m"
+ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss/puppet_idempotent.yaml serve -f json -o pretty -l ":8081" --endpoint "/idempotent" --cache '0s'
 
 [Install]
 WantedBy=multi-user.target

data/lib/terraform/gcp/linux/systemd/goss-noop.service

@@ -2,7 +2,7 @@
 Description=unit file for goss server acpt endpoint service
 
 [Service]
-ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss/puppet_noop.yaml serve -f json -l ":8082" --endpoint "/noop" --cache "10m"
+ExecStart=/usr/local/bin/goss -g /opt/cem_acpt/goss/puppet_noop.yaml serve -f json -o pretty -l ":8082" --endpoint "/noop" --cache '0s'
 
 [Install]
 WantedBy=multi-user.target

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cem_acpt
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.10.4
 platform: ruby
 authors:
 - puppetlabs
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-11 00:00:00.000000000 Z
+date: 2024-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async-http
@@ -280,6 +280,7 @@ files:
 - lib/cem_acpt/version.rb
 - lib/terraform/gcp/linux/goss/puppet_idempotent.yaml
 - lib/terraform/gcp/linux/goss/puppet_noop.yaml
+- lib/terraform/gcp/linux/log_service/log_service.rb
 - lib/terraform/gcp/linux/main.tf
 - lib/terraform/gcp/linux/systemd/goss-acpt.service
 - lib/terraform/gcp/linux/systemd/goss-idempotent.service
@@ -313,7 +314,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: CEM Acceptance Tests