cem_acpt 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f33bff59cae8ea713a55134ce914c83aa3583e16e56cd0d5aca4bc7106df092
-  data.tar.gz: e1f8f76188ad32473da89fc389eead2b9d63af2bc8856cf3b30863b6454f89d2
+  metadata.gz: 853cc9a89c0feaf798bb5efdfc0c8af47d2664121cddd3794b65a61d5dc80851
+  data.tar.gz: 6d0d5c67e27918d7897ea567a4134249c8a81e23ea95e6a3850a65585ebdd3e0
 SHA512:
-  metadata.gz: 8a54dfbf7dcf40684b3d00bec253822943cacd7388d3dabaaf6d6ed260811c5862d694544ea07f2cd135274cd9227aaa16e00e0dcc712e41c906bfffaa8fef17
-  data.tar.gz: '06678cf120f230d981d5247c39b688e8ca96ed13225e0595595579041232e17c3220044b4f51091ab6900e10b8b4f2aedd0115da901a9619025e7576aa7ceec2'
+  metadata.gz: 244c701ba966fa878c399efba4953db7be8f5977c906501f9d2da8410583ebf2c2c505c687dc03e8cd97b325f362b28472f68c089065cda21f83d01d16c15d22
+  data.tar.gz: ae7ca222e4264ed32ed47df2745d76c8347c1b5eab94cd2f0278af581c7c7108fb006232c5bbb4d37b2455559c0826a5630498cd620d4fd85a8948b8a0856c22

data/.rubocop.yml

@@ -3,7 +3,7 @@ require:
 - rubocop-performance
 - rubocop-rspec
 AllCops:
-  NewCops: enable
+  NewCops: disable
   SuggestExtensions: false
   DisplayCopNames: true
   TargetRubyVersion: '3.2'
@@ -414,6 +414,8 @@ Style/HashAsLastArrayItem:
   Enabled: false
 Style/HashLikeCase:
   Enabled: false
+Style/HashSyntax:
+  Enabled: false
 Style/HashTransformKeys:
   Enabled: false
 Style/HashTransformValues:

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    cem_acpt (0.7.2)
+    cem_acpt (0.8.0)
       async-http (>= 0.60, < 0.70)
       bcrypt_pbkdf (>= 1.0, < 2.0)
       deep_merge (>= 1.2, < 2.0)
       ed25519 (>= 1.2, < 2.0)
       puppet-modulebuilder (>= 0.0.1)
-      ruby-terraform (~> 1.7)
+      winrm (>= 2.3, < 3.0)
 
 GEM
   remote: https://rubygems.org/
@@ -30,25 +30,32 @@ GEM
     async-pool (0.4.0)
       async (>= 1.25)
     bcrypt_pbkdf (1.1.0)
+    builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
     console (1.16.2)
       fiber-local
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
     ed25519 (1.3.0)
+    erubi (1.12.0)
+    ffi (1.15.5)
     fiber-local (1.0.0)
-    hamster (3.0.0)
-      concurrent-ruby (~> 1.0)
-    immutable-struct (2.4.1)
+    gssapi (1.3.1)
+      ffi (>= 1.0.1)
+    gyoku (1.4.0)
+      builder (>= 2.1.2)
+      rexml (~> 3.0)
+    httpclient (2.8.3)
     io-event (1.1.7)
     json (2.6.3)
-    lino (3.1.0)
-      hamster (~> 3.0)
-      open4 (~> 1.3)
+    little-plugger (1.1.4)
+    logging (2.3.1)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.14)
     method_source (1.0.0)
     minitar (0.9)
-    open4 (1.3.4)
+    multi_json (1.15.0)
+    nori (2.6.0)
     parallel (1.22.1)
     parser (3.2.1.1)
       ast (~> 2.4.1)
@@ -95,15 +102,34 @@ GEM
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.28.0)
       parser (>= 3.2.1.0)
+    rubocop-capybara (2.18.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.23.1)
+      rubocop (~> 1.33)
+    rubocop-performance (1.19.0)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rspec (2.23.2)
+      rubocop (~> 1.33)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
-    ruby-terraform (1.7.0)
-      immutable-struct (~> 2.4)
-      lino (~> 3.0)
+    rubyntlm (0.6.3)
     timers (4.3.5)
     traces (0.9.1)
     unicode-display_width (2.4.2)
+    winrm (2.3.6)
+      builder (>= 2.1.2)
+      erubi (~> 1.8)
+      gssapi (~> 1.2)
+      gyoku (~> 1.0)
+      httpclient (~> 2.2, >= 2.2.0.2)
+      logging (>= 1.6.1, < 3.0)
+      nori (~> 2.0)
+      rubyntlm (~> 0.6.0, >= 0.6.3)
 
 PLATFORMS
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
@@ -113,6 +139,8 @@ DEPENDENCIES
   rake (>= 12.0)
   rspec (>= 3.0)
   rubocop
+  rubocop-performance
+  rubocop-rspec
 
 BUNDLED WITH
-   2.4.6
+   2.4.18

data/README.md

@@ -89,6 +89,10 @@ Options set in the user-specific config file will be overridden by options set i
 
 CemAcpt can be configured using command-line options. These options are specified using the `--<option>` syntax. For a full list of options, run `cem_acpt -h`. Options specified on the command-line will override options specified in the config file and environment variables.
 
+#### Tracing
+
+To aide in development, you can enable tracing on CemAcpt's Ruby code execution by using the `--trace` flag. Traces are logged only for events executed by CemAcpt code itself, and the logger code is exempt from tracing. **CAUTION** Tracing produces **a lot** of logs and it's advised to only use it when logging to a file. Additionally, the specific trace events that are logged can be specified using the `--trace-events` flag. Tracing is enabled by the [TracePoint](https://ruby-doc.org/core-3.1.0/TracePoint.html) class.
+
 ## Goss
 
 [Goss](https://github.com/goss-org/goss/blob/master/docs/manual.md) is the core infrastructure testing tool used by CemAcpt. The `goss.yaml` file is used to specify the tests to run. Please see the [Goss documentation](https://github.com/goss-org/goss/blob/master/docs/manual.md#goss-test-creation) for more information on how to write Goss tests.
@@ -248,3 +252,7 @@ images:
 ```
 
 See [sample_config.yaml](sample_config.yaml) for a more complete example.
+
+### Testing with cem_windows
+
+While testing with cem_windows follows pretty much the same outline as cem_linux, there are a few extra step that we have to do when bootstrapping the generated node. Two of the most prominents steps are enabling long path and using NSSM to start the goss service. When we enable long path on Windows, it allows the command `puppet module install` to install dependencies correctly (specifically the dsc modules). As for [NSSM](https://nssm.cc/), stands for Non-sucking service manager, it is necessary for us to use this tool because without it, we will not be able to create services for Goss. Windows services cannot run from an executable but rather through a Windows service project that can be build with Visual Studio. NSSM allows us to bypass having to create Windows service project for Goss and create services directly from the Goss executable.

data/cem_acpt.gemspec

@@ -32,7 +32,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'deep_merge', '>= 1.2', '< 2.0'
   spec.add_runtime_dependency 'ed25519', '>= 1.2', '< 2.0'
   spec.add_runtime_dependency 'puppet-modulebuilder', '>= 0.0.1'
-  spec.add_runtime_dependency 'ruby-terraform', '~> 1.7'
+  spec.add_runtime_dependency 'winrm', '>= 2.3', '< 3.0'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-performance'
+  spec.add_development_dependency 'rubocop-rspec'
 end

data/lib/cem_acpt/action_result.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  # Wrapper class for the result of an action. Provides a common interface for
+  # getting reportable data from the result.
+  class ActionResult
+    def initialize(result)
+      @result = if result.instance_of?(CemAcpt::Goss::Api::ActionResponse)
+                  result
+                else
+                  ErrorActionResult.new(result)
+                end
+    end
+
+    def error?
+      @result.is_a?(ErrorActionResult)
+    end
+
+    def method_missing(method_name, *args, **kwargs, &block)
+      if @result.respond_to?(method_name)
+        @result.send(method_name, *args, **kwargs, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      @result.respond_to?(method_name, include_private) || super
+    end
+  end
+
+  # Wrapper class for an error raised during an action.
+  class ErrorActionResult
+    attr_reader :error
+
+    def initialize(error)
+      @error = error
+    end
+
+    def error?
+      true
+    end
+
+    def success?
+      false
+    end
+
+    def to_s
+      "#<#{@error.class.name}:0x#{@error.object_id.to_s(16)}>"
+    end
+
+    def inspect
+      to_s
+    end
+
+    def to_h
+      {
+        class: @error.class.name,
+        message: @error.message,
+        backtrace: @error.backtrace,
+      }
+    end
+
+    def status
+      1
+    end
+    alias http_status status
+
+    def results
+      @error.backtrace
+    end
+
+    def results?
+      true
+    end
+
+    def summary
+      @error.message
+    end
+
+    def summary?
+      true
+    end
+  end
+end

data/lib/cem_acpt/cli.rb

@@ -9,7 +9,7 @@ module CemAcpt
       options = {}
       parser = OptionParser.new do |opts|
         opts.banner = "Usage: #{command} [options]"
-      
+
         opts.on('-h', '--help', 'Show this help message') do
           puts opts
           exit 0
@@ -21,17 +21,17 @@ module CemAcpt
           opts.on('-m', '--module-dir DIR', 'Set module directory. Example: -m "/tmp/module"') do |m|
             options[:module_dir] = m
           end
-  
+
           opts.on('-a', '--only-actions ACTIONS', 'Set actions. Example: -a "acpt,noop"') do |a|
             options[:actions] ||= {}
             options[:actions][:only] = a.split(',')
           end
-        
+
           opts.on('-A', '--except-actions ACTIONS', 'Set excluded actions. Example: -A "noop,idempotent"') do |a|
             options[:actions] ||= {}
             options[:actions][:except] = a.split(',')
           end
-        
+
           opts.on('-t', '--tests TESTS', 'Set tests. Example: -t "test1,test2"') do |t|
             options[:tests] = t.split(',')
           end
@@ -40,6 +40,10 @@ module CemAcpt
             options[:dry_run] = true
           end
 
+          opts.on('--no-build-images', 'Do not build images. Can be combined with --no-destroy-nodes to just provision nodes.') do
+            options[:no_build_images] = true
+          end
+
           opts.on('--no-linux', 'Do not build Linux images') do
             options[:cem_acpt_image] ||= {}
             options[:cem_acpt_image][:no_linux] = true
@@ -54,67 +58,87 @@ module CemAcpt
             options[:image_name_filter] = Regexp.new(f)
           end
         end
-      
+
         opts.on('-D', '--debug', 'Enable debug logging') do
           options[:log_level] = 'debug'
         end
-      
+
         opts.on('-L', '--log-file FILE', 'Log to FILE') do |file|
           options[:log_file] = file
         end
-      
+
         opts.on('-O', '--options OPTS', 'Set options. Example: -P "param1=value1,param2=value2"') do |o|
           params = o.split(',').map { |s| s.split('=') }.to_h
           params.transform_keys(&:to_sym).each do |k, v|
             options[k] = v
           end
         end
-      
+
         opts.on('-p', '--platform PLATFORM', 'Set platform. Example: -p "gcp"') do |p|
           options[:platform] = p
         end
-      
+
         opts.on('-c', '--config-file FILE', 'Set config file. Example: -c "/tmp/config.yaml"') do |c|
           options[:config_file] = c
         end
-      
+
         opts.on('-I', '--CI', 'Run in CI mode (modifies logging for Github Actions output)') do
           options[:ci_mode] = true
           options[:log_format] = 'github_action'
         end
-      
+
         opts.on('-E', '--no-destroy-nodes', 'Do not destroy nodes') do
           options[:no_destroy_nodes] = true
         end
-      
+
         opts.on('-q', '--quiet', 'Do not log to stdout') do
           options[:quiet] = true
         end
-      
+
         opts.on('-v', '--verbose', 'Enables verbose logging mode') do
           options[:verbose] = true
         end
-      
+
         opts.on('-S', '--no-epehemeral-ssh-key', 'Do not generate an ephemeral SSH key for test suites') do
           options[:no_ephemeral_ssh_key] = true
         end
-      
+
         opts.on('-V', '--version', 'Show the cem_acpt version') do
           cmd = :version
         end
-      
+
         opts.on('-Y', '--print-yaml-config', 'Loads and prints the config as YAML. Other specified options will be added to the config.') do
           cmd = :print_yaml_config
         end
-      
+
         opts.on('-X', '--explain-config', 'Loads and prints the config explanation. Other specified options will be added to the config.') do
           cmd = :print_explain_config
         end
-      
-        # NOT IMPLEMENTED
-        # opts.on('-R', '--reuse-nodes', 'Reuses test nodes if compatible node inventory exists') do
-        #   options[:reuse_nodes] = true
-        # end
+
+        opts.on('--trace', 'Enables trace output') do
+          options[:trace] = true
+        end
+
+        opts.on(
+          '--trace-events [EVENTS]',
+          [
+            'Comma-separated list of trace events to enable. Example: --trace-events "line,call,b_call,thread_begin,thread_end"',
+            'See here for list of events: https://ruby-doc.org/core-2.5.0/TracePoint.html#class-TracePoint-label-Events.',
+            'Default: line,call,b_call,thread_begin,thread_end',
+          ].join(' '),
+        ) do |events|
+          options[:trace_events] = events.split(',')
+        end
+
+        opts.on('--puppet-no-debug', 'Disables Puppet debug logging') do
+          options[:puppet] ||= {}
+          options[:puppet][:no_debug] = true
+        end
+
+        opts.on('--puppet-no-verbose', 'Disables Puppet verbose logging') do
+          options[:puppet] ||= {}
+          options[:puppet][:no_verbose] = true
+        end
       end
       parser.parse!
       [cmd, options]

data/lib/cem_acpt/config/base.rb

@@ -13,8 +13,9 @@ module CemAcpt
     using CemAcpt::CoreExt::ExtendedHash
 
     # Base class for other config classes
+    # Child classes should provide the following constant:
+    #   - VALID_KEYS - provide an array of valid top-level keys for the config as symbols
     # Child classes should implement the following methods:
-    #   - valid_keys - provide an array of valid top-level keys for the config as symbols
     #   - defaults - provide a hash of default values for the config
     #   - env_var_prefix - provide a string to prefix environment variables with (defaults to 'CEM_ACPT').
     #                      This will be converted to uppercase, have all non-alphanumeric characters replaced with
@@ -22,6 +23,23 @@ module CemAcpt
     #                      variable name.
     # However, they can override any of the methods in this class.
     class Base
+      BASE_VALID_KEYS = %i[
+        ci_mode
+        config_file
+        log_level
+        log_file
+        log_format
+        no_destroy_nodes
+        no_ephemeral_ssh_key
+        platform
+        provisioner
+        puppet
+        quiet
+        terraform
+        user_config
+        verbose
+      ].freeze
+
       attr_reader :config, :env_vars
 
       def initialize(opts: {}, config_file: nil, load_user_config: true)
@@ -36,7 +54,11 @@ module CemAcpt
 
       # @return [Array<Symbol>] Valid top-level keys for the config
       def valid_keys
-        []
+        if self.class.const_defined?(:VALID_KEYS)
+          (BASE_VALID_KEYS + self.class.const_get(:VALID_KEYS)).uniq
+        else
+          BASE_VALID_KEYS
+        end
       end
 
       # @return [Hash] The default configuration
@@ -64,6 +86,7 @@ module CemAcpt
         add_static_options!(@config)
         @config.format! # Symbolize keys of all hashes
         validate_config!
+        @dot_key_cache = {}
         # Freeze the config so it can't be modified
         # This helps with thread safety and deterministic behavior
         @config.freeze
@@ -94,14 +117,14 @@ module CemAcpt
         if key.is_a?(Symbol)
           @config[key].dup
         elsif key.is_a?(String)
-          @config.dget(key).dup
+          get(key)
         else
           raise ArgumentError, "Invalid key type '#{key.class}'"
         end
       end
 
       def get(dot_key)
-        @config.dget(dot_key).dup
+        @dot_key_cache[dot_key] ||= @config.dget(dot_key).dup
       end
       alias dget get
 

data/lib/cem_acpt/config/cem_acpt.rb

@@ -8,30 +8,13 @@ module CemAcpt
     class CemAcpt < Base
       VALID_KEYS = %i[
         actions
-        ci_mode
-        config_file
         image_name_builder
-        log_level
-        log_file
-        log_format
         module_dir
         node_data
-        no_destroy_nodes
-        no_ephemeral_ssh_key
-        platform
-        provisioner
-        quiet
-        terraform
         test_data
         tests
-        user_config
-        verbose
       ].freeze
 
-      def valid_keys
-        VALID_KEYS
-      end
-
       # The default configuration
       def defaults
         {

data/lib/cem_acpt/config/cem_acpt_image.rb

@@ -8,28 +8,12 @@ module CemAcpt
     class CemAcptImage < Base
       VALID_KEYS = %i[
         cem_acpt_image
-        ci_mode
-        config_file
         dry_run
         images
         image_name_filter
-        log_level
-        log_file
-        log_format
-        no_destroy_nodes
-        no_ephemeral_ssh_key
-        platform
-        provisioner
-        quiet
-        terraform
-        user_config
-        verbose
+        no_build_images
       ].freeze
 
-      def valid_keys
-        VALID_KEYS
-      end
-
       def env_var_prefix
         'CEM_ACPT_IMAGE'
       end
@@ -49,6 +33,7 @@ module CemAcpt
           log_level: 'info',
           log_file: nil,
           log_format: 'text',
+          no_destroy_nodes: false,
           no_ephemeral_ssh_key: false,
           platform: {
             name: 'gcp',

data/lib/cem_acpt/goss/api.rb

@@ -5,12 +5,15 @@ require 'async/barrier'
 require 'async/http/internet'
 require 'json'
 require_relative 'api/action_response'
+require_relative '../logging'
 
 module CemAcpt
   module Goss
     # Holds methods for interacting with the Goss API running on a test node.
     module Api
       class << self
+        include CemAcpt::Logging
+
         # The actions that can be run against the Goss API. The key is the action
         # name and the value is the port/endpoint of the action.
         ACTIONS = {
@@ -40,16 +43,17 @@ module CemAcpt
           raise ArgumentError, 'results must be a Queue-like object implementing #<<' unless results.respond_to?(:<<)
           raise ArgumentError, 'only must be an Array' unless except.is_a?(Array)
           raise ArgumentError, 'except must be an Array' unless except.is_a?(Array)
+          logger.info('CemAcpt::Goss::Api') { "Running test actions against #{hosts.size} host(s)..." }
           only.map!(&:to_sym)
           except.map!(&:to_sym)
           only_specified = !only.empty?
           except_specified = !except.empty?
-          Async do
+          task = Async do
             internet = Async::HTTP::Internet.new
             barrier = Async::Barrier.new
             barrier.async do
               hosts.each do |host|
-                ACTIONS.keys.each do |action, _|
+                ACTIONS.each_key do |action|
                   next if only_specified && !only.include?(action)
                   next if except_specified && except.include?(action)
 
@@ -62,6 +66,8 @@ module CemAcpt
             internet&.close
             results.close if results.respond_to?(:close)
           end
+          task.wait
+          logger.info('CemAcpt::Goss::Api') { 'Finished running test actions, returning results...' }
           results
         end