cem_acpt 0.7.1 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b51e308bc7002bf4732a8f9aba7f8a43f2a62c5af67d1f6fffc6d524da9d683f
-  data.tar.gz: 53de22ef68903167653ac98136ed657c6c34a88adc308d4191fdfec94860b294
+  metadata.gz: 251ddea81d792241accdd3e63fb1b33ce655e458c08939f6da6a043cb2eeeeef
+  data.tar.gz: 7cf6e742a5375c190bba491e607b742263e40e15b82e68d1c383d9ca634e7b4f
 SHA512:
-  metadata.gz: aa5189d13239222956bf6480ae2fa82513d241b88757e8dd3ef81d89bbbc576b71e1c5af3fbe01c3edf5a9ff3dc36506538be9572c3d647acf6536da8c3023c2
-  data.tar.gz: be23f52aa2291ba5a82e60654b2aa07de3bdc262e4795447ac54b0f38ba780a669a9984fdd64cc9dd3fcaf94932f283c598af1f8e74d67a4a44ce322490b2fdd
+  metadata.gz: 79709f1e861e2eea1b31be633076271fefa027656dcfb8e78c252d0492286cf6376547d8756772447b28c1fe6a625cf0fee289f4a84d7e5b7e9bbfa2ef182f71
+  data.tar.gz: 7fc8c5351bce46273f5d57e743e1865c6d844f89fd702da451082042921cf9aae01473f90b11a3a53e810c6e2f53132de9e75d7d8d0b97c08db105c2abe172f5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cem_acpt (0.7.1)
+    cem_acpt (0.7.3)
       async-http (>= 0.60, < 0.70)
       bcrypt_pbkdf (>= 1.0, < 2.0)
       deep_merge (>= 1.2, < 2.0)

data/lib/cem_acpt/cli.rb

@@ -36,15 +36,27 @@ module CemAcpt
             options[:tests] = t.split(',')
           end
         when :cem_acpt_image
-          opts.on('-U', '--no-linux', 'Do not build Linux images') do
+          opts.on('--dry-run', 'Logs the information for the images that would be created. Does not create images.') do
+            options[:dry_run] = true
+          end
+
+          opts.on('--no-build-images', 'Do not build images. Can be combined with --no-destroy-nodes to just provision nodes.') do
+            options[:no_build_images] = true
+          end
+
+          opts.on('--no-linux', 'Do not build Linux images') do
             options[:cem_acpt_image] ||= {}
             options[:cem_acpt_image][:no_linux] = true
           end
 
-          opts.on('-W', '--no-windows', 'Do not build Windows images') do
+          opts.on('--no-windows', 'Do not build Windows images') do
             options[:cem_acpt_image] ||= {}
             options[:cem_acpt_image][:no_windows] = true
           end
+
+          opts.on('-F', '--filter FILTER', 'Filter images to build by name with regex. Example: -F "(alma|rhel)-8"') do |f|
+            options[:image_name_filter] = Regexp.new(f)
+          end
         end
       
         opts.on('-D', '--debug', 'Enable debug logging') do

data/lib/cem_acpt/config/base.rb

@@ -13,8 +13,9 @@ module CemAcpt
     using CemAcpt::CoreExt::ExtendedHash
 
     # Base class for other config classes
+    # Child classes should provide the following constant:
+    #   - VALID_KEYS - provide an array of valid top-level keys for the config as symbols
     # Child classes should implement the following methods:
-    #   - valid_keys - provide an array of valid top-level keys for the config as symbols
     #   - defaults - provide a hash of default values for the config
     #   - env_var_prefix - provide a string to prefix environment variables with (defaults to 'CEM_ACPT').
     #                      This will be converted to uppercase, have all non-alphanumeric characters replaced with
@@ -22,6 +23,22 @@ module CemAcpt
     #                      variable name.
     # However, they can override any of the methods in this class.
     class Base
+      BASE_VALID_KEYS = %i[
+        ci_mode
+        config_file
+        log_level
+        log_file
+        log_format
+        no_destroy_nodes
+        no_ephemeral_ssh_key
+        platform
+        provisioner
+        quiet
+        terraform
+        user_config
+        verbose
+      ].freeze
+
       attr_reader :config, :env_vars
 
       def initialize(opts: {}, config_file: nil, load_user_config: true)
@@ -36,7 +53,11 @@ module CemAcpt
 
       # @return [Array<Symbol>] Valid top-level keys for the config
       def valid_keys
-        []
+        if self.class.const_defined?(:VALID_KEYS)
+          (BASE_VALID_KEYS + self.class.const_get(:VALID_KEYS)).uniq
+        else
+          BASE_VALID_KEYS
+        end
       end
 
       # @return [Hash] The default configuration

data/lib/cem_acpt/config/cem_acpt.rb

@@ -8,30 +8,13 @@ module CemAcpt
     class CemAcpt < Base
       VALID_KEYS = %i[
         actions
-        ci_mode
-        config_file
         image_name_builder
-        log_level
-        log_file
-        log_format
         module_dir
         node_data
-        no_destroy_nodes
-        no_ephemeral_ssh_key
-        platform
-        provisioner
-        quiet
-        terraform
         test_data
         tests
-        user_config
-        verbose
       ].freeze
 
-      def valid_keys
-        VALID_KEYS
-      end
-
       # The default configuration
       def defaults
         {

data/lib/cem_acpt/config/cem_acpt_image.rb

@@ -7,26 +7,13 @@ module CemAcpt
     # Holds the configuration for cem_acpt_image
     class CemAcptImage < Base
       VALID_KEYS = %i[
-        ci_mode
-        config_file
+        cem_acpt_image
+        dry_run
         images
-        log_level
-        log_file
-        log_format
-        no_destroy_nodes
-        no_ephemeral_ssh_key
-        platform
-        provisioner
-        quiet
-        terraform
-        user_config
-        verbose
+        image_name_filter
+        no_build_images
       ].freeze
 
-      def valid_keys
-        VALID_KEYS
-      end
-
       def env_var_prefix
         'CEM_ACPT_IMAGE'
       end
@@ -34,12 +21,19 @@ module CemAcpt
       # The default configuration
       def defaults
         {
+          cem_acpt_image: {
+            no_linux: false,
+            no_windows: false,
+          },
           ci_mode: false,
           config_file: nil,
+          dry_run: false,
           images: {},
+          image_name_filter: Regexp.new('.*'),
           log_level: 'info',
           log_file: nil,
           log_format: 'text',
+          no_destroy_nodes: false,
           no_ephemeral_ssh_key: false,
           platform: {
             name: 'gcp',

data/lib/cem_acpt/image_builder/provision_commands.rb

@@ -80,6 +80,7 @@ module CemAcpt
           'centos' => 'EnterpriseLinuxFamily',
           'rhel' => 'EnterpriseLinuxFamily',
           'alma' => 'EnterpriseLinuxFamily',
+          'oel' => 'EnterpriseLinuxFamily',
           'windows' => 'WindowsFamily',
         }.freeze
 

data/lib/cem_acpt/image_builder.rb

@@ -14,7 +14,17 @@ module CemAcpt
   # This module contains the classes and methods for building test node images
   module ImageBuilder
     def self.build_images(config)
-      TerraformBuilder.new(config).run
+      include CemAcpt::Logging
+
+      builder = TerraformBuilder.new(config)
+      builder.run
+      logger.info('ImageBuilder') { "Image builder finished after #{builder.duration} seconds" }
+      if builder.exit_code.zero?
+        logger.info('ImageBuilder') { 'Image builder finished successfully' }
+      else
+        logger.error('ImageBuilder') { "Image builder finished with exit code #{builder.exit_code}" }
+      end
+      exit builder.exit_code
     end
 
     # This class builds test node images using Terraform
@@ -27,6 +37,8 @@ module CemAcpt
       DEFAULT_VARS_FILE = 'imagevars.json'
       include CemAcpt::Logging
 
+      attr_reader :exit_code, :duration
+
       def initialize(config)
         @config = config
         @image_terraform_dir = File.join(config.get('terraform.dir'), 'image', config.get('platform.name'))
@@ -36,7 +48,7 @@ module CemAcpt
         @linux_tfvars = { node_data: {} }
         @windows_tfvars = { node_data: {} }
         @duration = 0
-        @exit_code = 0
+        @exit_code = 1
         @private_key = nil
         @public_key = nil
       end
@@ -44,17 +56,19 @@ module CemAcpt
       def run
         @start_time = Time.now
         logger.with_ci_group("CemAcptImage v#{CemAcpt::VERSION} run started at #{@start_time}") do
-          logger.info('CemAcptImage') { "Using working directory: #{@working_dir}..." }
-          keep_terminal_alive
           @all_tfvars = new_tfvars(@config)
-          @working_dir = new_working_dir
           @linux_tfvars, @windows_tfvars = divide_tfvars_by_os(@all_tfvars)
-          save_vars_to_file!('linux', @linux_tfvars)
-          save_vars_to_file!('windows', @windows_tfvars)
-          terraform_init
           image_types = []
           image_types << [@linux_tfvars, 'linux'] unless no_linux?
           image_types << [@windows_tfvars, 'windows'] unless no_windows?
+          return dry_run(image_types) if @config.get('dry_run')
+
+          @working_dir = new_working_dir
+          logger.info('CemAcptImage') { "Using working directory: #{@working_dir}..." }
+          keep_terminal_alive
+          save_vars_to_file!('linux', @linux_tfvars)
+          save_vars_to_file!('windows', @windows_tfvars)
+          terraform_init
           image_types.each do |tfvars, os_str|
             terraform_plan(os_str, tfvars, DEFAULT_PLAN_NAME)
             begin
@@ -63,14 +77,27 @@ module CemAcpt
               output.each do |instance_name, data|
                 logger.info('CemAcptImage') { "Stopping instance #{instance_name}..." }
                 @exec.run('compute', 'instances', 'stop', instance_name)
-                deprecate_old_images_in_family(data['image_family'])
-                create_image_from_disk(data['disk_link'], image_name_from_image_family(data['image_family']), data['image_family'])
+                unless @config.get('no_build_images')
+                  deprecate_old_images_in_family(data['image_family'])
+                  create_image_from_disk(data['disk_link'], image_name_from_image_family(data['image_family']), data['image_family'])
+                end
+                @exit_code = 0
               end
             ensure
-              terraform_destroy(os_str, tfvars)
+              terraform_destroy(os_str, tfvars) unless @config.get('no_destroy_nodes')
             end
           end
         end
+      rescue StandardError => e
+        logger.error('CemAcptImage') { "Image builder failed with error: #{e}" }
+        logger.verbose('CemAcptImage') { e.backtrace.join("\n") }
+        @exit_code = 1
+      ensure
+        if @start_time
+          @duration = Time.now - @start_time
+        else
+          @duration = 0
+        end
       end
 
       private
@@ -107,6 +134,15 @@ module CemAcpt
         @linux_tfvars[:node_data].empty? || @config.get('cem_acpt_image.no_linux')
       end
 
+      def dry_run(image_types)
+        logger.info('CemAcptImage') { 'Dry run mode enabled. No images will be built.' }
+        image_types.each do |tfvars, os_str|
+          logger.info('CemAcptImage') { "Dry run for #{os_str}..." }
+          logger.info('CemAcptImage') { "Terraform vars:\n#{JSON.pretty_generate(tfvars)}" }
+        end
+        @exit_code = 0
+      end
+
       def terraform
         return @terraform if defined?(@terraform)
 
@@ -233,6 +269,8 @@ module CemAcpt
         tfvars[:private_key] = private_key if private_key
         tfvars[:public_key] = public_key if public_key
         config.get('images').each do |image_name, image|
+          next if config.get('image_name_filter')&.respond_to?(:match?) && !config.get('image_name_filter').match?(image_name)
+
           platform = new_platform(config, **tfvars)
           tfvars.merge!(platform.platform_data)
           provision_commands = CemAcpt::ImageBuilder::ProvisionCommands.provision_commands(

data/lib/cem_acpt/provision/terraform/linux.rb

@@ -40,7 +40,23 @@ module CemAcpt
             commands << "sudo systemctl start #{file} && sudo systemctl enable #{file}"
           end
         end
-        commands << "sudo #{puppet_bin_path} apply #{destination_provision_directory}/#{puppet_manifest_file}"
+        commands << apply_command
+      end
+
+      private
+
+      def apply_command
+        cmd = [
+          'sudo',
+          puppet_bin_path,
+          'apply',
+          '--logdest',
+          'console,/opt/cem_acpt/provision_apply.log',
+        ]
+        cmd << '--debug' if @config.debug?
+        cmd << '--verbose' if @config.verbose?
+        cmd << "#{destination_provision_directory}/#{puppet_manifest_file}"
+        cmd.join(' ')
       end
     end
   end

data/lib/cem_acpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CemAcpt
-  VERSION = '0.7.1'
+  VERSION = '0.7.3'
 end

data/lib/terraform/gcp/linux/goss/puppet_idempotent.yaml

@@ -1,9 +1,9 @@
 command:
   puppet_idempotent:
-    exec: 'sudo /opt/puppetlabs/puppet/bin/puppet apply --verbose --detailed-exitcodes /opt/cem_acpt/manifest.pp'
+    exec: 'sudo /opt/puppetlabs/puppet/bin/puppet apply --logdest console,/opt/cem_acpt/idempotent_apply.log --debug --verbose --detailed-exitcodes /opt/cem_acpt/manifest.pp'
     timeout: 300000 # 5 mins in miliseconds
     stdout:
-      - "Applied catalog in"
+      - "/Applied catalog in.*/"
     stderr:
       - ""
     exit-status: 0

data/lib/terraform/gcp/linux/goss/puppet_noop.yaml

@@ -1,6 +1,6 @@
 command:
   puppet_noop:
-    exec: 'sudo /opt/puppetlabs/puppet/bin/puppet apply --verbose --detailed-exitcodes --noop /opt/cem_acpt/manifest.pp'
+    exec: 'sudo /opt/puppetlabs/puppet/bin/puppet apply --logdest console,/opt/cem_acpt/noop_apply.log --debug --verbose --detailed-exitcodes --noop /opt/cem_acpt/manifest.pp'
     timeout: 300000 # 5 mins in miliseconds
     stdout:
       - "Applied catalog in"

data/lib/terraform/gcp/windows/goss/puppet_idempotent.yaml

@@ -0,0 +1,9 @@
+command:
+  puppet_idempotent:
+    exec: 'C:\Program Files\Puppet Labs\Puppet\bin\puppet apply --verbose --detailed-exitcodes C:\cem_acpt\manifest.pp'
+    timeout: 300000 # 5 mins in miliseconds
+    stdout:
+      - "Applied catalog in"
+    stderr:
+      - ""
+    exit-status: 0

data/lib/terraform/gcp/windows/goss/puppet_noop.yaml

@@ -0,0 +1,12 @@
+command:
+  puppet_noop:
+    exec: 'C:\Program Files\Puppet Labs\Puppet\bin\puppet apply --verbose --detailed-exitcodes --noop C:\cem_acpt\manifest.pp'
+    timeout: 300000 # 5 mins in miliseconds
+    stdout:
+      - "Applied catalog in"
+    stderr:
+      - ""
+    exit-status:
+      or:
+        - 0
+        - 2

data/lib/terraform/gcp/windows/main.tf

@@ -0,0 +1,89 @@
+terraform {
+  required_providers {
+      google = {
+          source = "hashicorp/google"
+          version = "4.59.0"
+      }
+  }
+}
+
+variable "credentials_file" {
+    type = string
+}
+
+variable "project" {
+    type = string
+}
+
+variable "region" {
+    type = string
+}
+
+variable "zone" {
+    type = string
+}
+
+variable "subnetwork" {
+    type = string
+}
+
+variable "node_data" {
+    type = map(object({
+        machine_type = string
+        image = string
+        disk_size = number
+        test_name = string
+    }))
+}
+
+provider "google" {
+    credentials = file(var.credentials_file)
+    project = var.project
+    region = var.region
+    zone = var.zone
+}
+
+resource "google_compute_instance" "acpt-test-node" {
+    provider = google
+    for_each = var.node_data
+    name = each.key
+    machine_type = each.value.machine_type
+
+    boot_disk {
+        initialize_params {
+            image = each.value.image
+            size = each.value.disk_size
+            type = "pd-standard"
+        }
+    }
+
+    scheduling {
+        preemptible = true
+        automatic_restart = false
+        provisioning_model = "SPOT"
+        instance_termination_action = "DELETE"
+    }
+
+    network_interface {
+        subnetwork = var.subnetwork
+        access_config {
+            network_tier = "STANDARD"
+        }
+    }
+
+    metadata = {
+        "enable-oslogin" = "TRUE"
+        "cem-acpt-test" = each.value.test_name
+    }
+
+    tags = [ "cem-acpt-test-node" ]
+}
+
+output "instance_name_ip" {
+    value = {
+        for k, v in google_compute_instance.acpt-test-node : v.name => {
+            ip = v.network_interface.0.access_config.0.nat_ip,
+            test_name = v.metadata.cem-acpt-test,
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cem_acpt
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.3
 platform: ruby
 authors:
 - puppetlabs
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-17 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async-http
@@ -210,6 +210,9 @@ files:
 - lib/terraform/gcp/linux/systemd/goss-idempotent.service
 - lib/terraform/gcp/linux/systemd/goss-noop.service
 - lib/terraform/gcp/windows/.keep
+- lib/terraform/gcp/windows/goss/puppet_idempotent.yaml
+- lib/terraform/gcp/windows/goss/puppet_noop.yaml
+- lib/terraform/gcp/windows/main.tf
 - lib/terraform/image/gcp/linux/main.tf
 - lib/terraform/image/gcp/windows/.keep
 - sample_config.yaml